From 757cb2d0ae4699f50d1dad205a777376e60d27d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 10:13:03 +0200 Subject: [PATCH 01/15] wip: S5 runtime invariant --- crates/perry-runtime/src/arena/block.rs | 4 + crates/perry-runtime/src/gc/alloc_point.rs | 341 ++++++++++++++++++ crates/perry-runtime/src/gc/cycle.rs | 34 ++ crates/perry-runtime/src/gc/instruments.rs | 1 + crates/perry-runtime/src/gc/mod.rs | 28 +- crates/perry-runtime/src/gc/policy.rs | 265 +++++++++----- crates/perry-runtime/src/gc/roots.rs | 3 +- .../perry-runtime/src/gc/roots/stack_maps.rs | 68 ++-- .../src/gc/roots/stack_maps_frame_verify.rs | 144 ++++++++ .../src/gc/roots/stack_maps_lazy.rs | 15 + crates/perry-runtime/src/gc/scan_fallback.rs | 40 +- crates/perry-runtime/src/gc/schedule.rs | 7 +- .../src/gc/tests/alloc_point_invariant.rs | 240 ++++++++++++ .../perry-runtime/src/gc/tests/debt_pacer.rs | 8 +- .../src/gc/tests/env_knob_parse.rs | 39 +- crates/perry-runtime/src/gc/tests/mod.rs | 1 + crates/perry-runtime/src/gc/tests/support.rs | 14 + .../compile/optimized_libs/freshness.rs | 1 + 18 files changed, 1064 insertions(+), 189 deletions(-) create mode 100644 crates/perry-runtime/src/gc/alloc_point.rs create mode 100644 crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs create mode 100644 crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs diff --git a/crates/perry-runtime/src/arena/block.rs b/crates/perry-runtime/src/arena/block.rs index ab15913f07..c9c83fadaf 100644 --- a/crates/perry-runtime/src/arena/block.rs +++ b/crates/perry-runtime/src/arena/block.rs @@ -961,6 +961,10 @@ pub(crate) unsafe fn arena_cell_alloc(arena: *mut Arena, size: usize, align: usi // on the first cut of #7022, where the reservation still happened inside // `alloc_fresh_block` under the borrow.) let fresh = reserve_arena_block(size); + // Decision 10 of RFC deferred collection: growth an unsafe zone forced + // (no poll or valve can collect inside one). Diagnostic; one relaxed load + // on the block-acquire path only. + crate::gc::note_block_if_unsafe_zone(block_size_for(size)); let _borrow = ArenaBorrowGuard::new(); (*arena).install_reserved_block(fresh); diff --git a/crates/perry-runtime/src/gc/alloc_point.rs b/crates/perry-runtime/src/gc/alloc_point.rs new file mode 100644 index 0000000000..51c70add6a --- /dev/null +++ b/crates/perry-runtime/src/gc/alloc_point.rs @@ -0,0 +1,341 @@ +//! The allocation-point invariant (RFC deferred collection, step S5; D1/D2 in +//! `docs/src/internals/rfc-deferred-collection.md`). +//! +//! > **D2.** An allocation may take a new block, arm the poll word, run +//! > heap-only budgeted work (mark propagation, weak processing, sweep, +//! > reclaim), or run a conservative non-moving collection in exactly two +//! > cases: the valve fires, or the OldReclaim arm becomes due. It never starts +//! > a phase that reads frame roots *precisely*, and it never starts a moving +//! > phase. +//! +//! "Allocation point" is not a guess about the caller. It is exactly the +//! dynamic extent of `gc_check_trigger`'s evaluation, which every allocation +//! slow path funnels into (the arena block-full path, every `gc_malloc`, the +//! explicit JSON mid-parse checks, and the root-lock flush of a deferred +//! `CheckTrigger`). [`AllocationPointGuard`] marks that extent; everything the +//! collector does inside it is held to D2: +//! +//! * a synchronous collection started there must have the conservative scan +//! forced ([`assert_d2_synchronous_collection`]) — which also makes the +//! copying minor ineligible, so it cannot move; +//! * a budgeted cycle whose next work is a frame-root phase (`RootScan`, +//! `FinalRootRemark`) is PARKED instead of stepped: the step returns, and the +//! poll word is armed so the next declared poll serves the phase with a +//! precise root set (`policy::gc_safepoint_moving_minor`). +//! +//! The one exception is the parked-cycle valve ([`parked_valve_due`]): a +//! program that allocates [`super::GC_MOVING_DEFER_SLACK_BYTES`] past the +//! point a cycle parked, without reaching a single poll, has its root phase +//! served at the allocation point. It is counted, it is a hard CI failure on +//! the gap suite and the ratchet probes (decision 5), and its soundness rests +//! on today's codegen treating every allocating call as a statepoint — see the +//! note on [`note_parked_valve_fired`]. +//! +//! Also here, because they share the "what did allocation do" question: +//! the valve ledger (`PERRY_GC_VALVE_LEDGER`, decision 5) and the bytes the +//! arena grew by inside `GC_UNSAFE_ZONES` (decision 10, diagnostic only). + +use std::cell::Cell; +use std::io::Write; +use std::sync::atomic::{AtomicU64, Ordering}; + +thread_local! { + /// Depth of `gc_check_trigger` evaluations on this thread. Nesting is + /// possible (a root-lock flush inside an evaluation), so a counter rather + /// than a flag. + static ALLOC_POINT_DEPTH: Cell = const { Cell::new(0) }; + /// `Some(arena_total)` while a budgeted cycle is parked at a frame-root + /// phase, recording the arena size when it parked (the valve measures its + /// slack from here, the way the nursery deferral measures from + /// `GC_SAFEPOINT_DEFER_ARENA_BASE`). + static PARKED_AT: Cell> = const { Cell::new(None) }; +} + +/// RAII marker for the dynamic extent of one allocation-point trigger +/// evaluation. +pub(super) struct AllocationPointGuard(()); + +impl AllocationPointGuard { + #[inline] + pub(super) fn enter() -> Self { + ALLOC_POINT_DEPTH.with(|depth| depth.set(depth.get() + 1)); + Self(()) + } +} + +impl Drop for AllocationPointGuard { + #[inline] + fn drop(&mut self) { + ALLOC_POINT_DEPTH.with(|depth| depth.set(depth.get().saturating_sub(1))); + } +} + +/// Whether the current thread is inside an allocation-point trigger +/// evaluation. +#[inline] +pub(super) fn at_allocation_point() -> bool { + ALLOC_POINT_DEPTH.with(|depth| depth.get() != 0) +} + +/// Run `f` with the allocation-point marker lifted. For the parked-cycle +/// valve only: it is the one path that deliberately serves a root phase from +/// an allocation point, and it is counted. +pub(super) fn with_allocation_point_lifted(f: impl FnOnce() -> R) -> R { + let saved = ALLOC_POINT_DEPTH.with(|depth| depth.replace(0)); + let result = f(); + ALLOC_POINT_DEPTH.with(|depth| depth.set(saved)); + result +} + +/// D2's enforcement for synchronous collections: a collection that begins at +/// an allocation point must scan conservatively (and therefore cannot move). +/// +/// Called at the two synchronous chokepoints (`gc_collect_minor_with_trigger_inner` +/// and `gc_collect_full_mark_sweep_with_trigger`). Every allocation-point arm +/// that collects — OldReclaim, the nursery valve, the polls-off direct minor, +/// the emergency reclaim — takes `ManualGcScanGuard::force_full_scan` first, +/// so this is structurally unreachable. It panics in every build rather than +/// healing: a heal path nothing can reach is an untested mode (the kill +/// policy), and the check is one thread-local read per collection. +#[inline] +pub(super) fn assert_d2_synchronous_collection() { + if !at_allocation_point() { + return; + } + if matches!( + super::roots::conservative_stack_scan_decision(), + super::roots::ConservativeStackScanDecision::Scan + ) { + return; + } + D2_VIOLATIONS.fetch_add(1, Ordering::Relaxed); + panic!( + "perry GC invariant D2 violated: a precise-root collection began at an \ + allocation point. Allocation may only arm the poll, run heap-only \ + work, or run a conservative non-moving collection (the valve, \ + OldReclaim, emergency reclaim). See docs/src/internals/rfc-deferred-collection.md." + ); +} + +static D2_VIOLATIONS: AtomicU64 = AtomicU64::new(0); +static ROOT_PHASES_PARKED: AtomicU64 = AtomicU64::new(0); +static ROOT_PHASES_SERVED_AT_POLL: AtomicU64 = AtomicU64::new(0); +static PARKED_VALVE_FIRES: AtomicU64 = AtomicU64::new(0); +static OWED_REQUESTS_ROUTED: AtomicU64 = AtomicU64::new(0); +static OWED_REQUESTS_SERVED: AtomicU64 = AtomicU64::new(0); +static UNSAFE_ZONE_GROWTH_BYTES: AtomicU64 = AtomicU64::new(0); +static UNSAFE_ZONE_GROWTH_EVENTS: AtomicU64 = AtomicU64::new(0); + +/// An allocation point found the active budgeted cycle about to read frame +/// roots. Arms the poll (via `arm`) the first time for this park and records +/// where it parked. Returns whether this call started a new park. +pub(super) fn park_root_phase(arena_total: usize, arm: impl FnOnce()) -> bool { + PARKED_AT.with(|parked| { + if parked.get().is_some() { + return false; + } + parked.set(Some(arena_total)); + ROOT_PHASES_PARKED.fetch_add(1, Ordering::Relaxed); + arm(); + true + }) +} + +/// The cycle is no longer waiting at a frame-root phase (it was served, or it +/// ended). Idempotent. +#[inline] +pub(super) fn clear_park() { + PARKED_AT.with(|parked| parked.set(None)); +} + +/// Whether a cycle is currently parked at a frame-root phase on this thread. +#[inline] +pub(super) fn root_phase_parked() -> bool { + PARKED_AT.with(|parked| parked.get().is_some()) +} + +/// The parked cycle has waited `slack` arena bytes past its park point with no +/// poll to serve it. +pub(super) fn parked_valve_due(arena_total: usize, slack: usize) -> bool { + PARKED_AT.with(|parked| { + parked + .get() + .is_some_and(|base| arena_total >= base.saturating_add(slack)) + }) +} + +/// A declared poll served a parked root phase. +pub(super) fn note_root_phase_served_at_poll() { + ROOT_PHASES_SERVED_AT_POLL.fetch_add(1, Ordering::Relaxed); +} + +/// The parked-cycle valve fired: a budgeted cycle's frame-root phase ran at an +/// allocation point because no poll was reached within the slack. +/// +/// ★ This is the one place S5 still reads frame roots precisely at an +/// allocation point, and it is sound for the same reason A-assist was sound +/// before S5: codegen still treats every allocating call as a statepoint, so +/// the frame that allocated is mapped. That stops being true at S6 (L2b makes +/// `AllocOnly` helpers leaves). Before S6 this arm must either be proven +/// unreachable — the gate in `scripts/gc_valve_ledger_check.py` holds it at +/// zero on the gap suite and the ratchet probes — or be replaced by an abort of +/// the parked cycle followed by the conservative valve. Budgeted cycles are +/// classifier-mode and cannot take the conservative scan themselves. +pub(super) fn note_parked_valve_fired() { + PARKED_VALVE_FIRES.fetch_add(1, Ordering::Relaxed); + if super::gc_diag_enabled() { + eprintln!( + "[gc-alloc-point] parked_valve fired count={}", + PARKED_VALVE_FIRES.load(Ordering::Relaxed) + ); + } +} + +/// A root-lock exit had a collection (not just a trigger check) deferred to +/// it, and routed it to the poll instead of running it (the RFC's "D stops +/// collecting"). +pub(super) fn note_owed_request_routed() { + OWED_REQUESTS_ROUTED.fetch_add(1, Ordering::Relaxed); +} + +pub(super) fn note_owed_request_served() { + OWED_REQUESTS_SERVED.fetch_add(1, Ordering::Relaxed); +} + +/// Decision 10: an arena block was taken while `GC_UNSAFE_ZONES` was held, so +/// nothing — no poll, no valve — could collect. Diagnostic only; the arena's +/// block-acquire slow path is the only caller. +#[inline] +pub(crate) fn note_block_if_unsafe_zone(bytes: usize) { + if !super::gc_blocked_by_unsafe_zone() { + return; + } + UNSAFE_ZONE_GROWTH_BYTES.fetch_add(bytes as u64, Ordering::Relaxed); + UNSAFE_ZONE_GROWTH_EVENTS.fetch_add(1, Ordering::Relaxed); +} + +/// Snapshot of this module's counters, for the exit summary and tests. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AllocPointCounters { + pub d2_violations: u64, + pub root_phases_parked: u64, + pub root_phases_served_at_poll: u64, + pub parked_valve_fires: u64, + pub owed_requests_routed: u64, + pub owed_requests_served: u64, + pub unsafe_zone_growth_bytes: u64, + pub unsafe_zone_growth_events: u64, +} + +pub fn alloc_point_counters() -> AllocPointCounters { + AllocPointCounters { + d2_violations: D2_VIOLATIONS.load(Ordering::Relaxed), + root_phases_parked: ROOT_PHASES_PARKED.load(Ordering::Relaxed), + root_phases_served_at_poll: ROOT_PHASES_SERVED_AT_POLL.load(Ordering::Relaxed), + parked_valve_fires: PARKED_VALVE_FIRES.load(Ordering::Relaxed), + owed_requests_routed: OWED_REQUESTS_ROUTED.load(Ordering::Relaxed), + owed_requests_served: OWED_REQUESTS_SERVED.load(Ordering::Relaxed), + unsafe_zone_growth_bytes: UNSAFE_ZONE_GROWTH_BYTES.load(Ordering::Relaxed), + unsafe_zone_growth_events: UNSAFE_ZONE_GROWTH_EVENTS.load(Ordering::Relaxed), + } +} + +#[cfg(test)] +pub(crate) fn reset_alloc_point_counters() { + for counter in [ + &D2_VIOLATIONS, + &ROOT_PHASES_PARKED, + &ROOT_PHASES_SERVED_AT_POLL, + &PARKED_VALVE_FIRES, + &OWED_REQUESTS_ROUTED, + &OWED_REQUESTS_SERVED, + &UNSAFE_ZONE_GROWTH_BYTES, + &UNSAFE_ZONE_GROWTH_EVENTS, + ] { + counter.store(0, Ordering::Relaxed); + } + clear_park(); +} + +/// The `[gc-alloc-point]` exit line (`PERRY_GC_DIAG=1`). +pub(super) fn alloc_point_exit_line() -> String { + let c = alloc_point_counters(); + format!( + "[gc-alloc-point] valve_fires={} parked_valve_fires={} old_reclaim_alloc_point={} \ + emergency_reclaims={} root_phases_parked={} root_phases_served_at_poll={} \ + owed_requests_routed={} owed_requests_served={} safepoint_drains={} \ + unsafe_zone_growth_bytes={} unsafe_zone_growth_events={}", + super::scan_fallback::scan_fallback_count_any_thread( + super::ConservativeScanSite::NurseryChurnSlackValve + ), + c.parked_valve_fires, + super::scan_fallback::scan_fallback_count_any_thread( + super::ConservativeScanSite::OldReclaimAllocPoint + ), + super::scan_fallback::scan_fallback_count_any_thread( + super::ConservativeScanSite::EmergencyReclaim + ), + c.root_phases_parked, + c.root_phases_served_at_poll, + c.owed_requests_routed, + c.owed_requests_served, + super::scan_fallback::safepoint_drain_total_any_thread(), + c.unsafe_zone_growth_bytes, + c.unsafe_zone_growth_events, + ) +} + +/// Decision 5's ledger: with `PERRY_GC_VALVE_LEDGER=` set, every process +/// appends one line at exit recording whether an allocation-point valve fired. +/// +/// The line is written whether or not anything fired — that is the "counter +/// asserting the check actually ran": the gate (`scripts/gc_valve_ledger_check.py`) +/// requires one line per test it ran, so a harness that stopped passing the +/// variable, or a binary whose exit path skipped the funnel, reads as a +/// failure rather than as a clean run. Append-only, one `write` per line, so +/// concurrent processes do not interleave within a line. +pub(super) fn write_valve_ledger_line() { + static WRITTEN: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + let Some(path) = std::env::var_os("PERRY_GC_VALVE_LEDGER").filter(|p| !p.is_empty()) else { + return; + }; + if !crate::native_handle::is_main_thread_or_unrecorded() { + return; + } + if WRITTEN.swap(true, Ordering::SeqCst) { + return; + } + let c = alloc_point_counters(); + let valve = + super::scan_fallback::scan_fallback_count_any_thread(super::ConservativeScanSite::NurseryChurnSlackValve); + let exe = std::env::current_exe() + .ok() + .and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())) + .unwrap_or_else(|| "?".to_string()); + let line = format!( + "v1 exe={exe} pid={} valve_fires={valve} parked_valve_fires={} d2_violations={} \ + old_reclaim_alloc_point={} root_phases_served_at_poll={} safepoint_drains={}\n", + std::process::id(), + c.parked_valve_fires, + c.d2_violations, + super::scan_fallback::scan_fallback_count_any_thread( + super::ConservativeScanSite::OldReclaimAllocPoint + ), + c.root_phases_served_at_poll, + super::scan_fallback::safepoint_drain_total_any_thread(), + ); + if let Ok(mut file) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + { + let _ = file.write_all(line.as_bytes()); + } +} + +/// Move the park point back so the parked-cycle valve is due on the next +/// allocation point, without allocating the slack for real. +#[cfg(test)] +pub(crate) fn test_set_park_base(base: usize) { + PARKED_AT.with(|parked| parked.set(Some(base))); +} diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 07165d1d33..31f6ded372 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -731,6 +731,30 @@ impl GcCycleState { self.phase } + /// Whether this cycle's next step reads frame roots: the root scan, or the + /// budgeted final remark (the barrier-seed drain that precedes it runs in + /// the same step, so it counts once it is the current subphase and the + /// remark is due next). RFC deferred collection S5 (D2): an allocation + /// point never starts such a step on a budgeted cycle. + pub(super) fn next_step_reads_frame_roots(&self) -> bool { + match self.phase { + GcCyclePhase::RootScan => true, + GcCyclePhase::AtomicFinalize => self.atomic_finalize.as_ref().is_some_and(|state| { + state.subphase == AtomicFinalizeSubphase::FinalRootRemark + }), + _ => false, + } + } + + /// D2's guard inside the stepper: a BUDGETED cycle does not enter a + /// frame-root phase from an allocation point. Synchronous cycles are never + /// refused here — the only ones that start at an allocation point force the + /// conservative scan, which `alloc_point::assert_d2_synchronous_collection` + /// checks, and refusing one would spin `run_to_completion` forever. + fn frame_root_phase_refused(&self) -> bool { + self.progress_kind.is_budgeted() && super::alloc_point::at_allocation_point() + } + #[cfg(test)] pub(super) fn atomic_finalize_subphase_for_tests(&self) -> Option<&'static str> { let subphase = self.atomic_finalize.as_ref()?.subphase; @@ -919,6 +943,10 @@ impl GcCycleState { } fn step_root_scan(&mut self, budget: GcWorkBudget) { + if self.frame_root_phase_refused() { + // Parked: the caller arms the poll (`policy.rs`). + return; + } let valid_ptrs = self.valid_ptrs.as_ref().expect("valid pointer set built"); let consider_evacuation = self .minor @@ -1101,6 +1129,12 @@ impl GcCycleState { | AtomicFinalizeSubphase::RememberedSetRebuild | AtomicFinalizeSubphase::WeakProcessing ); + if subphase == AtomicFinalizeSubphase::FinalRootRemark + && self.frame_root_phase_refused() + { + // Parked before the remark; see `step_root_scan`. + break; + } let sub_budget = if sliced { budget.work_units } else { diff --git a/crates/perry-runtime/src/gc/instruments.rs b/crates/perry-runtime/src/gc/instruments.rs index 778579e39e..5ffa37da0c 100644 --- a/crates/perry-runtime/src/gc/instruments.rs +++ b/crates/perry-runtime/src/gc/instruments.rs @@ -506,6 +506,7 @@ pub(crate) const INSTRUMENT_KNOBS: &[&str] = &[ "PERRY_ALLOC_SITE_SAMPLE", "PERRY_GC_VERIFY_MARK", "PERRY_GC_VERIFY_CLASSIFIER", + "PERRY_GC_VERIFY_FRAMES", "PERRY_STACK_SYMBOLS", ]; diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 757fa71150..c4f59fadc2 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -35,6 +35,11 @@ use std::time::{Duration, Instant}; mod types; pub use types::*; +/// RFC deferred collection S5: allocation never begins a precise or moving +/// collection phase (D2). See the module docs. +mod alloc_point; +pub(crate) use alloc_point::note_block_if_unsafe_zone; +pub use alloc_point::{alloc_point_counters, AllocPointCounters}; mod json_defer; mod policy; pub(crate) use json_defer::JsonParseAllocation; @@ -370,10 +375,11 @@ fn gc_collect_minor_with_trigger_inner( escalation: FullEscalation, copying: CopyingFastPath, ) -> GcCollectOutcome { - // PERRY_GC_SAFEPOINT_ONLY: held for the whole collection so every - // consumer of the scan decision (root scan, copying eligibility, - // evacuation pinning, verifier) sees the same healed answer. - let _contract_heal = policy::contract_scan_heal_guard(); + // D2 (RFC deferred collection S5): a collection that begins at an + // allocation point must already have the conservative scan forced, which + // also makes the copying minor ineligible. Structurally unreachable; + // panics rather than heals. + alloc_point::assert_d2_synchronous_collection(); gc_drain_active_budgeted_cycle(); // Barriers-off ⇒ the remembered set is not being maintained, and a // minor's black-leafed old parents would hide live children. Route @@ -826,10 +832,8 @@ fn gc_collect_full_mark_sweep_with_trigger(trigger: GcTriggerSnapshot) -> GcColl // cannot be deferred any further than this. roots::ensure_stack_maps_built(); - // PERRY_GC_SAFEPOINT_ONLY: see gc_collect_minor_with_trigger. Manual - // gc() engages its own force_full_scan first, which this detects as - // already-Scan and no-ops. - let _contract_heal = policy::contract_scan_heal_guard(); + // D2: see gc_collect_minor_with_trigger_inner. + alloc_point::assert_d2_synchronous_collection(); gc_drain_active_budgeted_cycle(); GC_TRIGGER_BUMPED.with(|c| c.set(false)); diag_sites::full_started(diag_sites::take_full_site(), trigger.kind); @@ -1431,6 +1435,7 @@ pub extern "C" fn js_gc_release_current_thread_collection_side_allocations() { diag_sites::report_charges("exit"); diag_sites::report_primitive_dispatch("exit"); emit_incremental_liveness_diag(); + alloc_point::write_valve_ledger_line(); emit_schedule_liveness_verdict(); } @@ -1480,6 +1485,13 @@ fn emit_incremental_liveness_diag() { poll_arm::poll_armed_count(), trace::forwarded_stub_membership_recoveries(), ); + eprintln!("{}", alloc_point::alloc_point_exit_line()); + let (frames_verified, unmapped_generated) = roots::frame_verify_counters(); + eprintln!( + "[gc-verify-frames] armed={} frames_verified={frames_verified} \ + unmapped_generated={unmapped_generated}", + roots::stack_maps_frame_verify_active(), + ); idle_reclaim::emit_diag(); idle_compact::emit_diag(); arena_right_size::emit_diag(); diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index ee6adf6909..267de78090 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1400,92 +1400,18 @@ crate::perry_thread_local! { /// Meaningless while `GC_SAFEPOINT_PENDING` is false. pub(super) static GC_SAFEPOINT_DEFER_ARENA_BASE: Cell = const { Cell::new(0) }; /// True while a DECLARED safepoint drain is running: a loop back-edge - /// poll, the outermost microtask-pump moving minor, or an explicit - /// `gc()`. Consumed by the `PERRY_GC_SAFEPOINT_ONLY` contract assert in - /// the root-scan subphase. + /// poll or the outermost microtask-pump moving minor. Diagnostic only + /// (the copying minor's trace line reports it). The property the old + /// `PERRY_GC_SAFEPOINT_ONLY` contract asserted with it — "a precise-root + /// collection begins only at a declared safepoint" — is now the default + /// invariant, enforced at the allocation point (`gc/alloc_point.rs`). pub(super) static GC_AT_DECLARED_SAFEPOINT: Cell = const { Cell::new(false) }; -} - -/// `PERRY_GC_SAFEPOINT_ONLY` — research contract for the native-root modes -/// (`exp/stackmap-viability`): a collection that skips the conservative stack -/// scan consumes only precise roots, and with native stack maps active those -/// roots exist only at mapped PCs — so such a collection may begin only at a -/// declared safepoint; anywhere else it must scan conservatively. Codegen -/// reads the same env to stop emitting statepoints around audited -/// allocate-but-never-reenter helpers; the enforcement in `cycle.rs` is what -/// turns the property from emergent (every possibly-collecting call happens -/// to be mapped) into enforced. -/// -/// `1`/`on`/`true` — HEAL: an undeclared precise-root cycle has the -/// conservative scan forced for that cycle (sound: the scan restores -/// liveness, and a conservatively-scanned cycle is non-moving). This is the -/// measuring mode: alloc-point full collections are legitimate today and -/// simply pay the scan. -/// `strict` — PANIC on any undeclared precise-root cycle. This is the gate -/// mode that proves the enforcement is live. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum SafepointOnlyContract { - Off, - Heal, - Strict, -} - -pub(super) fn gc_safepoint_only_contract() -> SafepointOnlyContract { - use std::sync::OnceLock; - static CACHED: OnceLock = OnceLock::new(); - *crate::once_init::get_or_init(&CACHED, || { - safepoint_only_contract_from_value(std::env::var("PERRY_GC_SAFEPOINT_ONLY").ok().as_deref()) - }) -} - -/// Pure value→contract mapping (#7991), so both directions are testable without -/// touching the process environment. The boolean arm shares the one GC -/// boolean-ish vocabulary; `strict` is this knob's own third state. -pub(super) fn safepoint_only_contract_from_value(raw: Option<&str>) -> SafepointOnlyContract { - if matches!( - raw.map(|v| v.trim().to_ascii_lowercase()).as_deref(), - Some("strict") - ) { - return SafepointOnlyContract::Strict; - } - if super::env_flag_from_value(raw) { - return SafepointOnlyContract::Heal; - } - SafepointOnlyContract::Off -} - -/// Contract enforcement chokepoint, called once at every synchronous -/// collection entry. When an undeclared precise-root collection is about to -/// begin, heal mode returns a scan-override guard that must be held for the -/// WHOLE collection: it flips the thread-local override that every consumer -/// of `conservative_stack_scan_decision()` reads — the root-scan subphase, -/// copying-minor eligibility, and the evacuation verifier alike. A previous -/// revision healed by overriding a local variable inside the root-scan -/// subphase only; copying-minor eligibility still read the global decision, -/// concluded there were no conservative roots to pin, and forced evacuation -/// moved objects that raw native-stack words still pointed at. -pub(super) fn contract_scan_heal_guard() -> Option { - if gc_safepoint_only_contract() == SafepointOnlyContract::Off { - return None; - } - if !super::roots::native_stack_maps_active() || GC_AT_DECLARED_SAFEPOINT.with(Cell::get) { - return None; - } - if matches!( - super::roots::conservative_stack_scan_decision(), - super::roots::ConservativeStackScanDecision::Scan - ) { - return None; - } - if gc_safepoint_only_contract() == SafepointOnlyContract::Strict { - panic!( - "PERRY_GC_SAFEPOINT_ONLY: precise-root collection began outside \ - a declared safepoint" - ); - } - Some(super::roots::ManualGcScanGuard::force_full_scan( - super::ConservativeScanSite::SafepointContractHeal, - )) + /// A collection a root-lock exit owed but did not run (RFC deferred + /// collection S5: "D stops collecting"). Served by the next declared poll + /// in `gc_safepoint_moving_minor`; `CheckTrigger` never lands here — it is + /// an allocation-point evaluation and runs at the flush as before. + static GC_POLL_OWED_REQUEST: Cell = + const { Cell::new(DeferredGcRequest::None) }; } /// RAII marker for a declared-safepoint drain. Nesting-safe: restores the @@ -1618,27 +1544,58 @@ pub(super) fn flush_deferred_gc_request() { } match take_deferred_gc_request() { DeferredGcRequest::None => {} + // An allocation-point evaluation, held to D2 like any other: it may + // only arm the poll or run a conservative non-moving arm. + DeferredGcRequest::CheckTrigger => gc_check_trigger(), + // RFC deferred collection S5, "D stops collecting" (option 2 of + // #11523): a root-lock exit is an arbitrary point inside a runtime + // helper, not a declared poll, so a collection that was requested + // while the lock was held is handed to the next poll instead of + // running here. This closes the #11523 class structurally rather than + // one noncollecting guard at a time. + request => route_owed_request_to_poll(request), + } +} + +/// Park a deferred collection for the next declared poll and arm the poll. +fn route_owed_request_to_poll(request: DeferredGcRequest) { + GC_POLL_OWED_REQUEST.with(|owed| owed.set(owed.get().merge(request))); + super::alloc_point::note_owed_request_routed(); + arm_precise_safepoint(); +} + +/// Run a collection a root-lock exit owed, at a declared poll. Returns whether +/// one was owed. The unsafe-zone checks are the ones the flush used to make. +fn serve_owed_request_at_poll() -> bool { + let request = GC_POLL_OWED_REQUEST.with(|owed| owed.replace(DeferredGcRequest::None)); + match request { + DeferredGcRequest::None => return false, DeferredGcRequest::CheckTrigger => gc_check_trigger(), DeferredGcRequest::DirectMinor => { - if gc_blocked_by_unsafe_zone() { - return; + if !gc_blocked_by_unsafe_zone() { + gc_collect_minor_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct)) + .emit_after_current(); } - gc_collect_minor_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct)) - .emit_after_current(); } DeferredGcRequest::Collect(GcTriggerKind::Manual) => { - if manual_gc_blocked_by_unsafe_zone() { - return; + if !manual_gc_blocked_by_unsafe_zone() { + manual_gc_collect_now(); } - manual_gc_collect_now(); } DeferredGcRequest::Collect(kind) => { - if gc_blocked_by_unsafe_zone() { - return; + if !gc_blocked_by_unsafe_zone() { + gc_collect_inner_with_trigger(GcTriggerSnapshot::capture(kind)) + .emit_after_current(); } - gc_collect_inner_with_trigger(GcTriggerSnapshot::capture(kind)).emit_after_current(); } } + super::alloc_point::note_owed_request_served(); + true +} + +#[cfg(test)] +pub(super) fn poll_owed_request_pending() -> bool { + GC_POLL_OWED_REQUEST.with(|owed| !matches!(owed.get(), DeferredGcRequest::None)) } pub fn gc_suppress() { @@ -3151,6 +3108,13 @@ fn gc_check_trigger_evaluate() { if GC_BUDGETED_STEP_ACTIVE.with(Cell::get) { return; } + // RFC deferred collection S5 (D2): everything below runs at an ALLOCATION + // POINT. It may arm the poll, run heap-only budgeted work, or run one of + // the conservative non-moving arms (OldReclaim, the nursery valve). It + // never starts a phase that reads frame roots precisely or moves — the + // budgeted stepper parks at those phases and the synchronous chokepoints + // assert it (`gc/alloc_point.rs`). + let _alloc_point = super::alloc_point::AllocationPointGuard::enter(); // Issue #62: single TLS access covers both `in_alloc` and `suppressed`. let flags = GC_FLAGS.with(|f| f.get()); if flags & GC_FLAG_SUPPRESSED != 0 { @@ -3471,6 +3435,36 @@ fn gc_check_trigger_evaluate() { return; } + // S5 (D2): a budgeted cycle whose next work reads frame roots (`RootScan`, + // `FinalRootRemark`) does not advance from an allocation point. It is + // parked, the poll is armed, and the next declared poll serves the phase + // with a precise root set (`gc_safepoint_moving_minor`). The heap-only + // phases around it keep advancing from assists exactly as before. + // + // The park has a valve, measured like the nursery deferral's: a program + // that allocates the slack past the park point without reaching any poll + // has the phase served here instead, so a straight-line body cannot hold + // a cycle — and with it every other collection, which an active cycle + // blocks — open forever. That is counted, and gated to zero in CI; see + // `alloc_point::note_parked_valve_fired` for why it is sound until S6. + if gc_budgeted_cycle_active() { + if budgeted_cycle_next_step_reads_frame_roots() { + let arena_total = crate::arena::arena_total_bytes(); + if !super::alloc_point::park_root_phase(arena_total, arm_precise_safepoint) + && super::alloc_point::parked_valve_due( + arena_total, + gc_moving_defer_slack_dyn_bytes(), + ) + { + super::alloc_point::note_parked_valve_fired(); + super::diag_sites::trigger_decision("alloc_point_slack", "parked_root_phase"); + super::alloc_point::with_allocation_point_lifted(serve_budgeted_root_phase); + } + return; + } + super::alloc_point::clear_park(); + } + let units = gc_mutator_assist_scaled_work_units(); let probe = super::diag_sites::ChargeProbe::begin(); let _ = @@ -3797,6 +3791,24 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { let in_alloc = flags & (GC_FLAG_IN_ALLOC | GC_FLAG_SUPPRESSED) != 0; let unsafe_zone = gc_blocked_by_unsafe_zone(); let root_lock = GC_ROOT_LOCK_DEPTH.with(|depth| depth.get() != 0); + if !(in_alloc || unsafe_zone || root_lock) { + // S5: a collection a root-lock exit owed runs here, at the declared + // poll, rather than at the lock exit (`flush_deferred_gc_request`). + if serve_owed_request_at_poll() { + set_safepoint_pending(false); + return true; + } + // S5 (D2): the budgeted cycle's frame-root phases run at declared + // points only. An allocation point that reached one parked the cycle + // and armed this poll; serve the phase now, with precise roots. + if gc_budgeted_cycle_active() && budgeted_cycle_next_step_reads_frame_roots() { + let _declared = DeclaredSafepointGuard::enter(); + serve_budgeted_root_phase(); + super::alloc_point::note_root_phase_served_at_poll(); + set_safepoint_pending(false); + return true; + } + } let budgeted = gc_budgeted_cycle_active(); if in_alloc || unsafe_zone || root_lock || budgeted { // Blocked right now — leave GC_SAFEPOINT_PENDING set so the next poll @@ -4663,6 +4675,14 @@ pub(super) fn gc_idle_reclaim_step(budget_us: u64) -> GcStepReport { /// stale would make `moving_defer_within_slack` read an already-exceeded /// baseline and disable deferral for the rest of the process, the #7024 shape). fn defer_nursery_cap_to_precise_safepoint() { + arm_precise_safepoint(); +} + +/// Arm the next declared poll, recording the arena baseline the nursery valve +/// measures its slack from — exactly as the nursery deferral does. Shared by +/// every arm that hands work to a poll: the nursery cap, a parked budgeted +/// root phase, and a collection a root-lock exit owed (S5). +fn arm_precise_safepoint() { if GC_SAFEPOINT_PENDING.with(Cell::get) { return; } @@ -4813,8 +4833,59 @@ fn gc_budgeted_start_or_step( }); match outcome { - BudgetedStepOutcome::Result(result) => result, - BudgetedStepOutcome::Completed(cycle) => gc_finish_budgeted_cycle(cycle), + BudgetedStepOutcome::Result(result) => { + // S5 (D2): an assist that just walked the cycle up to a frame-root + // phase parks it there and arms the poll. The step itself refused + // to enter the phase (`GcCycleState::step`). + if super::alloc_point::at_allocation_point() { + if budgeted_cycle_next_step_reads_frame_roots() { + super::alloc_point::park_root_phase( + crate::arena::arena_total_bytes(), + arm_precise_safepoint, + ); + } + } else if !budgeted_cycle_next_step_reads_frame_roots() { + super::alloc_point::clear_park(); + } + result + } + BudgetedStepOutcome::Completed(cycle) => { + super::alloc_point::clear_park(); + gc_finish_budgeted_cycle(cycle) + } + } +} + +/// Whether the active budgeted cycle's next step reads frame roots — a phase +/// D2 forbids an allocation point to start. +fn budgeted_cycle_next_step_reads_frame_roots() -> bool { + GC_BUDGETED_CYCLE.with(|slot| { + slot.borrow() + .as_ref() + .is_some_and(|cycle| cycle.state.next_step_reads_frame_roots()) + }) +} + +/// Advance the active budgeted cycle through its frame-root phase. Called at a +/// declared poll, or by the counted parked-cycle valve. Unbounded work for the +/// phase itself: `RootScan` is bounded by the root set, and `FinalRootRemark` +/// is atomic by design (`gc-step-bounds.md`). Stops as soon as the next step +/// no longer reads frame roots, so the heap-only work that follows stays with +/// the assists and host steps. +fn serve_budgeted_root_phase() { + // A handful of steps: the build of the valid-pointer set may precede the + // root scan, and the barrier-seed drain precedes the remark. + for _ in 0..8 { + if !gc_budgeted_cycle_active() || !budgeted_cycle_next_step_reads_frame_roots() { + break; + } + let result = gc_budgeted_step_work_units_inner(usize::MAX); + if result.status == JS_GC_STEP_STATUS_SKIPPED { + break; + } + } + if !budgeted_cycle_next_step_reads_frame_roots() { + super::alloc_point::clear_park(); } } diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index 338a561d53..caebb227b4 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -12,7 +12,8 @@ pub(crate) use stack_maps::census_rows::stack_map_index_census; mod temp_roots; pub(super) use stack_maps::ensure_built as ensure_stack_maps_built; pub(super) use stack_maps::initialize as initialize_stack_maps; -pub(super) use stack_maps::native_maps_active as native_stack_maps_active; +pub(super) use stack_maps::frame_verify::active as stack_maps_frame_verify_active; +pub(super) use stack_maps::frame_verify::counters as frame_verify_counters; pub(super) use stack_maps::publish_rewrite_walk_stats as stack_maps_publish_rewrite_walk_stats; pub(super) use stack_maps::record_native_stack_walk_source; pub(super) use stack_maps::verify_native_slots_post_walk as stack_maps_native_slot_verify; diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index 33703b5bb5..a9eb6cba38 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -127,6 +127,9 @@ struct StackMapIndex { /// Sorted by address. Duplicates are kept, not deduplicated: two entries /// can share a relocated address and each brings its own records. functions: Vec, + /// Sorted start addresses of generated functions the map lists with ZERO + /// records (the unmapped-frame verifier's view; `stack_maps_frame_verify.rs`). + unrecorded_functions: Vec, eager: Option, } @@ -803,13 +806,6 @@ pub(in crate::gc) fn ensure_built() { STACK_MAPS_INITIALIZED.store(true, Ordering::Release); } -/// Whether this image carries any native stack-map records — i.e. whether -/// precise frame roots depend on mapped PCs at all. Consumed by the -/// `PERRY_GC_SAFEPOINT_ONLY` contract assert. -pub(in crate::gc) fn native_maps_active() -> bool { - !stack_maps().index.is_empty() -} - fn stack_maps() -> RwLockReadGuard<'static, PublishedStackMapIndex> { STACK_MAPS.read() } @@ -877,13 +873,14 @@ fn build_index_from_sections( // fourth gate-failure mode (the gate runs, its subject never did), so // fail loudly instead. In practice this can only mean a binary whose // compiler and runtime disagree about the map format. - let mut functions = Vec::new(); + let (mut functions, mut unrecorded_functions) = (Vec::new(), Vec::new()); for (index, section) in sections.iter().enumerate() { let section_index = u16::try_from(index).unwrap_or_else(|_| { panic!("perry: {} loaded images carry a GC map section; the index addresses them with a u16", sections.len()) }); let origin = origins[index]; - if lazy::parse_function_table(section_index, section, origin, &mut functions).is_none() { + let (out, unrecorded) = (&mut functions, &mut unrecorded_functions); + if lazy::parse_functions(section_index, section, origin, out, unrecorded).is_none() { undecodable_section(section.len()); } } @@ -892,6 +889,7 @@ fn build_index_from_sections( // symbol, or the linker can fold identical code, and each entry brings its // own records. Deduplicating would drop one set silently. crate::cold_sort::sort_by_u64_key(&mut functions, |entry| entry.address as u64); + unrecorded_functions.sort_unstable(); let eager = match mode { IndexMode::Lazy => None, @@ -901,6 +899,7 @@ fn build_index_from_sections( mode, sections, functions, + unrecorded_functions, eager, } } @@ -1302,14 +1301,9 @@ fn sve_vector_length_bytes() -> Option { pub(super) fn visit_stack_map_root_slots( visit: &mut impl FnMut(MutableRootSlot), ) -> NativeStackWalkStats { - // The invariant is not "initialize ran" but "an empty index means this - // image genuinely has no native roots". Stating it that way keeps the - // check live in EVERY configuration: perry-runtime's unit tests reach the - // scan without `js_gc_init`, and they pass because their harness carries - // no gc-map section — the right reason — rather than by being exempted - // from the check. Exempting them by build config would leave no check in - // precisely the configuration where the index is legitimately unbuilt, - // which is a hole the moment a test binary does carry statepoint frames. + // The invariant is "an empty index means this image genuinely has no + // native roots", not "initialize ran": that keeps the check live in every + // configuration, unit tests (which carry no gc-map section) included. assert!( stack_maps_initialized() || !image_has_stack_map_sections(), "perry: the native root scan ran before the stack-map index was built. \ @@ -1323,21 +1317,17 @@ pub(super) fn visit_stack_map_root_slots( if index.is_empty() { return NativeStackWalkStats::default(); } - match walker_mode() { + // The unmapped-frame verifier needs each frame's function start, which the + // unwinder reports and the x29-chain walk does not. + let mode = if frame_verify::active() { WalkerMode::Unwind } else { walker_mode() }; + match mode { WalkerMode::Unwind => unwind::visit(index, &mut |root: ResolvedRoot| { root.visit_with_context(visit) }), WalkerMode::Fast => { - // No whole-image `chain_walkable` precondition any more. v4 decided - // it once by scanning every root slot in the section — 4.9M of them - // for claude-code — which a lazy index cannot do and should not: - // the fast walk now checks the frame it is about to resolve and - // fails closed to the unwinder if THAT record uses a base it cannot - // reconstruct. That is strictly narrower than disabling the fast - // path for the whole image because one function somewhere uses an - // exotic register, and it reuses the mid-walk bail this walker - // already performs for `x19_is_body_sp` and an unvalidated - // `caller_fp`. + // No whole-image `chain_walkable` precondition (v4 scanned 4.9M + // slots for it): the fast walk checks each frame it resolves and + // fails closed to the unwinder for a base it cannot reconstruct. if let Some(stats) = fp_chain::visit(index, &mut |root: ResolvedRoot| { root.visit_with_context(visit) }) { @@ -1368,6 +1358,7 @@ mod unwind { argument: *mut c_void, ) -> i32; fn _Unwind_GetIP(context: *mut UnwindContext) -> usize; + fn _Unwind_GetRegionStart(context: *mut UnwindContext) -> usize; fn _Unwind_GetGR(context: *mut UnwindContext, register: i32) -> usize; /// The frame's canonical frame address — the supported way to reach a /// frame's stack pointer. `_Unwind_GetGR` on the SP column is not a @@ -1437,6 +1428,10 @@ mod unwind { } let index = state.index; let Some(matched) = index.match_records(ip) else { + if frame_verify::active() { + let start = _Unwind_GetRegionStart(context); + frame_verify::unmatched_frame(index, ip, start); + } return 0; }; let mut records = index.matched(&matched); @@ -1459,13 +1454,7 @@ mod unwind { // alike — so there is no return-address adjustment to make and // no per-architecture constant left to get wrong. // - // It stayed invisible because this is the FALLBACK path: on - // aarch64 the x29 chain walk normally answers, and wherever it - // bailed this read unrelated words instead of the roots, which - // nothing downstream can notice — no code knows what a root slot - // is supposed to contain. Cross-checked directly on - // `02_survivor_promotion`: at the CFA the slot holds a NaN-boxed - // pointer (`0x7ffd…`); one frame lower it holds a stack address. + // (#7392 has why it stayed invisible: this is the fallback path.) let base = if location.dwarf_reg == ARCH_DWARF_SP { _Unwind_GetCFA(context) } else { @@ -1700,6 +1689,12 @@ mod unwind { let entry = unsafe { RtlLookupFunctionEntry(context.rip, &mut image_base, std::ptr::null_mut()) }; + let rip = context.rip as usize; + if frame_verify::active() && !entry.is_null() && index.match_records(rip).is_none() { + // RUNTIME_FUNCTION.BeginAddress is its first u32, image-relative. + let begin = unsafe { *(entry as *const u32) } as usize + image_base as usize; + frame_verify::unmatched_frame(index, rip, begin); + } if entry.is_null() { // No unwind info. On Win64 only the innermost frame can be a // leaf (a function that has performed a call must carry @@ -1947,6 +1942,9 @@ mod lazy; #[path = "stack_maps_index.rs"] mod index; +#[path = "stack_maps_frame_verify.rs"] +pub(in crate::gc) mod frame_verify; + #[path = "stack_maps_decode.rs"] mod decode; use decode::parse_gc_map; diff --git a/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs b/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs new file mode 100644 index 0000000000..158fdd0971 --- /dev/null +++ b/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs @@ -0,0 +1,144 @@ +//! The unmapped-frame verifier — the runtime safety net of RFC deferred +//! collection (§2, "Soundness against the #11522 class", layer 3). +//! +//! The native-root walkers look each frame's return address up in the GC map +//! and, when nothing matches, move on: a Rust runtime frame legitimately has no +//! map. A GENERATED frame with no map at its return address is something else: +//! the frame is suspended at a call codegen marked `gc-leaf-function` (so RS4GC +//! recorded nothing there), and a collection is running anyway. That call's +//! callee did collect. Its caller's roots were not visited, so a moving +//! collection leaves them stale and a non-moving one may free what they name — +//! the #11522 / #11523 failure, reported by nobody. +//! +//! This module makes that frame fail loudly. The question "is this a generated +//! function?" is answered from the function's start address (the unwinder's +//! region start), looked up in the set of statepoint-strategy functions the GC +//! map lists. Functions with records are always listed; functions whose every +//! call is a leaf have no records, and codegen lists them as zero-record +//! entries when the program was compiled with the GC instruments +//! (`PERRY_GC_INSTRUMENTS=1`, `PERRY_GC_VERIFY_FRAMES` or a schedule seed at +//! compile time). A v6 runtime that predates this skips such entries, so the +//! format did not change. +//! +//! Only a PRECISE collection is checked: a conservative one scans the whole +//! native stack, unmapped frames included, and does not move. +//! +//! Armed by `PERRY_GC_VERIFY_FRAMES=1`, by a resolved `PERRY_GC_SCHEDULE_SEED` +//! (the pairing the RFC names: at `RATE=1` every legal collection point is +//! checked), and in `debug_assertions` builds (`gcaudit`). Off in release: it +//! needs the unwinder rather than the x29-chain walk, and the zero-record +//! entries cost map bytes; see the S5 changelog for the numbers. + +use super::StackMapIndex; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::OnceLock; + +static FRAMES_CHECKED: AtomicU64 = AtomicU64::new(0); +static GENERATED_UNMAPPED: AtomicU64 = AtomicU64::new(0); + +/// Whether the verifier is armed for this process. +pub(in crate::gc) fn active() -> bool { + static ACTIVE: OnceLock = OnceLock::new(); + *crate::once_init::get_or_init(&ACTIVE, || { + cfg!(debug_assertions) || knob_enabled() || crate::gc::schedule::gc_schedule_enabled() + }) +} + +fn knob_enabled() -> bool { + #[cfg(not(feature = "gc-instruments"))] + return false; + #[cfg(feature = "gc-instruments")] + crate::gc::env_flag_enabled("PERRY_GC_VERIFY_FRAMES") +} + +/// A walker found no record for the frame whose return address is `ip` and +/// whose function starts at `function_start`. Panics if that function is a +/// generated statepoint-strategy function and this collection is precise. +pub(super) fn unmatched_frame(index: &StackMapIndex, ip: usize, function_start: usize) { + FRAMES_CHECKED.fetch_add(1, Ordering::Relaxed); + if function_start == 0 || !index.is_generated_function(function_start) { + return; + } + if matches!( + crate::gc::conservative_stack_scan_decision(), + crate::gc::ConservativeStackScanDecision::Scan + ) { + return; + } + GENERATED_UNMAPPED.fetch_add(1, Ordering::Relaxed); + unmapped_generated_frame(ip, function_start); +} + +#[cold] +#[inline(never)] +fn unmapped_generated_frame(ip: usize, function_start: usize) -> ! { + panic!( + "perry GC safety net: a precise collection walked a GENERATED frame with no \ + stack map at its call site (function {function_start:#x}, return address \ + {ip:#x}, offset {:#x}). The call was compiled as one that cannot collect \ + (`gc-leaf-function`), and a collection began inside it, so this frame's \ + roots were not visited. See RFC deferred collection §2 (the #11522 class).", + ip.wrapping_sub(function_start) + ) +} + +/// `(frames_checked, generated_unmapped)` — the verifier's liveness counters. +pub(in crate::gc) fn counters() -> (u64, u64) { + ( + FRAMES_CHECKED.load(Ordering::Relaxed), + GENERATED_UNMAPPED.load(Ordering::Relaxed), + ) +} + +impl StackMapIndex { + /// Whether `function_start` is a generated statepoint-strategy function: + /// one with records, or a zero-record entry codegen listed. + pub(super) fn is_generated_function(&self, function_start: usize) -> bool { + self.functions + .binary_search_by_key(&function_start, |entry| entry.address) + .is_ok() + || self.unrecorded_functions.binary_search(&function_start).is_ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Sabotage for the classification the verifier rests on: a zero-record + /// entry must make its function "generated", a Rust frame must not be, and + /// an unmatched frame in a generated function under a precise collection + /// must panic. + #[test] + fn unmatched_generated_frame_fails_loudly_and_runtime_frames_do_not() { + let mut index = StackMapIndex::default(); + index.unrecorded_functions = vec![0x1000, 0x2000]; + assert!(index.is_generated_function(0x2000)); + assert!(!index.is_generated_function(0x3000)); + + // A runtime frame: ignored. + unmatched_frame(&index, 0x3010, 0x3000); + let prev = + crate::gc::set_conservative_stack_scan_override(Some(crate::gc::ConservativeStackScanMode::Disabled)); + let result = std::panic::catch_unwind(|| unmatched_frame(&index, 0x2010, 0x2000)); + crate::gc::set_conservative_stack_scan_override(prev); + let message = result.expect_err("an unmapped generated frame must panic"); + let text = message + .downcast_ref::() + .map(String::as_str) + .unwrap_or_default(); + assert!(text.contains("safety net"), "{text}"); + assert!(counters().1 >= 1); + } + + /// A conservative collection is not checked: it scanned the frame. + #[test] + fn a_conservative_collection_tolerates_unmapped_generated_frames() { + let mut index = StackMapIndex::default(); + index.unrecorded_functions = vec![0x4000]; + let prev = + crate::gc::set_conservative_stack_scan_override(Some(crate::gc::ConservativeStackScanMode::Full)); + unmatched_frame(&index, 0x4010, 0x4000); + crate::gc::set_conservative_stack_scan_override(prev); + } +} diff --git a/crates/perry-runtime/src/gc/roots/stack_maps_lazy.rs b/crates/perry-runtime/src/gc/roots/stack_maps_lazy.rs index 422fec7e52..0e7e22d7a2 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps_lazy.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps_lazy.rs @@ -354,6 +354,20 @@ pub(super) fn parse_function_table( bytes: &[u8], origin: usize, out: &mut Vec, +) -> Option<()> { + parse_functions(section, bytes, origin, out, &mut Vec::new()) +} + +/// [`parse_function_table`], also collecting the address of every ZERO-record +/// function entry into `unrecorded`. Codegen lists those only in an +/// instrumented build, for the unmapped-frame verifier; they never enter the +/// record index (see the note at the `record_count == 0` check). +pub(super) fn parse_functions( + section: u16, + bytes: &[u8], + origin: usize, + out: &mut Vec, + unrecorded: &mut Vec, ) -> Option<()> { let mut base = 0usize; while base + 16 <= bytes.len() { @@ -441,6 +455,7 @@ pub(super) fn parse_function_table( // derived its function list from records and so excluded these by // construction; excluding them here keeps that property. if record_count == 0 { + unrecorded.push(address); continue; } out.push(FunctionEntry { diff --git a/crates/perry-runtime/src/gc/scan_fallback.rs b/crates/perry-runtime/src/gc/scan_fallback.rs index c3915ca637..eaeb41e71a 100644 --- a/crates/perry-runtime/src/gc/scan_fallback.rs +++ b/crates/perry-runtime/src/gc/scan_fallback.rs @@ -51,6 +51,7 @@ //! rather than observed. The two compose — census first, enforcement second. use std::cell::Cell; +use std::sync::atomic::{AtomicU64, Ordering}; /// A `force_full_scan()` callsite. Ordering matters only for the counter array. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -79,12 +80,12 @@ pub(crate) enum ConservativeScanSite { /// less. That is a different risk with a different proof obligation, and /// bundling it would have made one A/B answer two questions. ManualMinor, - /// `PERRY_GC_SAFEPOINT_ONLY` heal (#7174 research): a precise-root - /// collection began outside a declared safepoint, so the contract forces - /// the scan for that cycle rather than consuming roots that native - /// stack maps only describe at mapped PCs. Automatic, and research-mode - /// only — it cannot fire unless the contract env is set. - SafepointContractHeal, + // ★ There is deliberately no `SafepointContractHeal` variant any more. + // `PERRY_GC_SAFEPOINT_ONLY`'s heal arm forced the scan when a precise-root + // collection began outside a declared safepoint. RFC deferred collection + // step S5 made that property the default invariant (`gc/alloc_point.rs`): + // an allocation point never starts a precise collection, so nothing can + // produce the heal, and the arm was deleted under the knob kill-policy. // ★ There is deliberately no `ManualCollect` variant either. `gc()` used to // force the scan (#4977) and be counted here; #7558 established that the // precise root set covers its callsite and removed the force. The variant @@ -104,7 +105,7 @@ pub(crate) enum ConservativeScanSite { } impl ConservativeScanSite { - pub(crate) const COUNT: usize = 5; + pub(crate) const COUNT: usize = 4; const fn index(self) -> usize { match self { @@ -112,7 +113,6 @@ impl ConservativeScanSite { Self::NurseryChurnSlackValve => 1, Self::EmergencyReclaim => 2, Self::ManualMinor => 3, - Self::SafepointContractHeal => 4, } } @@ -122,7 +122,6 @@ impl ConservativeScanSite { Self::NurseryChurnSlackValve => "nursery_churn_slack_valve", Self::EmergencyReclaim => "emergency_reclaim", Self::ManualMinor => "manual_minor", - Self::SafepointContractHeal => "safepoint_contract_heal", } } @@ -133,8 +132,7 @@ impl ConservativeScanSite { match self { Self::OldReclaimAllocPoint | Self::NurseryChurnSlackValve - | Self::EmergencyReclaim - | Self::SafepointContractHeal => true, + | Self::EmergencyReclaim => true, Self::ManualMinor => false, } } @@ -145,7 +143,6 @@ impl ConservativeScanSite { Self::NurseryChurnSlackValve, Self::EmergencyReclaim, Self::ManualMinor, - Self::SafepointContractHeal, ]; } @@ -191,6 +188,23 @@ impl SafepointDrainKind { } } +/// Process-global mirrors of the per-thread counters below, for the exit +/// summary and the valve ledger (`gc/alloc_point.rs`): a valve that fires on a +/// worker thread must still show up in the process's one ledger line. +static SCAN_FALLBACKS_ALL_THREADS: [AtomicU64; ConservativeScanSite::COUNT] = + [const { AtomicU64::new(0) }; ConservativeScanSite::COUNT]; +static SAFEPOINT_DRAINS_ALL_THREADS: AtomicU64 = AtomicU64::new(0); + +/// `site`'s count summed over every thread of this process. +pub(crate) fn scan_fallback_count_any_thread(site: ConservativeScanSite) -> u64 { + SCAN_FALLBACKS_ALL_THREADS[site.index()].load(Ordering::Relaxed) +} + +/// Precise safepoint drains (every kind) summed over every thread. +pub(crate) fn safepoint_drain_total_any_thread() -> u64 { + SAFEPOINT_DRAINS_ALL_THREADS.load(Ordering::Relaxed) +} + thread_local! { static SCAN_FALLBACKS: Cell<[u64; ConservativeScanSite::COUNT]> = const { Cell::new([0; ConservativeScanSite::COUNT]) }; @@ -207,6 +221,7 @@ thread_local! { /// prints a line so an ops/benchmark run shows which sites a program reaches /// and how often. pub(crate) fn record_scan_fallback(site: ConservativeScanSite) { + SCAN_FALLBACKS_ALL_THREADS[site.index()].fetch_add(1, Ordering::Relaxed); let count = SCAN_FALLBACKS.with(|c| { let mut counts = c.get(); counts[site.index()] = counts[site.index()].saturating_add(1); @@ -227,6 +242,7 @@ pub(crate) fn record_scan_fallback(site: ConservativeScanSite) { /// the collection that a `force_full_scan()` site would otherwise have run /// conservatively at an allocation point. pub(crate) fn record_safepoint_drain(kind: SafepointDrainKind) { + SAFEPOINT_DRAINS_ALL_THREADS.fetch_add(1, Ordering::Relaxed); let count = SAFEPOINT_DRAINS.with(|c| { let mut counts = c.get(); counts[kind.index()] = counts[kind.index()].saturating_add(1); diff --git a/crates/perry-runtime/src/gc/schedule.rs b/crates/perry-runtime/src/gc/schedule.rs index 1960135fdb..4bbf5de6ad 100644 --- a/crates/perry-runtime/src/gc/schedule.rs +++ b/crates/perry-runtime/src/gc/schedule.rs @@ -676,9 +676,14 @@ pub(crate) fn report_exit_summary() { if SUMMARY_EMITTED.swap(true, Ordering::SeqCst) { return; } + // `frames_verified` is the unmapped-frame verifier's liveness counter + // (RFC deferred collection S5): a seeded run arms it, and a run that walked + // no unmatched frame at all has not exercised it. + let (frames_verified, _) = super::roots::frame_verify_counters(); eprintln!( "[gc-schedule] done: seed={seed} safepoints={} scheduled_collections={} \ - polls_paced={} copying_minors={} moved_objects={} loop_polls={}", + polls_paced={} copying_minors={} moved_objects={} loop_polls={} \ + frames_verified={frames_verified}", gc_schedule_safepoints(), gc_schedule_forced_collections(), schedule_polls_paced(), diff --git a/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs b/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs new file mode 100644 index 0000000000..cc879b6662 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs @@ -0,0 +1,240 @@ +//! RFC deferred collection S5 — the allocation-point invariant (D2). +//! +//! These tests hold the runtime to "an allocation never starts a phase that +//! reads frame roots precisely, and never starts a moving phase", and show the +//! declared poll picks up what the allocation point declined. Each asserts its +//! subject was live (the phase really was reached, the poll really served it) +//! rather than only that nothing broke. + +use super::super::alloc_point; +use super::super::*; +use super::support::*; + +fn reset_old_reclaim_pressure() { + let old_in_use = crate::arena::old_gen_in_use_bytes(); + GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|bytes| bytes.set(old_in_use)); + GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(false)); +} + +fn live_test_string(bytes: &'static [u8]) -> usize { + crate::string::js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32) as usize +} + +fn cycle_phase() -> Option { + let mut status = JsGcStepResult::default(); + (js_gc_step_status(&mut status) == JS_GC_STEP_STATUS_ACTIVE).then_some(status.phase) +} + +/// Start a budgeted (legacy-pacing) cycle from an allocation point and walk it +/// with allocation-point assists alone until it stops advancing. Returns the +/// phase it parked at. +fn assist_until_parked(max_assists: usize) -> u32 { + let mut last = None; + let mut stalled = 0; + for _ in 0..max_assists { + gc_check_trigger(); + let phase = cycle_phase().expect("the budgeted cycle must still be active"); + if Some(phase) == last { + stalled += 1; + if stalled >= 3 && alloc_point::root_phase_parked() { + return phase; + } + } else { + stalled = 0; + } + last = Some(phase); + } + panic!("allocation-point assists never parked the cycle (last phase {last:?})"); +} + +fn start_assist_cycle(label: &'static [u8]) -> GcTriggerThresholdTestGuard { + let trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + reset_old_reclaim_pressure(); + let live = live_test_string(label); + js_shadow_slot_set(0, string_bits(live)); + for _ in 0..(GC_MUTATOR_ASSIST_WORK_UNITS * 4) { + let _ = young_leaf(); + } + trigger_guard.make_arena_trigger_due(); + gc_check_trigger(); + assert!( + gc_budgeted_cycle_active(), + "the allocation point must have started a budgeted cycle (the subject of every test here)" + ); + trigger_guard +} + +/// The whole contract, end to end: allocation-point assists advance the +/// heap-only phases, park at BOTH frame-root phases (`RootScan` and the final +/// remark) with the poll armed, and a declared poll serves each one; the cycle +/// then completes and the rooted value survives. +#[test] +fn assists_park_at_both_root_phases_and_the_poll_serves_them() { + let _legacy_pacing = crate::gc::policy::force_legacy_gc_pacing(); + let _guard = CopyingNurseryTestGuard::new(1); + alloc_point::reset_alloc_point_counters(); + set_safepoint_pending(false); + let _trigger = start_assist_cycle(b"d2_both_root_phases_live"); + + // Phase 1: the root scan. + let parked = assist_until_parked(10_000); + assert_eq!( + parked, + GcCyclePhase::RootScan.ffi_code(), + "the first frame-root phase an assist reaches is the root scan" + ); + assert!( + GC_SAFEPOINT_PENDING.with(std::cell::Cell::get), + "parking must arm the poll, or nothing ever serves the phase" + ); + let parked_count = alloc_point::alloc_point_counters().root_phases_parked; + assert!(parked_count >= 1); + // More allocation does not move it. + for _ in 0..32 { + gc_check_trigger(); + } + assert_eq!(cycle_phase(), Some(GcCyclePhase::RootScan.ffi_code())); + + assert!(gc_safepoint_moving_minor(), "the poll must handle the parked phase"); + let served = alloc_point::alloc_point_counters().root_phases_served_at_poll; + assert_eq!(served, 1, "the poll served exactly the parked root scan"); + assert_ne!(cycle_phase(), Some(GcCyclePhase::RootScan.ffi_code())); + assert!(!alloc_point::root_phase_parked()); + + // Phase 2: the remark. Assists carry the cycle through marking and park + // again before the remark. + let parked = assist_until_parked(500_000); + assert_eq!( + parked, + GcCyclePhase::AtomicFinalize.ffi_code(), + "the second frame-root phase is the final remark" + ); + assert!(gc_safepoint_moving_minor()); + assert_eq!( + alloc_point::alloc_point_counters().root_phases_served_at_poll, + 2 + ); + + // The rest is heap-only and completes from assists alone. + let before = gc_collection_count(); + for _ in 0..500_000 { + gc_check_trigger(); + if !gc_budgeted_cycle_active() { + break; + } + } + assert!(!gc_budgeted_cycle_active(), "the heap-only tail completes from assists"); + assert!(gc_collection_count() > before); + assert_eq!(alloc_point::alloc_point_counters().parked_valve_fires, 0); + let live_after = (js_shadow_slot_get(0) & POINTER_MASK) as *const crate::StringHeader; + unsafe { + assert_string_bytes(live_after, b"d2_both_root_phases_live"); + } +} + +/// Sabotage for the synchronous chokepoint: a precise collection started from +/// inside an allocation-point evaluation must fail loudly, not run. (Every +/// real allocation-point arm forces the conservative scan first, so the only +/// way to reach this is to plant the violation.) +#[test] +#[should_panic(expected = "invariant D2 violated")] +fn a_precise_collection_begun_at_an_allocation_point_panics() { + let _guard = CopyingNurseryTestGuard::new(1); + let _disabled = ConservativeScanDisabledGuard::new(); + let _alloc_point = alloc_point::AllocationPointGuard::enter(); + let _ = gc_collect_minor_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct)); +} + +/// The same chokepoint does NOT fire for the arms D2 allows: a conservative +/// collection at an allocation point runs normally. +#[test] +fn a_conservative_collection_at_an_allocation_point_is_allowed() { + let _guard = CopyingNurseryTestGuard::new(1); + let before = gc_collection_count(); + { + let _alloc_point = alloc_point::AllocationPointGuard::enter(); + let _scan = + ManualGcScanGuard::force_full_scan(ConservativeScanSite::NurseryChurnSlackValve); + let _ = gc_collect_minor_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct)); + } + assert!(gc_collection_count() > before); + assert_eq!(alloc_point::alloc_point_counters().d2_violations, 0); +} + +/// "D stops collecting": a collection requested while a root lock is held is +/// not run at the lock exit (an arbitrary point inside a runtime helper) but +/// handed to the next declared poll. +#[test] +fn root_lock_exit_hands_an_owed_collection_to_the_poll() { + let _guard = CopyingNurseryTestGuard::new(1); + let _trigger = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + alloc_point::reset_alloc_point_counters(); + set_safepoint_pending(false); + let before = gc_collection_count(); + super::super::roots::enter_gc_root_lock(); + assert_eq!(gc_collect_minor(), 0, "a locked collection defers"); + super::super::roots::exit_gc_root_lock(); + assert_eq!( + gc_collection_count(), + before, + "the lock exit must not collect (it is not a declared poll)" + ); + assert!(super::super::policy::poll_owed_request_pending()); + assert!(GC_SAFEPOINT_PENDING.with(std::cell::Cell::get)); + assert_eq!(alloc_point::alloc_point_counters().owed_requests_routed, 1); + + assert!(gc_safepoint_moving_minor()); + assert!( + gc_collection_count() > before, + "the poll ran the owed collection" + ); + assert!(!super::super::policy::poll_owed_request_pending()); + assert_eq!(alloc_point::alloc_point_counters().owed_requests_served, 1); +} + +/// The parked-cycle valve: an allocation point that has grown the slack past +/// the park point with no poll serves the phase itself, and says so. +#[test] +fn parked_cycle_valve_fires_after_the_slack_and_is_counted() { + let _legacy_pacing = crate::gc::policy::force_legacy_gc_pacing(); + let _guard = CopyingNurseryTestGuard::new(1); + alloc_point::reset_alloc_point_counters(); + set_safepoint_pending(false); + let _trigger = start_assist_cycle(b"d2_parked_valve_live"); + assert_eq!(assist_until_parked(10_000), GcCyclePhase::RootScan.ffi_code()); + + // Pretend the program allocated the whole slack since parking. + alloc_point::test_set_park_base(0); + gc_check_trigger(); + + let counters = alloc_point::alloc_point_counters(); + assert_eq!(counters.parked_valve_fires, 1, "the valve fired once"); + assert_ne!( + cycle_phase(), + Some(GcCyclePhase::RootScan.ffi_code()), + "the valve served the phase" + ); + assert!(counters.d2_violations == 0); + assert!(alloc_point::alloc_point_exit_line().contains("parked_valve_fires=1")); + let completed = complete_budgeted_gc_cycle(); + assert_eq!(completed.status, JS_GC_STEP_STATUS_COMPLETED); +} + +/// Decision 10: growth inside an unsafe zone is counted (diagnostic only). +#[test] +fn block_growth_inside_an_unsafe_zone_is_counted() { + let _guard = CopyingNurseryTestGuard::new(1); + let _trigger = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + alloc_point::reset_alloc_point_counters(); + let before = alloc_point::alloc_point_counters(); + assert_eq!(before.unsafe_zone_growth_events, 0); + let previous = + super::super::policy::unsafe_zone_test_override::set_unsafe_zone_blocked_for_test(Some(true)); + crate::gc::note_block_if_unsafe_zone(1 << 20); + super::super::policy::unsafe_zone_test_override::set_unsafe_zone_blocked_for_test(Some(false)); + crate::gc::note_block_if_unsafe_zone(1 << 20); + super::super::policy::unsafe_zone_test_override::set_unsafe_zone_blocked_for_test(previous); + let after = alloc_point::alloc_point_counters(); + assert_eq!(after.unsafe_zone_growth_events, 1, "only the in-zone block counts"); + assert_eq!(after.unsafe_zone_growth_bytes, 1 << 20); +} diff --git a/crates/perry-runtime/src/gc/tests/debt_pacer.rs b/crates/perry-runtime/src/gc/tests/debt_pacer.rs index fbc1e03cb1..735a12ac61 100644 --- a/crates/perry-runtime/src/gc/tests/debt_pacer.rs +++ b/crates/perry-runtime/src/gc/tests/debt_pacer.rs @@ -171,7 +171,9 @@ fn active_cycle_gc_check_trigger_calls_pay_bounded_assist_work() { /// #6180: allocation-side mutator assists must drive the *entire* budgeted /// cycle to completion — through `AtomicFinalize`, `Sweep`, and `Reclaim` — /// using only the slice of work performed from `gc_check_trigger` (the -/// allocator), never a host safepoint (`js_gc_step_work_units`). +/// allocator) plus the loop's own back-edge poll, never a host safepoint +/// (`js_gc_step_work_units`). Since RFC deferred collection S5 the two +/// frame-root phases run at that poll rather than in an assist (D2). /// /// Before #6180 the assist path bailed at the first non-mark phase, so a pure /// compute loop that never reached the event pump would start a cycle, advance @@ -221,6 +223,8 @@ fn allocation_assists_complete_finalize_sweep_and_reclaim() { let mut completed = false; for _ in 0..500_000 { gc_check_trigger(); + // S5: the frame-root phases wait for the loop's back-edge poll. + back_edge_poll(); js_gc_step_status(&mut status); if status.phase == GcCyclePhase::AtomicFinalize.ffi_code() { reached_finalize = true; @@ -673,6 +677,8 @@ fn debt_scaled_assists_cannot_be_outrun_by_allocation() { let _ = young_leaf(); } gc_check_trigger(); + // S5: the frame-root phases wait for the loop's back-edge poll. + back_edge_poll(); calls += 1; assert!( calls <= 300, diff --git a/crates/perry-runtime/src/gc/tests/env_knob_parse.rs b/crates/perry-runtime/src/gc/tests/env_knob_parse.rs index 8a0bf964f4..0d3bb02c19 100644 --- a/crates/perry-runtime/src/gc/tests/env_knob_parse.rs +++ b/crates/perry-runtime/src/gc/tests/env_knob_parse.rs @@ -25,7 +25,6 @@ //! which libtest thread ran first (`knob_overrides` in `gc/mod.rs` records //! what that cost us — 5 failures in 100 runs across three unrelated cases). -use super::super::policy::{safepoint_only_contract_from_value, SafepointOnlyContract}; use super::super::{env_default_on_from_value, env_flag_from_value}; /// Every spelling a human might reasonably use to mean "off", plus the two @@ -156,39 +155,11 @@ fn the_two_vocabularies_disagree_only_on_the_unrecognised_case() { } } -/// `PERRY_GC_SAFEPOINT_ONLY` is three-state. Its boolean arm must share the one -/// vocabulary; only `strict` is its own. -#[test] -fn safepoint_only_is_three_state_over_the_shared_vocabulary() { - for raw in OFF_SPELLINGS { - assert_eq!( - safepoint_only_contract_from_value(*raw), - SafepointOnlyContract::Off, - "{raw:?} must leave the safepoint-only contract Off" - ); - } - for raw in ON_SPELLINGS { - assert_eq!( - safepoint_only_contract_from_value(Some(raw)), - SafepointOnlyContract::Heal, - "{raw:?} must select Heal" - ); - } - for raw in ["strict", "STRICT", " strict "] { - assert_eq!( - safepoint_only_contract_from_value(Some(raw)), - SafepointOnlyContract::Strict, - "{raw:?} must select Strict" - ); - } - for raw in UNRECOGNISED { - assert_eq!( - safepoint_only_contract_from_value(Some(raw)), - SafepointOnlyContract::Off, - "{raw:?} is unrecognised and must not arm a contract enforcer" - ); - } -} +// `PERRY_GC_SAFEPOINT_ONLY`'s runtime contract (Off/Heal/Strict) is gone: RFC +// deferred collection S5 made "a precise collection begins only at a declared +// point" the default invariant (`gc/alloc_point.rs`, tested in +// `alloc_point_invariant.rs`). The knob survives only as codegen's research +// switch for `AllocNoReentry` leaves, which it parses itself. /// The decisive arm: the **live cached reader**, initialised in a child /// process under a real `PERRY_GC_DIAG=0`. diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index d4de64fe1b..ed5431d92b 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -1,5 +1,6 @@ mod adopt_census; mod alloc; +mod alloc_point_invariant; mod arena_right_size; mod arguments_objects; mod array_named_props; diff --git a/crates/perry-runtime/src/gc/tests/support.rs b/crates/perry-runtime/src/gc/tests/support.rs index 8cfd348382..0d7608d965 100644 --- a/crates/perry-runtime/src/gc/tests/support.rs +++ b/crates/perry-runtime/src/gc/tests/support.rs @@ -161,6 +161,20 @@ pub(super) fn gc_collection_count() -> u64 { GC_STATS.with(|s| s.borrow().collection_count) } +/// What a compiled allocating loop does at its back-edge: reach the declared +/// poll. Since RFC deferred collection S5 an allocation point parks a budgeted +/// cycle at its frame-root phases (`RootScan`, the final remark) and only a +/// declared poll serves them, so a test that drives a cycle "from allocation +/// alone" models the compiled loop — allocation plus its back-edge poll — by +/// calling this after each assist. With a budgeted cycle active the poll +/// either serves the parked phase or is blocked; it never starts a collection +/// of its own, so it does not change what else the test observes. +pub(super) fn back_edge_poll() { + if gc_budgeted_cycle_active() { + let _ = gc_safepoint_moving_minor(); + } +} + pub(super) fn complete_budgeted_gc_cycle() -> JsGcStepResult { let mut result = JsGcStepResult::default(); for _ in 0..500_000 { diff --git a/crates/perry/src/commands/compile/optimized_libs/freshness.rs b/crates/perry/src/commands/compile/optimized_libs/freshness.rs index 1b9ea5f13b..502fda373c 100644 --- a/crates/perry/src/commands/compile/optimized_libs/freshness.rs +++ b/crates/perry/src/commands/compile/optimized_libs/freshness.rs @@ -126,6 +126,7 @@ pub(crate) const GC_INSTRUMENT_KNOBS: &[&str] = &[ "PERRY_ALLOC_SITE_SAMPLE", "PERRY_GC_VERIFY_MARK", "PERRY_GC_VERIFY_CLASSIFIER", + "PERRY_GC_VERIFY_FRAMES", "PERRY_STACK_SYMBOLS", ]; From fcaaed38c1a55b192904c7a04c96ae51f6e3f957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 10:20:05 +0200 Subject: [PATCH 02/15] wip: S5 safety net codegen + valve gate --- .github/workflows/test.yml | 16 +++ crates/perry-codegen/src/function.rs | 15 +- crates/perry-codegen/src/gc_map.rs | 95 +++++++++++- crates/perry-codegen/src/module.rs | 2 + crates/perry-codegen/src/native_emit.rs | 1 + .../perry-runtime/src/gc/roots/stack_maps.rs | 12 +- .../src/gc/roots/stack_maps_frame_verify.rs | 43 +++++- .../perry/src/commands/compile/build_cache.rs | 6 + .../src/commands/compile/object_cache.rs | 9 ++ scripts/gc_valve_ledger_check.py | 135 ++++++++++++++++++ scripts/gc_valve_ratchet_probes.sh | 39 +++++ 11 files changed, 359 insertions(+), 14 deletions(-) create mode 100755 scripts/gc_valve_ledger_check.py create mode 100755 scripts/gc_valve_ratchet_probes.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2851339fab..e8b7251d08 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2289,6 +2289,11 @@ jobs: - name: GC rooting-bug instruments (inert-when-off, live-when-on) run: ./scripts/gc_instrument_smoke.sh target/release/perry + # GATING (RFC deferred collection, decision 5): no allocation-point valve + # fires on the GC ratchet probes, and every probe reports. + - name: GC valve ledger over the ratchet probes + run: ./scripts/gc_valve_ratchet_probes.sh target/release/perry + - name: Run GC write-barrier stress tests # Informational: these are ~200s nondeterministic corruption-window # hunts (#5029). Kept out of the gate so a flake never blocks a PR. @@ -3109,11 +3114,22 @@ jobs: export PERRY_BIN="$PWD/target/release/perry" export PERRY_RUNTIME_DIR="$PWD/target/release" fi + # RFC deferred collection, decision 5: an allocation-point GC valve + # firing on the gap suite is a hard failure. Every Perry binary the + # harness runs appends one line to this ledger at exit; the check + # below requires a line per passing test (the proof it ran) and + # zero valve firings on every line. + export PERRY_GC_VALVE_LEDGER="$RUNNER_TEMP/gc-valve-ledger.txt" + rm -f "$PERRY_GC_VALVE_LEDGER" + python3 scripts/gc_valve_ledger_check.py --self-test if [ "$GAP_TOTAL" = "1" ]; then ./scripts/run_gap_tests.sh else ./scripts/run_gap_tests.sh --shard "$GAP_SHARD/$GAP_TOTAL" fi + python3 scripts/gc_valve_ledger_check.py \ + --ledger "$PERRY_GC_VALVE_LEDGER" \ + --expect-min-from-report test-parity/reports/latest.json # Only produced by a `workflow_dispatch` with update_gap_snapshot=true # (one shard, whole suite). Download it and commit test-parity/ diff --git a/crates/perry-codegen/src/function.rs b/crates/perry-codegen/src/function.rs index 4a3b7335ef..c78ee7d3a7 100644 --- a/crates/perry-codegen/src/function.rs +++ b/crates/perry-codegen/src/function.rs @@ -895,6 +895,16 @@ impl LlFunction { self.stack_map_requested } + /// Whether this function is rendered with `gc "statepoint-example"`, i.e. + /// whether its frames are described by the native GC map. The one + /// predicate the renderer and the GC map's zero-record listing share + /// (`gc_map::note_statepoint_functions`). + pub(crate) fn uses_statepoint_strategy(&self) -> bool { + self.stack_map_requested + && !self.force_shadow_frame + && crate::codegen::helpers::native_stack_roots_enabled() + } + /// Label of the last-created block — convenience for expression codegen /// that needs to feed a phi node the predecessor label after compiling a /// sub-expression whose control flow may have split. @@ -1010,10 +1020,7 @@ impl LlFunction { // on it and reintroduce the relocation fan-out the spill avoids. Its // `stack_map_requested` is already false (enable_shadow_frame_inner // took the shadow branch), so this is belt-and-braces. - let gc_strategy = if self.stack_map_requested - && !self.force_shadow_frame - && crate::codegen::helpers::native_stack_roots_enabled() - { + let gc_strategy = if self.uses_statepoint_strategy() { " gc \"statepoint-example\"" } else { "" diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs index be61d57b89..dc19f6fd38 100644 --- a/crates/perry-codegen/src/gc_map.rs +++ b/crates/perry-codegen/src/gc_map.rs @@ -1214,6 +1214,95 @@ struct GcMapStats { roots: usize, } +/// Names of every statepoint-strategy function codegen rendered in this +/// process, recorded only while [`list_unrecorded_functions`] holds. Names are +/// module-prefixed, so a process-wide union across modules is unambiguous. +static STATEPOINT_FUNCTIONS: std::sync::Mutex>> = + std::sync::Mutex::new(None); + +/// RFC deferred collection S5 (the unmapped-frame verifier): list, in the GC +/// map, the statepoint-strategy functions that have NO records — functions +/// whose every call is a `gc-leaf-function`. The runtime keeps them out of the +/// record index (v6 already skips zero-record entries) and uses them only to +/// recognise a generated frame that a collection found at an unmapped call. +/// +/// Instrumented builds only: `PERRY_GC_INSTRUMENTS=1`, `PERRY_GC_VERIFY_FRAMES` +/// or `PERRY_GC_SCHEDULE_SEED` set at compile time — the same condition that +/// links the runtime's instruments. A shipped binary pays no map bytes for it. +pub(crate) fn list_unrecorded_functions() -> bool { + use std::sync::OnceLock; + static ON: OnceLock = OnceLock::new(); + *ON.get_or_init(|| { + let set = |value: Result| { + value.is_ok_and(|v| !v.trim().is_empty()) + }; + std::env::var("PERRY_GC_INSTRUMENTS") + .is_ok_and(|v| matches!(v.trim(), "1" | "true" | "on" | "yes")) + || set(std::env::var("PERRY_GC_VERIFY_FRAMES")) + || set(std::env::var("PERRY_GC_SCHEDULE_SEED")) + }) +} + +/// Record `functions`' statepoint-strategy members for +/// [`append_unrecorded_functions`]. A no-op unless listing is on. +pub(crate) fn note_statepoint_functions(functions: &[&crate::function::LlFunction]) { + if !list_unrecorded_functions() { + return; + } + let mut guard = STATEPOINT_FUNCTIONS.lock().unwrap_or_else(|p| p.into_inner()); + let set = guard.get_or_insert_with(Default::default); + for function in functions { + if function.uses_statepoint_strategy() { + set.insert(function.name.clone()); + } + } +} + +/// Append a zero-record entry for every recorded statepoint-strategy function +/// DEFINED in this assembly that the stack map does not already list. +fn append_unrecorded_functions( + lines: &[&str], + block: &RawBlock, + target: &str, + functions: &mut Vec, +) { + if !list_unrecorded_functions() { + return; + } + let guard = STATEPOINT_FUNCTIONS.lock().unwrap_or_else(|p| p.into_inner()); + let Some(names) = guard.as_ref() else { + return; + }; + let prefix = if matches!(format_for(target), ObjectFormat::MachO) { + "_" + } else { + "" + }; + let mut listed: std::collections::HashSet = + functions.iter().map(|f| f.symbol.clone()).collect(); + for (index, line) in lines.iter().enumerate() { + if (block.start_line..block.end_line).contains(&index) { + continue; + } + let Some((label, rest)) = line.split_once(':') else { + continue; + }; + if !(rest.is_empty() || rest.starts_with([' ', '\t'])) || label.starts_with(['\t', ' ', '.', '"']) { + continue; + } + let Some(name) = label.strip_prefix(prefix) else { + continue; + }; + if names.contains(name) && listed.insert(label.to_string()) { + functions.push(FunctionMap { + symbol: label.to_string(), + stack_size: 0, + records: Vec::new(), + }); + } + } +} + /// Rewrite the LLVM stack-map block in `asm` into the compact map. /// /// Returns `None` when there is no stack-map block to rewrite (the common case @@ -1229,7 +1318,8 @@ fn compact_stack_map_asm(asm: &str, target: &str) -> Result Vec> { let funcs = self.deduped_function_refs(); + crate::gc_map::note_statepoint_functions(&funcs); let gc_leaf_callees = Arc::new(if crate::codegen::helpers::native_stack_roots_enabled() { crate::gc_call_effects::transitive_leaf_functions(&funcs) } else { diff --git a/crates/perry-codegen/src/native_emit.rs b/crates/perry-codegen/src/native_emit.rs index 031d3005f3..38d2357415 100644 --- a/crates/perry-codegen/src/native_emit.rs +++ b/crates/perry-codegen/src/native_emit.rs @@ -95,6 +95,7 @@ fn build_native_module<'ctx>(context: &'ctx Context, llmod: &LlModule) -> Result skeleton.push_str(&format!("declare {} @{}({})\n", f.return_type, f.name, tys)); } let module = crate::inprocess::parse_ir_text(context, &skeleton, "perry_native_module")?; + crate::gc_map::note_statepoint_functions(&funcs); let gc_leaf_callees = crate::gc_call_effects::transitive_leaf_functions(&funcs); let (typed_insts, raw_insts) = stream_functions(context, &module, &funcs, false, &gc_leaf_callees)?; diff --git a/crates/perry-runtime/src/gc/roots/stack_maps.rs b/crates/perry-runtime/src/gc/roots/stack_maps.rs index a9eb6cba38..dd4d5e30c9 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps.rs @@ -1317,10 +1317,7 @@ pub(super) fn visit_stack_map_root_slots( if index.is_empty() { return NativeStackWalkStats::default(); } - // The unmapped-frame verifier needs each frame's function start, which the - // unwinder reports and the x29-chain walk does not. - let mode = if frame_verify::active() { WalkerMode::Unwind } else { walker_mode() }; - match mode { + match walker_mode() { WalkerMode::Unwind => unwind::visit(index, &mut |root: ResolvedRoot| { root.visit_with_context(visit) }), @@ -1830,7 +1827,12 @@ mod fp_chain { // table answers containment exactly and in one binary search, and // a filter that is even slightly too NARROW drops a real frame's // roots — which is not a tradeoff worth making to save a compare. - if let Some(matched) = index.match_records(return_address) { + let matched = index.match_records(return_address); + if matched.is_none() && frame_verify::active() { + let start = frame_verify::function_start_of(return_address); + frame_verify::unmatched_frame(index, return_address, start); + } + if let Some(matched) = matched { // The record describes the caller's frame; its locations are // relative to the caller's own x29, which is exactly the saved // word we just read. diff --git a/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs b/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs index 158fdd0971..59715243d2 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs @@ -25,9 +25,9 @@ //! //! Armed by `PERRY_GC_VERIFY_FRAMES=1`, by a resolved `PERRY_GC_SCHEDULE_SEED` //! (the pairing the RFC names: at `RATE=1` every legal collection point is -//! checked), and in `debug_assertions` builds (`gcaudit`). Off in release: it -//! needs the unwinder rather than the x29-chain walk, and the zero-record -//! entries cost map bytes; see the S5 changelog for the numbers. +//! checked), and in `debug_assertions` builds (`gcaudit`). Off in release by +//! default: every unmatched frame costs an unwind-table lookup, and the +//! zero-record entries cost map bytes; see the S5 changelog for the numbers. use super::StackMapIndex; use std::sync::atomic::{AtomicU64, Ordering}; @@ -82,6 +82,43 @@ fn unmapped_generated_frame(ip: usize, function_start: usize) -> ! { ) } +/// The start of the function containing `return_address`, for the x29-chain +/// walker, which (unlike the unwinder) never learns it. `_Unwind_Find_FDE` +/// answers from the same unwind tables the platform unwinder reads; 0 when +/// there are none. Only called for frames the map did not match, and only +/// while the verifier is armed. +#[cfg(all( + any(target_vendor = "apple", target_os = "linux"), + target_arch = "aarch64" +))] +pub(super) fn function_start_of(return_address: usize) -> usize { + #[repr(C)] + struct DwarfEhBases { + tbase: *mut std::ffi::c_void, + dbase: *mut std::ffi::c_void, + func: *mut std::ffi::c_void, + } + unsafe extern "C" { + fn _Unwind_Find_FDE( + pc: *mut std::ffi::c_void, + bases: *mut DwarfEhBases, + ) -> *const std::ffi::c_void; + } + let mut bases = DwarfEhBases { + tbase: std::ptr::null_mut(), + dbase: std::ptr::null_mut(), + func: std::ptr::null_mut(), + }; + // `- 1`: a call that ends its function returns just past the end. + let pc = return_address.wrapping_sub(1) as *mut std::ffi::c_void; + let fde = unsafe { _Unwind_Find_FDE(pc, &mut bases) }; + if fde.is_null() { + 0 + } else { + bases.func as usize + } +} + /// `(frames_checked, generated_unmapped)` — the verifier's liveness counters. pub(in crate::gc) fn counters() -> (u64, u64) { ( diff --git a/crates/perry/src/commands/compile/build_cache.rs b/crates/perry/src/commands/compile/build_cache.rs index 2a5d1d2f71..250ac08679 100644 --- a/crates/perry/src/commands/compile/build_cache.rs +++ b/crates/perry/src/commands/compile/build_cache.rs @@ -141,6 +141,12 @@ const BUILD_CACHE_ENV_VARS: &[&str] = &[ // #8583: selects the descriptor-backed lowering for large constant arrays. // The two paths emit different IR and therefore require distinct cache keys. "PERRY_GC_SAFEPOINT_ONLY", + // RFC deferred collection S5: any of these at compile time makes the GC + // map list zero-record statepoint functions for the unmapped-frame + // verifier (`gc_map::list_unrecorded_functions`), which changes the object. + "PERRY_GC_INSTRUMENTS", + "PERRY_GC_VERIFY_FRAMES", + "PERRY_GC_SCHEDULE_SEED", "PERRY_INLINE_SHADOW_SLOT", "PERRY_DISABLE_BUFFER_FAST_PATH", "PERRY_VERIFY_NATIVE_REGIONS", diff --git a/crates/perry/src/commands/compile/object_cache.rs b/crates/perry/src/commands/compile/object_cache.rs index b515b1a32c..5f1436e031 100644 --- a/crates/perry/src/commands/compile/object_cache.rs +++ b/crates/perry/src/commands/compile/object_cache.rs @@ -1142,6 +1142,15 @@ fn compute_object_cache_key_with_env( "env_gc_safepoint_only", env_var("PERRY_GC_SAFEPOINT_ONLY").as_deref().unwrap_or(""), ); + // RFC deferred collection S5: an instrumented compile lists zero-record + // statepoint functions in the GC map (the unmapped-frame verifier). + for var in [ + "PERRY_GC_INSTRUMENTS", + "PERRY_GC_VERIFY_FRAMES", + "PERRY_GC_SCHEDULE_SEED", + ] { + h.field(var, env_var(var).as_deref().unwrap_or("")); + } // #7088: flips the shadow-slot store between an inline sequence and the // `js_shadow_slot_*` calls. Two arms that shared a cached object would // silently measure the same code. diff --git a/scripts/gc_valve_ledger_check.py b/scripts/gc_valve_ledger_check.py new file mode 100755 index 0000000000..62fb039b72 --- /dev/null +++ b/scripts/gc_valve_ledger_check.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Gate: no allocation-point GC valve fired (RFC deferred collection, decision 5). + +Every Perry binary run with ``PERRY_GC_VALVE_LEDGER=`` appends exactly one +line to ```` at exit (``perry-runtime/src/gc/alloc_point.rs``):: + + v1 exe= pid= valve_fires= parked_valve_fires= d2_violations= ... + +This script fails when + +* any line records ``valve_fires`` (the nursery slack valve), + ``parked_valve_fires`` (a budgeted cycle's root phase served at an + allocation point) or ``d2_violations`` above zero — the valve is the one + allocation-point collection D2 permits, and it must stay exceptional; or +* the ledger has fewer lines than ``--expect-min`` — the counter that proves + the check ran. A harness that stopped exporting the variable, or a binary + whose exit path skipped the teardown funnel, would otherwise read as clean. + +``--expect-min-from-report`` takes the minimum from a parity report's +``summary.parity_pass``: every test that passed ran a Perry binary to a normal +exit, so each must have written a line. + +``OldReclaimAllocPoint`` is reported (``old_reclaim_alloc_point=``), never +gated: decision 1 keeps that arm at the allocation point. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import tempfile +from pathlib import Path + +GATED = ("valve_fires", "parked_valve_fires", "d2_violations") + + +def parse_line(line: str) -> dict[str, str]: + fields = {} + for token in line.split(): + if "=" in token: + key, value = token.split("=", 1) + fields[key] = value + return fields + + +def check(lines: list[str], expect_min: int) -> list[str]: + errors = [] + records = [line for line in lines if line.startswith("v1 ")] + malformed = [line for line in lines if line.strip() and not line.startswith("v1 ")] + for line in malformed: + errors.append(f"malformed ledger line: {line!r}") + if len(records) < expect_min: + errors.append( + f"only {len(records)} ledger line(s), expected at least {expect_min}: the " + "valve check did not run for every binary (is PERRY_GC_VALVE_LEDGER " + "reaching the tests?)" + ) + old_reclaim = 0 + for line in records: + fields = parse_line(line) + for key in GATED: + try: + value = int(fields.get(key, "missing")) + except ValueError: + errors.append(f"ledger line lacks an integer {key}: {line!r}") + continue + if value > 0: + errors.append(f"{key}={value} in {fields.get('exe', '?')}: {line.strip()}") + try: + old_reclaim += int(fields.get("old_reclaim_alloc_point", "0")) + except ValueError: + pass + print( + f"gc-valve-ledger: {len(records)} binaries checked, " + f"old_reclaim_alloc_point total={old_reclaim} (reported, not gated)" + ) + return errors + + +def expect_min_from_report(path: Path) -> int: + report = json.loads(path.read_text(encoding="utf-8")) + return int(report["summary"]["parity_pass"]) + + +def self_test() -> int: + clean = "v1 exe=a pid=1 valve_fires=0 parked_valve_fires=0 d2_violations=0 old_reclaim_alloc_point=2\n" + fired = "v1 exe=b pid=2 valve_fires=1 parked_valve_fires=0 d2_violations=0\n" + parked = "v1 exe=c pid=3 valve_fires=0 parked_valve_fires=4 d2_violations=0\n" + assert check([clean, clean], 2) == [] + assert check([clean, fired], 2), "a fired valve must fail" + assert check([clean, parked], 1), "a fired parked-cycle valve must fail" + assert check([clean], 2), "too few lines must fail (the check did not run)" + assert check([], 0) == [], "an explicit zero minimum with no lines is allowed" + assert check(["garbage\n"], 0), "a malformed line must fail" + with tempfile.TemporaryDirectory() as tmp: + report = Path(tmp) / "latest.json" + report.write_text(json.dumps({"summary": {"parity_pass": 7}}), encoding="utf-8") + assert expect_min_from_report(report) == 7 + print("gc_valve_ledger_check self-test: ok") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--ledger", type=Path) + parser.add_argument("--expect-min", type=int, default=None) + parser.add_argument("--expect-min-from-report", type=Path, default=None) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + if args.self_test: + return self_test() + if args.ledger is None: + parser.error("--ledger is required") + if args.expect_min is None and args.expect_min_from_report is None: + parser.error("one of --expect-min / --expect-min-from-report is required") + expect_min = args.expect_min or 0 + if args.expect_min_from_report is not None: + expect_min = max(expect_min, expect_min_from_report(args.expect_min_from_report)) + if expect_min < 1: + print("gc-valve-ledger: refusing a minimum of 0 — the gate could not fail", file=sys.stderr) + return 1 + lines = ( + args.ledger.read_text(encoding="utf-8").splitlines(keepends=True) + if args.ledger.exists() + else [] + ) + errors = check(lines, expect_min) + for error in errors: + print(f"gc-valve-ledger: FAIL: {error}", file=sys.stderr) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gc_valve_ratchet_probes.sh b/scripts/gc_valve_ratchet_probes.sh new file mode 100755 index 0000000000..52fdc2b968 --- /dev/null +++ b/scripts/gc_valve_ratchet_probes.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Decision 5 of RFC deferred collection, the ratchet-probe half: compile every +# GC ratchet probe once, run it once with PERRY_GC_VALVE_LEDGER set, and fail +# if any allocation-point valve fired or if any probe did not report +# (scripts/gc_valve_ledger_check.py). The gap-suite half runs in the gap-suite +# shards of test.yml. +# +# Usage: scripts/gc_valve_ratchet_probes.sh [path-to-perry] +set -euo pipefail + +PERRY_BIN="${1:-target/release/perry}" +if [[ ! -x "$PERRY_BIN" ]]; then + echo "FAIL: no perry binary at $PERRY_BIN" >&2 + exit 1 +fi +PERRY_BIN="$(cd "$(dirname "$PERRY_BIN")" && pwd)/$(basename "$PERRY_BIN")" +export PERRY_RUNTIME_DIR="${PERRY_RUNTIME_DIR:-$(dirname "$PERRY_BIN")}" +export PERRY_NO_AUTO_OPTIMIZE=1 + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +LEDGER="$WORK/ledger.txt" + +python3 "$ROOT/scripts/gc_valve_ledger_check.py" --self-test + +count=0 +for probe in "$ROOT"/benchmarks/gc_ratchet/probes/*.ts; do + name="$(basename "$probe" .ts)" + "$PERRY_BIN" compile "$probe" -o "$WORK/$name" > "$WORK/$name.compile.log" 2>&1 || { + echo "FAIL: compiling $name" >&2 + tail -20 "$WORK/$name.compile.log" >&2 + exit 1 + } + PERRY_GC_VALVE_LEDGER="$LEDGER" "$WORK/$name" > /dev/null + count=$((count + 1)) +done +echo "ran $count ratchet probes" +python3 "$ROOT/scripts/gc_valve_ledger_check.py" --ledger "$LEDGER" --expect-min "$count" From 88e54a33fdd2fce49e1353730627a2c2b4d3ac19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 10:28:28 +0200 Subject: [PATCH 03/15] wip: S5 fixes --- crates/perry-runtime/src/gc/alloc_point.rs | 37 +++++++++++----- crates/perry-runtime/src/gc/policy.rs | 38 ++++++++-------- crates/perry-runtime/src/gc/roots.rs | 3 +- .../perry-runtime/src/gc/roots/scan_mode.rs | 15 +++++++ .../src/gc/tests/alloc_point_invariant.rs | 6 ++- docs/src/internals/garbage-collector.md | 43 +++++++++++++++++-- 6 files changed, 109 insertions(+), 33 deletions(-) diff --git a/crates/perry-runtime/src/gc/alloc_point.rs b/crates/perry-runtime/src/gc/alloc_point.rs index 51c70add6a..e00dbbf70e 100644 --- a/crates/perry-runtime/src/gc/alloc_point.rs +++ b/crates/perry-runtime/src/gc/alloc_point.rs @@ -88,13 +88,18 @@ pub(super) fn with_allocation_point_lifted(f: impl FnOnce() -> R) -> R { } /// D2's enforcement for synchronous collections: a collection that begins at -/// an allocation point must scan conservatively (and therefore cannot move). +/// an allocation point must be one of the conservative arms, i.e. its caller +/// must have requested the conservative scan (`ManualGcScanGuard::force_full_scan`, +/// which also makes the copying minor ineligible). /// /// Called at the two synchronous chokepoints (`gc_collect_minor_with_trigger_inner` /// and `gc_collect_full_mark_sweep_with_trigger`). Every allocation-point arm /// that collects — OldReclaim, the nursery valve, the polls-off direct minor, -/// the emergency reclaim — takes `ManualGcScanGuard::force_full_scan` first, -/// so this is structurally unreachable. It panics in every build rather than +/// the emergency reclaim — takes that guard first, so this is structurally +/// unreachable. It checks the REQUEST, not the resulting scan decision: the +/// unit-test isolation guards and the `PERRY_CONSERVATIVE_STACK_SCAN=off` +/// bisection escape hatch both override the decision on purpose, and neither is +/// a new path into a precise collection. It panics in every build rather than /// healing: a heal path nothing can reach is an untested mode (the kill /// policy), and the check is one thread-local read per collection. #[inline] @@ -102,10 +107,7 @@ pub(super) fn assert_d2_synchronous_collection() { if !at_allocation_point() { return; } - if matches!( - super::roots::conservative_stack_scan_decision(), - super::roots::ConservativeStackScanDecision::Scan - ) { + if super::roots::conservative_scan_requested() { return; } D2_VIOLATIONS.fetch_add(1, Ordering::Relaxed); @@ -157,6 +159,8 @@ pub(super) fn root_phase_parked() -> bool { /// The parked cycle has waited `slack` arena bytes past its park point with no /// poll to serve it. pub(super) fn parked_valve_due(arena_total: usize, slack: usize) -> bool { + #[cfg(test)] + let slack = TEST_SLACK.with(Cell::get).unwrap_or(slack); PARKED_AT.with(|parked| { parked .get() @@ -333,9 +337,20 @@ pub(super) fn write_valve_ledger_line() { } } -/// Move the park point back so the parked-cycle valve is due on the next -/// allocation point, without allocating the slack for real. #[cfg(test)] -pub(crate) fn test_set_park_base(base: usize) { - PARKED_AT.with(|parked| parked.set(Some(base))); +thread_local! { + static TEST_SLACK: Cell> = const { Cell::new(None) }; +} + +/// Make the parked-cycle valve due on the next allocation point, without +/// allocating the slack for real: park "at zero" with a zero slack. +#[cfg(test)] +pub(crate) fn test_make_parked_valve_due() { + PARKED_AT.with(|parked| parked.set(Some(0))); + TEST_SLACK.with(|slack| slack.set(Some(0))); +} + +#[cfg(test)] +pub(crate) fn test_clear_parked_valve_override() { + TEST_SLACK.with(|slack| slack.set(None)); } diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 267de78090..651cec3075 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -3791,23 +3791,27 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { let in_alloc = flags & (GC_FLAG_IN_ALLOC | GC_FLAG_SUPPRESSED) != 0; let unsafe_zone = gc_blocked_by_unsafe_zone(); let root_lock = GC_ROOT_LOCK_DEPTH.with(|depth| depth.get() != 0); - if !(in_alloc || unsafe_zone || root_lock) { - // S5: a collection a root-lock exit owed runs here, at the declared - // poll, rather than at the lock exit (`flush_deferred_gc_request`). - if serve_owed_request_at_poll() { - set_safepoint_pending(false); - return true; - } - // S5 (D2): the budgeted cycle's frame-root phases run at declared - // points only. An allocation point that reached one parked the cycle - // and armed this poll; serve the phase now, with precise roots. - if gc_budgeted_cycle_active() && budgeted_cycle_next_step_reads_frame_roots() { - let _declared = DeclaredSafepointGuard::enter(); - serve_budgeted_root_phase(); - super::alloc_point::note_root_phase_served_at_poll(); - set_safepoint_pending(false); - return true; - } + // S5: a collection a root-lock exit owed runs here, at the declared poll, + // rather than at the lock exit (`flush_deferred_gc_request`). + if !(in_alloc || unsafe_zone || root_lock) && serve_owed_request_at_poll() { + set_safepoint_pending(false); + return true; + } + // S5 (D2): the budgeted cycle's frame-root phases run at declared points + // only. An allocation point that reached one parked the cycle and armed + // this poll; serve the phase now, with precise roots. The guard is the + // budgeted stepper's own resume guard, not the one above: a budgeted MINOR + // holds `GC_FLAG_IN_ALLOC` for its whole life (`new_minor_fallback`), so + // `in_alloc` is always set while one is parked. + if gc_budgeted_cycle_active() + && budgeted_cycle_next_step_reads_frame_roots() + && !gc_budgeted_resume_blocked() + { + let _declared = DeclaredSafepointGuard::enter(); + serve_budgeted_root_phase(); + super::alloc_point::note_root_phase_served_at_poll(); + set_safepoint_pending(false); + return true; } let budgeted = gc_budgeted_cycle_active(); if in_alloc || unsafe_zone || root_lock || budgeted { diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index caebb227b4..ac42b0b0de 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -46,7 +46,8 @@ pub(crate) use stack_roots::with_stack_roots; // `shadow_stack` re-exports below. #[allow(unused_imports)] pub(crate) use scan_mode::{ - conservative_stack_scan_decision, conservative_stack_scan_decision_for, + conservative_scan_requested, conservative_stack_scan_decision, + conservative_stack_scan_decision_for, conservative_stack_scan_mode, conservative_stack_scan_mode_from_value, resolve_conservative_stack_scan_mode, set_conservative_stack_scan_override, ConservativeStackScanDecision, ConservativeStackScanMode, ManualGcScanGuard, diff --git a/crates/perry-runtime/src/gc/roots/scan_mode.rs b/crates/perry-runtime/src/gc/roots/scan_mode.rs index 305a51dea6..f89cd484ec 100644 --- a/crates/perry-runtime/src/gc/roots/scan_mode.rs +++ b/crates/perry-runtime/src/gc/roots/scan_mode.rs @@ -104,9 +104,23 @@ pub(crate) struct ManualGcScanGuard { engaged: bool, } +thread_local! { + /// How many `ManualGcScanGuard`s are live on this thread, whether or not + /// each managed to pin the override. The allocation-point invariant + /// (`gc/alloc_point.rs`) checks this REQUEST, which no override can hide. + static SCAN_REQUESTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Whether some caller on this thread has requested the conservative scan for +/// the collection now running. +pub(crate) fn conservative_scan_requested() -> bool { + SCAN_REQUESTS.with(|c| c.get() != 0) +} + impl ManualGcScanGuard { pub(crate) fn force_full_scan(site: super::ConservativeScanSite) -> Self { super::record_scan_fallback(site); + SCAN_REQUESTS.with(|c| c.set(c.get() + 1)); let engaged = CONSERVATIVE_STACK_SCAN_OVERRIDE.with(|c| { if c.get().is_some() { return false; @@ -120,6 +134,7 @@ impl ManualGcScanGuard { impl Drop for ManualGcScanGuard { fn drop(&mut self) { + SCAN_REQUESTS.with(|c| c.set(c.get().saturating_sub(1))); if self.engaged { CONSERVATIVE_STACK_SCAN_OVERRIDE.with(|c| c.set(None)); } diff --git a/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs b/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs index cc879b6662..fa8fd9bd33 100644 --- a/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs +++ b/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs @@ -153,6 +153,9 @@ fn a_conservative_collection_at_an_allocation_point_is_allowed() { let before = gc_collection_count(); { let _alloc_point = alloc_point::AllocationPointGuard::enter(); + // The request is what D2 checks; the isolation guard above has pinned + // the scan decision itself, exactly as it does for every A-old/valve + // test in this crate. let _scan = ManualGcScanGuard::force_full_scan(ConservativeScanSite::NurseryChurnSlackValve); let _ = gc_collect_minor_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct)); @@ -204,8 +207,9 @@ fn parked_cycle_valve_fires_after_the_slack_and_is_counted() { assert_eq!(assist_until_parked(10_000), GcCyclePhase::RootScan.ffi_code()); // Pretend the program allocated the whole slack since parking. - alloc_point::test_set_park_base(0); + alloc_point::test_make_parked_valve_due(); gc_check_trigger(); + alloc_point::test_clear_parked_valve_override(); let counters = alloc_point::alloc_point_counters(); assert_eq!(counters.parked_valve_fires, 1, "the valve fired once"); diff --git a/docs/src/internals/garbage-collector.md b/docs/src/internals/garbage-collector.md index 69006a2d45..315d6a9331 100644 --- a/docs/src/internals/garbage-collector.md +++ b/docs/src/internals/garbage-collector.md @@ -68,6 +68,41 @@ ladder and fail on a mismatch (`crates/perry-runtime/src/gc/trigger_watermark.rs +**Where a collection may begin (RFC deferred collection, S5).** A phase that +reads frame roots precisely, or that moves anything, begins only at a declared +point: a loop back-edge or function-entry poll, the outermost microtask-pump +boundary, a host step (the event loop, regex quanta), or an explicit +collection request (`gc()`, `perry/gc`, memory pressure, idle reclaim). An +allocation — the dynamic extent of `gc_check_trigger`, which every allocation +slow path, the JSON mid-parse checks and the root-lock flush of a deferred +trigger check funnel into — may only take a block, arm the poll, run heap-only +budgeted work, or run one of the conservative non-moving arms: the nursery +slack valve, the old-gen reclaim arm, or the emergency reclaim. A budgeted +cycle that reaches its root scan or final remark from an allocation is parked +and served by the next poll; a collection requested while a root lock was held +runs at the next poll rather than at the lock exit. A synchronous collection +begun at an allocation point without requesting the conservative scan panics +in every build (`crates/perry-runtime/src/gc/alloc_point.rs`). The one remaining precise +read at an allocation point is the parked-cycle valve (a program that allocates +the valve slack past the park point without reaching any poll), which is +counted and held to zero in CI together with the nursery valve: +`PERRY_GC_VALVE_LEDGER=` makes every process append one line at exit, +and `scripts/gc_valve_ledger_check.py` gates the gap suite and the ratchet +probes on it. `PERRY_GC_DIAG=1` prints the counters on `[gc-alloc-point]`. + + + + +**The unmapped-frame verifier.** A precise collection that walks a generated +frame whose return address has no stack-map record — a frame suspended at a +call compiled as `gc-leaf-function` whose callee collected anyway — panics +instead of skipping the frame (`crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs`). +It is armed by `PERRY_GC_VERIFY_FRAMES=1`, by `PERRY_GC_SCHEDULE_SEED`, and in +`debug_assertions` builds. A program compiled with the GC instruments also lists +its zero-record statepoint functions in the GC map, so the verifier recognises +a generated frame even in a function whose every call is a leaf. + + **Per-live-object cost of a synchronous full.** Three parts of a full scale with the live set, and each has a cheaper exact form: @@ -351,14 +386,16 @@ Rooting stress uses `PERRY_GC_SCHEDULE_SEED`, `PERRY_GC_SCHEDULE_RATE`, `PERRY_GC_SCHEDULE_ALLOC_KB`, `PERRY_GC_FORCE_EVACUATE`, `PERRY_GC_VERIFY_EVACUATION`, `PERRY_GC_PROTECT_FROMSPACE`, `PERRY_GC_PROTECT_FROMSPACE_DEPTH`, -`PERRY_GC_FROMSPACE_SCAN`, and `PERRY_GC_FROMSPACE_SCAN_ABORT`. Their exact +`PERRY_GC_FROMSPACE_SCAN`, `PERRY_GC_FROMSPACE_SCAN_ABORT` and +`PERRY_GC_VERIFY_FRAMES`. Their exact contracts and non-vacuity requirements live in the [rooting invariant](gc-rooting-invariant.md). Research/bisection controls such as `PERRY_GC_INCREMENTAL`, `PERRY_GC_IDLE_RECLAIM`, `PERRY_GC_MAJOR_PACING_FLOOR_MB`, `PERRY_GC_MAJOR_PACING_GROWTH`, `PERRY_GC_MOVING_SAFEPOINT`, `PERRY_GC_MOVING_LOOP_POLLS`, -`PERRY_GC_SAFEPOINT_ONLY`, and `PERRY_STACKMAP_WALKER` are accepted but are not -additional supported collector modes. +`PERRY_GC_SAFEPOINT_ONLY` (a codegen-only research switch since S5 made its +runtime contract the default invariant), and `PERRY_STACKMAP_WALKER` are +accepted but are not additional supported collector modes. `scripts/check_gc_env_knobs.py` derives the accepted names from live runtime/codegen/compiler parsers and rejects a current document, executable From 7c74076dcdfe4b14e132aa245d7151631fa99a21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 10:31:51 +0200 Subject: [PATCH 04/15] wip: S5 poll-wait metric --- crates/perry-runtime/src/gc/alloc_point.rs | 21 +++++++++++++++++++-- crates/perry-runtime/src/gc/policy.rs | 13 +++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/crates/perry-runtime/src/gc/alloc_point.rs b/crates/perry-runtime/src/gc/alloc_point.rs index e00dbbf70e..97661e7630 100644 --- a/crates/perry-runtime/src/gc/alloc_point.rs +++ b/crates/perry-runtime/src/gc/alloc_point.rs @@ -127,6 +127,17 @@ static OWED_REQUESTS_ROUTED: AtomicU64 = AtomicU64::new(0); static OWED_REQUESTS_SERVED: AtomicU64 = AtomicU64::new(0); static UNSAFE_ZONE_GROWTH_BYTES: AtomicU64 = AtomicU64::new(0); static UNSAFE_ZONE_GROWTH_EVENTS: AtomicU64 = AtomicU64::new(0); +static MAX_POLL_WAIT_BYTES: AtomicU64 = AtomicU64::new(0); + +/// A deferred collection was drained (at a poll) or given up on (by the +/// valve) `waited` arena bytes after it was armed. The maximum is the measured +/// answer to decision 3's question — how far does a program allocate between +/// arming a collection and reaching a poll — and it is what a straight-line +/// body that needs statement-boundary polls would show. +#[inline] +pub(super) fn note_poll_wait(waited: usize) { + MAX_POLL_WAIT_BYTES.fetch_max(waited as u64, Ordering::Relaxed); +} /// An allocation point found the active budgeted cycle about to read frame /// roots. Arms the poll (via `arm`) the first time for this park and records @@ -229,6 +240,7 @@ pub struct AllocPointCounters { pub owed_requests_served: u64, pub unsafe_zone_growth_bytes: u64, pub unsafe_zone_growth_events: u64, + pub max_poll_wait_bytes: u64, } pub fn alloc_point_counters() -> AllocPointCounters { @@ -241,6 +253,7 @@ pub fn alloc_point_counters() -> AllocPointCounters { owed_requests_served: OWED_REQUESTS_SERVED.load(Ordering::Relaxed), unsafe_zone_growth_bytes: UNSAFE_ZONE_GROWTH_BYTES.load(Ordering::Relaxed), unsafe_zone_growth_events: UNSAFE_ZONE_GROWTH_EVENTS.load(Ordering::Relaxed), + max_poll_wait_bytes: MAX_POLL_WAIT_BYTES.load(Ordering::Relaxed), } } @@ -255,6 +268,7 @@ pub(crate) fn reset_alloc_point_counters() { &OWED_REQUESTS_SERVED, &UNSAFE_ZONE_GROWTH_BYTES, &UNSAFE_ZONE_GROWTH_EVENTS, + &MAX_POLL_WAIT_BYTES, ] { counter.store(0, Ordering::Relaxed); } @@ -268,7 +282,7 @@ pub(super) fn alloc_point_exit_line() -> String { "[gc-alloc-point] valve_fires={} parked_valve_fires={} old_reclaim_alloc_point={} \ emergency_reclaims={} root_phases_parked={} root_phases_served_at_poll={} \ owed_requests_routed={} owed_requests_served={} safepoint_drains={} \ - unsafe_zone_growth_bytes={} unsafe_zone_growth_events={}", + unsafe_zone_growth_bytes={} unsafe_zone_growth_events={} max_poll_wait_bytes={}", super::scan_fallback::scan_fallback_count_any_thread( super::ConservativeScanSite::NurseryChurnSlackValve ), @@ -286,6 +300,7 @@ pub(super) fn alloc_point_exit_line() -> String { super::scan_fallback::safepoint_drain_total_any_thread(), c.unsafe_zone_growth_bytes, c.unsafe_zone_growth_events, + c.max_poll_wait_bytes, ) } @@ -318,7 +333,8 @@ pub(super) fn write_valve_ledger_line() { .unwrap_or_else(|| "?".to_string()); let line = format!( "v1 exe={exe} pid={} valve_fires={valve} parked_valve_fires={} d2_violations={} \ - old_reclaim_alloc_point={} root_phases_served_at_poll={} safepoint_drains={}\n", + old_reclaim_alloc_point={} root_phases_served_at_poll={} safepoint_drains={} \ + max_poll_wait_bytes={}\n", std::process::id(), c.parked_valve_fires, c.d2_violations, @@ -327,6 +343,7 @@ pub(super) fn write_valve_ledger_line() { ), c.root_phases_served_at_poll, super::scan_fallback::safepoint_drain_total_any_thread(), + c.max_poll_wait_bytes, ); if let Ok(mut file) = std::fs::OpenOptions::new() .create(true) diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 651cec3075..b7bcc348b5 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -3333,6 +3333,7 @@ fn gc_check_trigger_evaluate() { } return; } + note_pending_poll_wait(); // The deferral never drained. The direct minor below IS the // collection that was owed, so retire the request — leaving it // pending would pin `GC_SAFEPOINT_DEFER_ARENA_BASE` at a stale, @@ -3794,6 +3795,7 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { // S5: a collection a root-lock exit owed runs here, at the declared poll, // rather than at the lock exit (`flush_deferred_gc_request`). if !(in_alloc || unsafe_zone || root_lock) && serve_owed_request_at_poll() { + note_pending_poll_wait(); set_safepoint_pending(false); return true; } @@ -3810,6 +3812,7 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { let _declared = DeclaredSafepointGuard::enter(); serve_budgeted_root_phase(); super::alloc_point::note_root_phase_served_at_poll(); + note_pending_poll_wait(); set_safepoint_pending(false); return true; } @@ -3852,6 +3855,7 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { // `set_safepoint_pending`, not a raw `.set(false)`: since #7735 the pending // flag is mirrored into the poll arming word, and clearing it behind the // mirror would leave the back-edge poll armed forever. + note_pending_poll_wait(); set_safepoint_pending(false); let _declared = DeclaredSafepointGuard::enter(); let kind = match due { @@ -4682,6 +4686,15 @@ fn defer_nursery_cap_to_precise_safepoint() { arm_precise_safepoint(); } +/// Record how far the arena grew between arming the pending poll and now +/// (`alloc_point::note_poll_wait`). A no-op when nothing was pending. +fn note_pending_poll_wait() { + if GC_SAFEPOINT_PENDING.with(Cell::get) { + let base = GC_SAFEPOINT_DEFER_ARENA_BASE.with(Cell::get); + super::alloc_point::note_poll_wait(crate::arena::arena_total_bytes().saturating_sub(base)); + } +} + /// Arm the next declared poll, recording the arena baseline the nursery valve /// measures its slack from — exactly as the nursery deferral does. Shared by /// every arm that hands work to a poll: the nursery cap, a parked budgeted From b9f39a1b927d292ca8d0a95b2cc54d992eb97f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 10:35:49 +0200 Subject: [PATCH 05/15] wip: S5 verifier arm + intent skip --- .../src/gc/roots/stack_maps_frame_verify.rs | 18 ++++++++---- scripts/gc_instrument_smoke.sh | 28 +++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs b/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs index 59715243d2..3acd9c2326 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs @@ -21,7 +21,8 @@ //! format did not change. //! //! Only a PRECISE collection is checked: a conservative one scans the whole -//! native stack, unmapped frames included, and does not move. +//! native stack, unmapped frames included, and does not move. So is a +//! collection whose caller requested the scan (the allocation-point arms). //! //! Armed by `PERRY_GC_VERIFY_FRAMES=1`, by a resolved `PERRY_GC_SCHEDULE_SEED` //! (the pairing the RFC names: at `RATE=1` every legal collection point is @@ -59,10 +60,17 @@ pub(super) fn unmatched_frame(index: &StackMapIndex, ip: usize, function_start: if function_start == 0 || !index.is_generated_function(function_start) { return; } - if matches!( - crate::gc::conservative_stack_scan_decision(), - crate::gc::ConservativeStackScanDecision::Scan - ) { + // A conservative collection scanned this frame's words and does not move. + // A collection that REQUESTED the scan is one of the allocation-point arms + // D2 allows, even when `PERRY_CONSERVATIVE_STACK_SCAN=off` (the bisection + // escape hatch) overrode the request: that frame is at an allocating call, + // which is exactly where the escape hatch is documented to be unsound. + if crate::gc::conservative_scan_requested() + || matches!( + crate::gc::conservative_stack_scan_decision(), + crate::gc::ConservativeStackScanDecision::Scan + ) + { return; } GENERATED_UNMAPPED.fetch_add(1, Ordering::Relaxed); diff --git a/scripts/gc_instrument_smoke.sh b/scripts/gc_instrument_smoke.sh index ac2a0e4776..4e6b7fe096 100755 --- a/scripts/gc_instrument_smoke.sh +++ b/scripts/gc_instrument_smoke.sh @@ -206,6 +206,34 @@ echo echo " [seeded schedule] pressure-only=$pressure_retired < seeded(0.25)=$sched_retired < rate-1=$rate1_retired" echo " [seeded schedule] same seed twice: $sched_retired == $sched_repeat (reproducible)" +# ---- arm 8: the unmapped-frame verifier (RFC deferred collection S5) --------- +# +# `PERRY_GC_VERIFY_FRAMES=1` and a resolved schedule seed both arm the +# verifier: a precise collection that walks a generated frame with no stack +# map at its call panics. Non-vacuity: `frames_verified` counts the unmatched +# frames it classified, so a run whose collections never walked a native frame +# (shadow-frame build, no precise collection) reads as 0 and fails here. +echo +echo "== arm 8: unmapped-frame verifier (explicit knob, then the seed) ==" +for arm in "PERRY_GC_VERIFY_FRAMES=1" "PERRY_GC_SCHEDULE_SEED=1 PERRY_GC_SCHEDULE_RATE=1"; do + set +e + # shellcheck disable=SC2086 + out="$(env $arm PERRY_GC_MOVING_LOOP_POLLS=1 PERRY_GC_DIAG=1 "$WORK/fixture" 2>&1)" + rc=$? + set -e + if [[ $rc -ne 0 ]] || ! grep -q '^bad 0$' <<<"$out"; then + echo "FAIL [verify-frames: $arm]: exit $rc" >&2 + echo "$out" | grep -E 'safety net|panicked|^bad' | head -5 >&2 + exit 1 + fi + verified="$(grep -o 'frames_verified=[0-9]*' <<<"$out" | head -1 | cut -d= -f2)" + if [[ -z "$verified" || "$verified" -eq 0 ]]; then + echo "FAIL [verify-frames: $arm]: frames_verified=${verified:-missing}; the verifier never ran." >&2 + exit 1 + fi + echo " [verify-frames: $arm] clean, frames_verified=$verified" +done + # ---- arm 7: the quarantine, aimed at real programs -------------------------- # # #7341. Everything above drives the instrument with PERRY_GC_MOVING_LOOP_POLLS From de5748e615bd0172c678ea30761cb9686ee5145e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 10:40:18 +0200 Subject: [PATCH 06/15] wip: lock-flush tests serve at the poll --- .../src/gc/tests/noncollecting_root_lock.rs | 7 +++++++ crates/perry-runtime/src/gc/tests/roots.rs | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/crates/perry-runtime/src/gc/tests/noncollecting_root_lock.rs b/crates/perry-runtime/src/gc/tests/noncollecting_root_lock.rs index 823678104f..58d60adf5b 100644 --- a/crates/perry-runtime/src/gc/tests/noncollecting_root_lock.rs +++ b/crates/perry-runtime/src/gc/tests/noncollecting_root_lock.rs @@ -115,6 +115,10 @@ fn the_same_plant_under_an_ordinary_lock_collects_on_release() { !deferred_gc_request_pending(), "an ordinary release flushes" ); + // RFC deferred collection S5: the flush hands a collection to the next + // declared poll (a plain trigger check stays an allocation-point check, + // which defers to the poll too); the poll runs it. + assert!(gc_safepoint_moving_minor()); assert!( gc_collection_count() > before, "the flushed request must have run a collection" @@ -145,5 +149,8 @@ fn clean_and_inherited_requests_are_not_violations() { assert_eq!(gc_collection_count(), before); } assert!(!deferred_gc_request_pending()); + // S5: the outer release hands the collection to the next poll. + assert_eq!(gc_collection_count(), before); + assert!(gc_safepoint_moving_minor()); assert!(gc_collection_count() > before); } diff --git a/crates/perry-runtime/src/gc/tests/roots.rs b/crates/perry-runtime/src/gc/tests/roots.rs index dbccd89229..a006d9b085 100644 --- a/crates/perry-runtime/src/gc/tests/roots.rs +++ b/crates/perry-runtime/src/gc/tests/roots.rs @@ -195,6 +195,10 @@ fn lock_safe_runtime_scanners_tui_hooks_defers_direct_minor_gc() { ); }); + // RFC deferred collection S5 ("D stops collecting"): the lock exit hands + // the owed collection to the next declared poll instead of running it. + assert_eq!(gc_collection_count(), before, "the lock exit must not collect (S5)"); + assert!(gc_safepoint_moving_minor(), "the poll must serve the owed collection"); assert!( gc_collection_count() > before, "deferred direct minor GC should run after the hook root lock is released" @@ -231,6 +235,10 @@ fn lock_safe_runtime_scanners_tui_state_defers_manual_gc() { ); }); + // RFC deferred collection S5 ("D stops collecting"): the lock exit hands + // the owed collection to the next declared poll instead of running it. + assert_eq!(gc_collection_count(), before, "the lock exit must not collect (S5)"); + assert!(gc_safepoint_moving_minor(), "the poll must serve the owed collection"); assert!( gc_collection_count() > before, "deferred manual GC should run after the state root lock is released" @@ -397,6 +405,10 @@ fn lock_safe_runtime_scanners_tui_hooks_defers_direct_full_gc() { ); }); + // RFC deferred collection S5 ("D stops collecting"): the lock exit hands + // the owed collection to the next declared poll instead of running it. + assert_eq!(gc_collection_count(), before, "the lock exit must not collect (S5)"); + assert!(gc_safepoint_moving_minor(), "the poll must serve the owed collection"); assert!( gc_collection_count() > before, "deferred direct full GC should run after the hook root lock is released" From d4f4884c07213c9400f5a7f0a08e2ff58db8317b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 10:43:43 +0200 Subject: [PATCH 07/15] wip: keep poll armed for an owed request --- crates/perry-runtime/src/gc/policy.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index b7bcc348b5..a1e4ac13b3 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1593,9 +1593,13 @@ fn serve_owed_request_at_poll() -> bool { true } +fn owed_request_pending() -> bool { + GC_POLL_OWED_REQUEST.with(|owed| !matches!(owed.get(), DeferredGcRequest::None)) +} + #[cfg(test)] pub(super) fn poll_owed_request_pending() -> bool { - GC_POLL_OWED_REQUEST.with(|owed| !matches!(owed.get(), DeferredGcRequest::None)) + owed_request_pending() } pub fn gc_suppress() { @@ -3812,8 +3816,12 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { let _declared = DeclaredSafepointGuard::enter(); serve_budgeted_root_phase(); super::alloc_point::note_root_phase_served_at_poll(); - note_pending_poll_wait(); - set_safepoint_pending(false); + // An owed collection the in-alloc guard above held back (a budgeted + // minor holds `GC_FLAG_IN_ALLOC`) keeps the poll armed for later. + if !owed_request_pending() { + note_pending_poll_wait(); + set_safepoint_pending(false); + } return true; } let budgeted = gc_budgeted_cycle_active(); From e477ae92de7a1ea3809526b9dc7a55fc4a262b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 10:45:02 +0200 Subject: [PATCH 08/15] wip: gc map listing follows re-render --- crates/perry-codegen/src/gc_map.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs index dc19f6fd38..f1de294a39 100644 --- a/crates/perry-codegen/src/gc_map.rs +++ b/crates/perry-codegen/src/gc_map.rs @@ -1252,8 +1252,13 @@ pub(crate) fn note_statepoint_functions(functions: &[&crate::function::LlFunctio let mut guard = STATEPOINT_FUNCTIONS.lock().unwrap_or_else(|p| p.into_inner()); let set = guard.get_or_insert_with(Default::default); for function in functions { + // The latest render is authoritative: the RS4GC budget retry can move a + // function onto a shadow frame and render it again, and a stale entry + // would make the verifier treat its unmapped frames as a leaf bug. if function.uses_statepoint_strategy() { set.insert(function.name.clone()); + } else { + set.remove(&function.name); } } } From 4945608f6737984142c941688cc3603b2e099e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 11:03:35 +0200 Subject: [PATCH 09/15] wip: custody inventories, test-only helpers --- crates/perry-runtime/src/gc/alloc_point.rs | 24 +++++++---------- .../src/gc/roots/stack_maps_lazy.rs | 1 + scripts/gc_runtime_root_holders.json | 26 ++++++++++++++++--- scripts/thread_exit_address_globals.json | 8 ++++++ 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/crates/perry-runtime/src/gc/alloc_point.rs b/crates/perry-runtime/src/gc/alloc_point.rs index 97661e7630..ed9f860adf 100644 --- a/crates/perry-runtime/src/gc/alloc_point.rs +++ b/crates/perry-runtime/src/gc/alloc_point.rs @@ -162,7 +162,7 @@ pub(super) fn clear_park() { } /// Whether a cycle is currently parked at a frame-root phase on this thread. -#[inline] +#[cfg(test)] pub(super) fn root_phase_parked() -> bool { PARKED_AT.with(|parked| parked.get().is_some()) } @@ -259,19 +259,15 @@ pub fn alloc_point_counters() -> AllocPointCounters { #[cfg(test)] pub(crate) fn reset_alloc_point_counters() { - for counter in [ - &D2_VIOLATIONS, - &ROOT_PHASES_PARKED, - &ROOT_PHASES_SERVED_AT_POLL, - &PARKED_VALVE_FIRES, - &OWED_REQUESTS_ROUTED, - &OWED_REQUESTS_SERVED, - &UNSAFE_ZONE_GROWTH_BYTES, - &UNSAFE_ZONE_GROWTH_EVENTS, - &MAX_POLL_WAIT_BYTES, - ] { - counter.store(0, Ordering::Relaxed); - } + D2_VIOLATIONS.store(0, Ordering::Relaxed); + ROOT_PHASES_PARKED.store(0, Ordering::Relaxed); + ROOT_PHASES_SERVED_AT_POLL.store(0, Ordering::Relaxed); + PARKED_VALVE_FIRES.store(0, Ordering::Relaxed); + OWED_REQUESTS_ROUTED.store(0, Ordering::Relaxed); + OWED_REQUESTS_SERVED.store(0, Ordering::Relaxed); + UNSAFE_ZONE_GROWTH_BYTES.store(0, Ordering::Relaxed); + UNSAFE_ZONE_GROWTH_EVENTS.store(0, Ordering::Relaxed); + MAX_POLL_WAIT_BYTES.store(0, Ordering::Relaxed); clear_park(); } diff --git a/crates/perry-runtime/src/gc/roots/stack_maps_lazy.rs b/crates/perry-runtime/src/gc/roots/stack_maps_lazy.rs index 0e7e22d7a2..90c804313f 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps_lazy.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps_lazy.rs @@ -349,6 +349,7 @@ pub(super) fn unzigzag(value: u32) -> i32 { /// /// `origin` is the runtime address of `bytes[0]`; the v6 function fields are /// offsets from their blob, so the table cannot be read without it. +#[cfg(test)] pub(super) fn parse_function_table( section: u16, bytes: &[u8], diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 04faee59c5..4b752aa16d 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -386,7 +386,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete \u2192 sweep-entry window of a synchronous full \u2014 where PASS1_MARKED is populated and consumed within one `run_to_completion` \u2014 is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize \u2014 INSIDE the window \u2014 the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes \u2014 in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete \u2192 sweep-entry window of a synchronous full \u2014 where PASS1_MARKED is populated and consumed within one `run_to_completion` \u2014 is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize \u2014 INSIDE the window \u2014 the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes \u2014 in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 for RFC deferred collection S5: `cycle.rs` gains guards that refuse a BUDGETED cycle's root scan and final remark at an allocation point (`frame_root_phase_refused` is false for every synchronous cycle, so `run_to_completion` is untouched); `mod.rs` adds a D2 assertion at the synchronous chokepoints that panics or returns before any phase runs, exit diagnostics, and module declarations; `policy.rs` changes trigger evaluation and poll routing, all outside a running cycle. None alters mark/sweep control flow or runs anything inside the mark-complete to sweep-entry window.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -402,9 +402,9 @@ }, "sources": { "crates/perry-runtime/src/gc/census.rs": "25601f25ac70aa998f8cb5c1939d11e7c43a96235d4edf39a78e261b68709471", - "crates/perry-runtime/src/gc/cycle.rs": "4744196ba5e9c5ac40912154cf5b45b4a618d81ddc776ab1095fbc585f27c878", - "crates/perry-runtime/src/gc/mod.rs": "bb38a949495846fd30598064bb04d8e8d384721c25a9a21d1029da1e0c5caa3c", - "crates/perry-runtime/src/gc/policy.rs": "7e2304ec822d26877df4eaf7aefb55c485c66425b8946b56a9b271a0abcc635b", + "crates/perry-runtime/src/gc/cycle.rs": "2f6f6bcaf7efa1c5b9d71e88ae258108cb2467ec08d802492d4a29a0785c533a", + "crates/perry-runtime/src/gc/mod.rs": "94a4c38a96cef565544f6f17971fd32c57763a98bc447826d14ec7df4f327ea8", + "crates/perry-runtime/src/gc/policy.rs": "35097e8c66f4581c969ab551d37e5996640ac2d94c2b4af86ce83b547871ed8a", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } } @@ -2772,6 +2772,24 @@ "name": "NOT_ANON_MEMO", "verdict": "not_a_gc_pointer", "why": "#10495: per-thread direct-mapped memo of u32 CLASS IDS proved not to be an anonymous literal shape's (is_anon_shape_class_id == false). Class ids are integers from the class registry, never heap addresses." + }, + { + "file": "crates/perry-runtime/src/gc/policy.rs", + "name": "GC_POLL_OWED_REQUEST", + "verdict": "not_a_gc_pointer", + "why": "A DeferredGcRequest enum (None/CheckTrigger/DirectMinor/Collect(kind)) naming which collection a root-lock exit owes the next poll (RFC deferred collection S5). A request kind, never an address." + }, + { + "file": "crates/perry-runtime/src/gc/alloc_point.rs", + "name": "TEST_SLACK", + "verdict": "not_a_gc_pointer", + "why": "Test-only override of the parked-cycle valve slack: a byte count, never an address." + }, + { + "file": "crates/perry-runtime/src/gc/alloc_point.rs", + "name": "PARKED_AT", + "verdict": "not_a_gc_pointer", + "why": "Arena byte total at which an allocation point parked a budgeted cycle at a frame-root phase (RFC deferred collection S5 parked-cycle valve): a byte count, never an address." } ], "_FRONTIER_README": "Identity-pinned debt ratchet over new perry-ui* candidates and otherwise-unclassified core raw/Perry TLS declarations (see the census docstring, \u201cThe identity-pinned frontier\u201d). A new uncovered holder fails until it is scanned, receives a researched holders verdict, or is deliberately pinned as debt. Moving a researched false positive to holders graduates it from this list. A fixed or classified holder makes its old frontier pin stale, so the receipt must be deleted.", diff --git a/scripts/thread_exit_address_globals.json b/scripts/thread_exit_address_globals.json index 9170d11a7b..73c8367419 100644 --- a/scripts/thread_exit_address_globals.json +++ b/scripts/thread_exit_address_globals.json @@ -4091,6 +4091,14 @@ ], "verdict": "no_heap_address", "why": "Holds (Handle, Parked) rows: a Handle is a native-registry id in [1, 0x40000) minted by the process-global shared pool (perry_ffi::shared_handle_id_pool), never an arena address, and Parked is {kind, NativeRegistrationIdentity{domain, serial, numeric_id}, epoch}. Payloads live in the process-global common HANDLES map, not in any thread's arena. A row cannot name a different object after reuse: release() retires only via begin_retirement_of(identity), which rejects any registration whose serial differs, and ids are reissued only after a full trace proves them unreferenced (#11453)." + }, + { + "file": "crates/perry-runtime/src/gc/scan_fallback.rs", + "names": [ + "SCAN_FALLBACKS_ALL_THREADS" + ], + "verdict": "no_heap_address", + "why": "Per-site counts of conservative-scan fallbacks summed over all threads (RFC deferred collection S5 valve ledger); an array of plain event counters, never an address." } ] } From 3a4eb34218640050da56114ed6136cec05c8273d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 12:58:03 +0200 Subject: [PATCH 10/15] perf: keep arena_cell_alloc inlinable (unsafe-zone note on the block-reserve path) --- crates/perry-runtime/src/arena/block.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/perry-runtime/src/arena/block.rs b/crates/perry-runtime/src/arena/block.rs index c9c83fadaf..20f279d183 100644 --- a/crates/perry-runtime/src/arena/block.rs +++ b/crates/perry-runtime/src/arena/block.rs @@ -380,7 +380,14 @@ pub(crate) fn new_object_start_bitmap(size: usize) -> Box<[u64]> { /// violation [`arena_cell_alloc`] exists to avoid, on the out-of-memory path. /// `arena_cell_alloc` is the only caller; every `&mut self` path uses /// [`alloc_block_no_gc`]. +#[inline(never)] pub(crate) fn reserve_arena_block(min_size: usize) -> ArenaBlock { + // Decision 10 of RFC deferred collection: growth an unsafe zone forced (no + // poll or valve can collect inside one). Diagnostic, and here rather than + // in `arena_cell_alloc`: that function is inlined into every allocation, + // and a call added there stops it being inlined (measured +3.6% retired + // instructions on bench_string_heavy). + crate::gc::note_block_if_unsafe_zone(block_size_for(min_size)); if let Some(block) = try_alloc_block(min_size, true) { return block; } @@ -961,10 +968,6 @@ pub(crate) unsafe fn arena_cell_alloc(arena: *mut Arena, size: usize, align: usi // on the first cut of #7022, where the reservation still happened inside // `alloc_fresh_block` under the borrow.) let fresh = reserve_arena_block(size); - // Decision 10 of RFC deferred collection: growth an unsafe zone forced - // (no poll or valve can collect inside one). Diagnostic; one relaxed load - // on the block-acquire path only. - crate::gc::note_block_if_unsafe_zone(block_size_for(size)); let _borrow = ArenaBorrowGuard::new(); (*arena).install_reserved_block(fresh); From 259fdb3160183cf7e2c36345012a5e48a948bef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 14:40:15 +0200 Subject: [PATCH 11/15] style: cargo fmt --- crates/perry-codegen/src/gc_map.rs | 17 ++++++---- crates/perry-runtime/src/gc/alloc_point.rs | 5 +-- crates/perry-runtime/src/gc/cycle.rs | 7 ++-- crates/perry-runtime/src/gc/roots.rs | 11 +++---- .../src/gc/roots/stack_maps_frame_verify.rs | 15 ++++++--- crates/perry-runtime/src/gc/scan_fallback.rs | 6 ++-- .../src/gc/tests/alloc_point_invariant.rs | 24 +++++++++++--- crates/perry-runtime/src/gc/tests/roots.rs | 33 +++++++++++++++---- 8 files changed, 82 insertions(+), 36 deletions(-) diff --git a/crates/perry-codegen/src/gc_map.rs b/crates/perry-codegen/src/gc_map.rs index f1de294a39..a53e475241 100644 --- a/crates/perry-codegen/src/gc_map.rs +++ b/crates/perry-codegen/src/gc_map.rs @@ -1233,9 +1233,8 @@ pub(crate) fn list_unrecorded_functions() -> bool { use std::sync::OnceLock; static ON: OnceLock = OnceLock::new(); *ON.get_or_init(|| { - let set = |value: Result| { - value.is_ok_and(|v| !v.trim().is_empty()) - }; + let set = + |value: Result| value.is_ok_and(|v| !v.trim().is_empty()); std::env::var("PERRY_GC_INSTRUMENTS") .is_ok_and(|v| matches!(v.trim(), "1" | "true" | "on" | "yes")) || set(std::env::var("PERRY_GC_VERIFY_FRAMES")) @@ -1249,7 +1248,9 @@ pub(crate) fn note_statepoint_functions(functions: &[&crate::function::LlFunctio if !list_unrecorded_functions() { return; } - let mut guard = STATEPOINT_FUNCTIONS.lock().unwrap_or_else(|p| p.into_inner()); + let mut guard = STATEPOINT_FUNCTIONS + .lock() + .unwrap_or_else(|p| p.into_inner()); let set = guard.get_or_insert_with(Default::default); for function in functions { // The latest render is authoritative: the RS4GC budget retry can move a @@ -1274,7 +1275,9 @@ fn append_unrecorded_functions( if !list_unrecorded_functions() { return; } - let guard = STATEPOINT_FUNCTIONS.lock().unwrap_or_else(|p| p.into_inner()); + let guard = STATEPOINT_FUNCTIONS + .lock() + .unwrap_or_else(|p| p.into_inner()); let Some(names) = guard.as_ref() else { return; }; @@ -1292,7 +1295,9 @@ fn append_unrecorded_functions( let Some((label, rest)) = line.split_once(':') else { continue; }; - if !(rest.is_empty() || rest.starts_with([' ', '\t'])) || label.starts_with(['\t', ' ', '.', '"']) { + if !(rest.is_empty() || rest.starts_with([' ', '\t'])) + || label.starts_with(['\t', ' ', '.', '"']) + { continue; } let Some(name) = label.strip_prefix(prefix) else { diff --git a/crates/perry-runtime/src/gc/alloc_point.rs b/crates/perry-runtime/src/gc/alloc_point.rs index ed9f860adf..ebfb54df81 100644 --- a/crates/perry-runtime/src/gc/alloc_point.rs +++ b/crates/perry-runtime/src/gc/alloc_point.rs @@ -321,8 +321,9 @@ pub(super) fn write_valve_ledger_line() { return; } let c = alloc_point_counters(); - let valve = - super::scan_fallback::scan_fallback_count_any_thread(super::ConservativeScanSite::NurseryChurnSlackValve); + let valve = super::scan_fallback::scan_fallback_count_any_thread( + super::ConservativeScanSite::NurseryChurnSlackValve, + ); let exe = std::env::current_exe() .ok() .and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())) diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 31f6ded372..6387b037f7 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -739,9 +739,10 @@ impl GcCycleState { pub(super) fn next_step_reads_frame_roots(&self) -> bool { match self.phase { GcCyclePhase::RootScan => true, - GcCyclePhase::AtomicFinalize => self.atomic_finalize.as_ref().is_some_and(|state| { - state.subphase == AtomicFinalizeSubphase::FinalRootRemark - }), + GcCyclePhase::AtomicFinalize => self + .atomic_finalize + .as_ref() + .is_some_and(|state| state.subphase == AtomicFinalizeSubphase::FinalRootRemark), _ => false, } } diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index ac42b0b0de..7a6cf084b5 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -11,9 +11,9 @@ mod stack_roots; pub(crate) use stack_maps::census_rows::stack_map_index_census; mod temp_roots; pub(super) use stack_maps::ensure_built as ensure_stack_maps_built; -pub(super) use stack_maps::initialize as initialize_stack_maps; pub(super) use stack_maps::frame_verify::active as stack_maps_frame_verify_active; pub(super) use stack_maps::frame_verify::counters as frame_verify_counters; +pub(super) use stack_maps::initialize as initialize_stack_maps; pub(super) use stack_maps::publish_rewrite_walk_stats as stack_maps_publish_rewrite_walk_stats; pub(super) use stack_maps::record_native_stack_walk_source; pub(super) use stack_maps::verify_native_slots_post_walk as stack_maps_native_slot_verify; @@ -47,11 +47,10 @@ pub(crate) use stack_roots::with_stack_roots; #[allow(unused_imports)] pub(crate) use scan_mode::{ conservative_scan_requested, conservative_stack_scan_decision, - conservative_stack_scan_decision_for, - conservative_stack_scan_mode, conservative_stack_scan_mode_from_value, - resolve_conservative_stack_scan_mode, set_conservative_stack_scan_override, - ConservativeStackScanDecision, ConservativeStackScanMode, ManualGcScanGuard, - CONSERVATIVE_STACK_SCAN_OVERRIDE, + conservative_stack_scan_decision_for, conservative_stack_scan_mode, + conservative_stack_scan_mode_from_value, resolve_conservative_stack_scan_mode, + set_conservative_stack_scan_override, ConservativeStackScanDecision, ConservativeStackScanMode, + ManualGcScanGuard, CONSERVATIVE_STACK_SCAN_OVERRIDE, }; pub(crate) use shadow_stack::shadow_stack_has_active_frame; pub(crate) use shadow_stack::SHADOW; diff --git a/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs b/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs index 3acd9c2326..97d1912742 100644 --- a/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs +++ b/crates/perry-runtime/src/gc/roots/stack_maps_frame_verify.rs @@ -142,7 +142,10 @@ impl StackMapIndex { self.functions .binary_search_by_key(&function_start, |entry| entry.address) .is_ok() - || self.unrecorded_functions.binary_search(&function_start).is_ok() + || self + .unrecorded_functions + .binary_search(&function_start) + .is_ok() } } @@ -163,8 +166,9 @@ mod tests { // A runtime frame: ignored. unmatched_frame(&index, 0x3010, 0x3000); - let prev = - crate::gc::set_conservative_stack_scan_override(Some(crate::gc::ConservativeStackScanMode::Disabled)); + let prev = crate::gc::set_conservative_stack_scan_override(Some( + crate::gc::ConservativeStackScanMode::Disabled, + )); let result = std::panic::catch_unwind(|| unmatched_frame(&index, 0x2010, 0x2000)); crate::gc::set_conservative_stack_scan_override(prev); let message = result.expect_err("an unmapped generated frame must panic"); @@ -181,8 +185,9 @@ mod tests { fn a_conservative_collection_tolerates_unmapped_generated_frames() { let mut index = StackMapIndex::default(); index.unrecorded_functions = vec![0x4000]; - let prev = - crate::gc::set_conservative_stack_scan_override(Some(crate::gc::ConservativeStackScanMode::Full)); + let prev = crate::gc::set_conservative_stack_scan_override(Some( + crate::gc::ConservativeStackScanMode::Full, + )); unmatched_frame(&index, 0x4010, 0x4000); crate::gc::set_conservative_stack_scan_override(prev); } diff --git a/crates/perry-runtime/src/gc/scan_fallback.rs b/crates/perry-runtime/src/gc/scan_fallback.rs index eaeb41e71a..46b761dbf4 100644 --- a/crates/perry-runtime/src/gc/scan_fallback.rs +++ b/crates/perry-runtime/src/gc/scan_fallback.rs @@ -130,9 +130,9 @@ impl ConservativeScanSite { /// collections a program pays for without asking for them. pub(crate) const fn is_automatic(self) -> bool { match self { - Self::OldReclaimAllocPoint - | Self::NurseryChurnSlackValve - | Self::EmergencyReclaim => true, + Self::OldReclaimAllocPoint | Self::NurseryChurnSlackValve | Self::EmergencyReclaim => { + true + } Self::ManualMinor => false, } } diff --git a/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs b/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs index fa8fd9bd33..28323dae2a 100644 --- a/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs +++ b/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs @@ -95,7 +95,10 @@ fn assists_park_at_both_root_phases_and_the_poll_serves_them() { } assert_eq!(cycle_phase(), Some(GcCyclePhase::RootScan.ffi_code())); - assert!(gc_safepoint_moving_minor(), "the poll must handle the parked phase"); + assert!( + gc_safepoint_moving_minor(), + "the poll must handle the parked phase" + ); let served = alloc_point::alloc_point_counters().root_phases_served_at_poll; assert_eq!(served, 1, "the poll served exactly the parked root scan"); assert_ne!(cycle_phase(), Some(GcCyclePhase::RootScan.ffi_code())); @@ -123,7 +126,10 @@ fn assists_park_at_both_root_phases_and_the_poll_serves_them() { break; } } - assert!(!gc_budgeted_cycle_active(), "the heap-only tail completes from assists"); + assert!( + !gc_budgeted_cycle_active(), + "the heap-only tail completes from assists" + ); assert!(gc_collection_count() > before); assert_eq!(alloc_point::alloc_point_counters().parked_valve_fires, 0); let live_after = (js_shadow_slot_get(0) & POINTER_MASK) as *const crate::StringHeader; @@ -204,7 +210,10 @@ fn parked_cycle_valve_fires_after_the_slack_and_is_counted() { alloc_point::reset_alloc_point_counters(); set_safepoint_pending(false); let _trigger = start_assist_cycle(b"d2_parked_valve_live"); - assert_eq!(assist_until_parked(10_000), GcCyclePhase::RootScan.ffi_code()); + assert_eq!( + assist_until_parked(10_000), + GcCyclePhase::RootScan.ffi_code() + ); // Pretend the program allocated the whole slack since parking. alloc_point::test_make_parked_valve_due(); @@ -233,12 +242,17 @@ fn block_growth_inside_an_unsafe_zone_is_counted() { let before = alloc_point::alloc_point_counters(); assert_eq!(before.unsafe_zone_growth_events, 0); let previous = - super::super::policy::unsafe_zone_test_override::set_unsafe_zone_blocked_for_test(Some(true)); + super::super::policy::unsafe_zone_test_override::set_unsafe_zone_blocked_for_test(Some( + true, + )); crate::gc::note_block_if_unsafe_zone(1 << 20); super::super::policy::unsafe_zone_test_override::set_unsafe_zone_blocked_for_test(Some(false)); crate::gc::note_block_if_unsafe_zone(1 << 20); super::super::policy::unsafe_zone_test_override::set_unsafe_zone_blocked_for_test(previous); let after = alloc_point::alloc_point_counters(); - assert_eq!(after.unsafe_zone_growth_events, 1, "only the in-zone block counts"); + assert_eq!( + after.unsafe_zone_growth_events, 1, + "only the in-zone block counts" + ); assert_eq!(after.unsafe_zone_growth_bytes, 1 << 20); } diff --git a/crates/perry-runtime/src/gc/tests/roots.rs b/crates/perry-runtime/src/gc/tests/roots.rs index a006d9b085..96763b27e8 100644 --- a/crates/perry-runtime/src/gc/tests/roots.rs +++ b/crates/perry-runtime/src/gc/tests/roots.rs @@ -197,8 +197,15 @@ fn lock_safe_runtime_scanners_tui_hooks_defers_direct_minor_gc() { // RFC deferred collection S5 ("D stops collecting"): the lock exit hands // the owed collection to the next declared poll instead of running it. - assert_eq!(gc_collection_count(), before, "the lock exit must not collect (S5)"); - assert!(gc_safepoint_moving_minor(), "the poll must serve the owed collection"); + assert_eq!( + gc_collection_count(), + before, + "the lock exit must not collect (S5)" + ); + assert!( + gc_safepoint_moving_minor(), + "the poll must serve the owed collection" + ); assert!( gc_collection_count() > before, "deferred direct minor GC should run after the hook root lock is released" @@ -237,8 +244,15 @@ fn lock_safe_runtime_scanners_tui_state_defers_manual_gc() { // RFC deferred collection S5 ("D stops collecting"): the lock exit hands // the owed collection to the next declared poll instead of running it. - assert_eq!(gc_collection_count(), before, "the lock exit must not collect (S5)"); - assert!(gc_safepoint_moving_minor(), "the poll must serve the owed collection"); + assert_eq!( + gc_collection_count(), + before, + "the lock exit must not collect (S5)" + ); + assert!( + gc_safepoint_moving_minor(), + "the poll must serve the owed collection" + ); assert!( gc_collection_count() > before, "deferred manual GC should run after the state root lock is released" @@ -407,8 +421,15 @@ fn lock_safe_runtime_scanners_tui_hooks_defers_direct_full_gc() { // RFC deferred collection S5 ("D stops collecting"): the lock exit hands // the owed collection to the next declared poll instead of running it. - assert_eq!(gc_collection_count(), before, "the lock exit must not collect (S5)"); - assert!(gc_safepoint_moving_minor(), "the poll must serve the owed collection"); + assert_eq!( + gc_collection_count(), + before, + "the lock exit must not collect (S5)" + ); + assert!( + gc_safepoint_moving_minor(), + "the poll must serve the owed collection" + ); assert!( gc_collection_count() > before, "deferred direct full GC should run after the hook root lock is released" From bbc27bc9ff101da2a42730f0787add415df29cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 14:45:04 +0200 Subject: [PATCH 12/15] changelog: #11630 allocation-point invariant --- changelog.d/11630-gc-alloc-point-invariant.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 changelog.d/11630-gc-alloc-point-invariant.md diff --git a/changelog.d/11630-gc-alloc-point-invariant.md b/changelog.d/11630-gc-alloc-point-invariant.md new file mode 100644 index 0000000000..715cb2f8aa --- /dev/null +++ b/changelog.d/11630-gc-alloc-point-invariant.md @@ -0,0 +1,14 @@ +- **GC: an allocation never begins a precise or moving collection phase (RFC deferred collection, step S5, runtime half).** The allocation point is the dynamic extent of `gc_check_trigger`, which every allocation slow path, the JSON mid-parse checks and the root-lock flush of a trigger check funnel into. Inside it the collector may now only take a block, arm the poll, run heap-only budgeted work, or run one of the conservative non-moving arms (the nursery slack valve, the old-gen reclaim arm, which decision 1 keeps at the allocation point, and the emergency reclaim). Each allocation-point entry was routed: + - **A-assist** (`policy.rs` `gc_budgeted_start_or_step`, `cycle.rs` `step_root_scan` / the `FinalRootRemark` subphase): a budgeted cycle whose next step reads frame roots is parked, the poll is armed, and the next declared poll serves the phase (`gc_safepoint_moving_minor`). Heap-only phases still advance from assists. A parked cycle has its own valve: after the nursery slack (64 MiB, budget-scaled) with no poll, the phase is served at the allocation point and counted (`parked_valve_fires`). That valve is sound only while allocating calls are statepoints; it must be proven unreachable or replaced before S6. + - **D** (`flush_deferred_gc_request`): a root-lock exit no longer runs a deferred minor, full or manual `gc()`. It hands it to the next poll (option 2 of #11523). A deferred trigger check still runs, as an allocation-point evaluation. + - **A-old, A-valve, the polls-off direct minor, A-emerg** keep their forced conservative scan. A synchronous collection begun at an allocation point without requesting that scan now panics in every build (`gc/alloc_point.rs`, `assert_d2_synchronous_collection`). The check reads the request, so the unit-test isolation guards and `PERRY_CONSERVATIVE_STACK_SCAN=off` do not trip it. +- **`PERRY_GC_SAFEPOINT_ONLY`'s runtime contract is deleted.** Its heal and strict arms are gone, with `ConservativeScanSite::SafepointContractHeal`, because D2 is now the default. Codegen still reads the variable as its research switch for `AllocNoReentry` leaves; S6 replaces that. +- **Unmapped-frame verifier** (`gc/roots/stack_maps_frame_verify.rs`). A precise collection that walks a generated statepoint function suspended at a call with no stack-map record now panics instead of skipping the frame. That is the #11522 shape: a `gc-leaf-function` call whose callee collected. It is armed by `PERRY_GC_VERIFY_FRAMES=1` (a new GC instrument), by `PERRY_GC_SCHEDULE_SEED`, and in `debug_assertions` builds. + - Function membership comes from the unwinder's region start, or `_Unwind_Find_FDE` on the x29-chain walker. + - An instrumented compile (`PERRY_GC_INSTRUMENTS=1`, `PERRY_GC_VERIFY_FRAMES` or a seed at compile time) also lists zero-record statepoint functions in the GC map. v6 runtimes already skip zero-record entries, so the format is unchanged. + - It is off in release: every unmatched frame costs an unwind-table lookup, and the listing costs map bytes. +- **Valve gate (decision 5).** With `PERRY_GC_VALVE_LEDGER=`, every process appends one line at exit recording its valve firings. `scripts/gc_valve_ledger_check.py` fails on any firing, and on fewer lines than passing tests, which proves the check ran. The pr-tier gap-suite shards and a new gc-stress step over the 14 ratchet probes (`scripts/gc_valve_ratchet_probes.sh`) run it. `OldReclaimAllocPoint` is reported, not gated. +- **Diagnostics.** `PERRY_GC_DIAG=1` prints `[gc-alloc-point]` and `[gc-verify-frames]`: + - valve firings, parked and served root phases, and owed requests; + - arena growth inside `GC_UNSAFE_ZONES` (decision 10, diagnostic only); + - `max_poll_wait_bytes`, the most the arena grew between arming a collection and reaching a poll (decision 3's measurement). From 7553c707375dbf8c38a90d6994e137d6518d1c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 15:04:43 +0200 Subject: [PATCH 13/15] gc: allocation-point and scan-request thread-locals use perry_thread_local! --- crates/perry-runtime/src/gc/alloc_point.rs | 4 ++-- crates/perry-runtime/src/gc/roots/scan_mode.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/perry-runtime/src/gc/alloc_point.rs b/crates/perry-runtime/src/gc/alloc_point.rs index ebfb54df81..32376a85a3 100644 --- a/crates/perry-runtime/src/gc/alloc_point.rs +++ b/crates/perry-runtime/src/gc/alloc_point.rs @@ -39,7 +39,7 @@ use std::cell::Cell; use std::io::Write; use std::sync::atomic::{AtomicU64, Ordering}; -thread_local! { +crate::perry_thread_local! { /// Depth of `gc_check_trigger` evaluations on this thread. Nesting is /// possible (a root-lock flush inside an evaluation), so a counter rather /// than a flag. @@ -352,7 +352,7 @@ pub(super) fn write_valve_ledger_line() { } #[cfg(test)] -thread_local! { +crate::perry_thread_local! { static TEST_SLACK: Cell> = const { Cell::new(None) }; } diff --git a/crates/perry-runtime/src/gc/roots/scan_mode.rs b/crates/perry-runtime/src/gc/roots/scan_mode.rs index f89cd484ec..a20cd85859 100644 --- a/crates/perry-runtime/src/gc/roots/scan_mode.rs +++ b/crates/perry-runtime/src/gc/roots/scan_mode.rs @@ -104,7 +104,7 @@ pub(crate) struct ManualGcScanGuard { engaged: bool, } -thread_local! { +crate::perry_thread_local! { /// How many `ManualGcScanGuard`s are live on this thread, whether or not /// each managed to pin the override. The allocation-point invariant /// (`gc/alloc_point.rs`) checks this REQUEST, which no override can hide. From 6638c71b0f4aec267ddef1b0dcba39059950e38b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 16:42:29 +0200 Subject: [PATCH 14/15] gc: a poll serves a parked root scan one host-sized slice at a time --- crates/perry-runtime/src/gc/policy.rs | 31 ++++++++++++++++--- .../src/gc/tests/alloc_point_invariant.rs | 20 +++++++++--- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index a1e4ac13b3..3a8469426c 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -3814,11 +3814,17 @@ pub(crate) fn gc_safepoint_moving_minor() -> bool { && !gc_budgeted_resume_blocked() { let _declared = DeclaredSafepointGuard::enter(); - serve_budgeted_root_phase(); - super::alloc_point::note_root_phase_served_at_poll(); + // One host-sized slice per poll, exactly as a host step slices the root + // scan; the poll stays armed until the phase is done, so the next poll + // continues it. Serving the whole scan in one step made the worst + // budgeted pause ~20% longer on the server fixture. + let done = serve_budgeted_root_phase_slice(); + if done { + super::alloc_point::note_root_phase_served_at_poll(); + } // An owed collection the in-alloc guard above held back (a budgeted // minor holds `GC_FLAG_IN_ALLOC`) keeps the poll armed for later. - if !owed_request_pending() { + if done && !owed_request_pending() { note_pending_poll_wait(); set_safepoint_pending(false); } @@ -4891,12 +4897,27 @@ fn budgeted_cycle_next_step_reads_frame_roots() -> bool { }) } -/// Advance the active budgeted cycle through its frame-root phase. Called at a -/// declared poll, or by the counted parked-cycle valve. Unbounded work for the +/// Advance the active budgeted cycle through its frame-root phase in one go. +/// Only the counted parked-cycle valve uses this; a declared poll serves one +/// slice at a time (`serve_budgeted_root_phase_slice`). Unbounded work for the /// phase itself: `RootScan` is bounded by the root set, and `FinalRootRemark` /// is atomic by design (`gc-step-bounds.md`). Stops as soon as the next step /// no longer reads frame roots, so the heap-only work that follows stays with /// the assists and host steps. +/// One normal-incremental slice of the active cycle's frame-root phase, at a +/// declared poll. Returns whether the cycle has left its frame-root phase. The +/// final remark is atomic by design and completes in its slice. +fn serve_budgeted_root_phase_slice() -> bool { + if gc_budgeted_cycle_active() && budgeted_cycle_next_step_reads_frame_roots() { + let _ = gc_budgeted_step_work_units_inner(GC_NORMAL_INCREMENTAL_WORK_UNITS); + } + let done = !gc_budgeted_cycle_active() || !budgeted_cycle_next_step_reads_frame_roots(); + if done { + super::alloc_point::clear_park(); + } + done +} + fn serve_budgeted_root_phase() { // A handful of steps: the build of the valid-pointer set may precede the // root scan, and the barrier-seed drain precedes the remark. diff --git a/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs b/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs index 28323dae2a..0fb89a8cc2 100644 --- a/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs +++ b/crates/perry-runtime/src/gc/tests/alloc_point_invariant.rs @@ -95,10 +95,21 @@ fn assists_park_at_both_root_phases_and_the_poll_serves_them() { } assert_eq!(cycle_phase(), Some(GcCyclePhase::RootScan.ffi_code())); - assert!( - gc_safepoint_moving_minor(), - "the poll must handle the parked phase" - ); + // The poll serves the phase one host-sized slice at a time and stays + // armed until it is done. + let mut polls = 0; + while cycle_phase() == Some(GcCyclePhase::RootScan.ffi_code()) { + assert!( + GC_SAFEPOINT_PENDING.with(std::cell::Cell::get), + "a partly served root phase must keep the poll armed" + ); + assert!( + gc_safepoint_moving_minor(), + "the poll must handle the parked phase" + ); + polls += 1; + assert!(polls < 10_000, "the poll never finished the root scan"); + } let served = alloc_point::alloc_point_counters().root_phases_served_at_poll; assert_eq!(served, 1, "the poll served exactly the parked root scan"); assert_ne!(cycle_phase(), Some(GcCyclePhase::RootScan.ffi_code())); @@ -112,6 +123,7 @@ fn assists_park_at_both_root_phases_and_the_poll_serves_them() { GcCyclePhase::AtomicFinalize.ffi_code(), "the second frame-root phase is the final remark" ); + // The final remark is atomic: one poll serves it. assert!(gc_safepoint_moving_minor()); assert_eq!( alloc_point::alloc_point_counters().root_phases_served_at_poll, From 3cfe20ff09d4e21870c05011ffbde0b69188e95c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 17:37:09 +0200 Subject: [PATCH 15/15] census pin: re-audit after S5 follow-ups --- scripts/gc_runtime_root_holders.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 4b752aa16d..63563556f2 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -386,7 +386,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete \u2192 sweep-entry window of a synchronous full \u2014 where PASS1_MARKED is populated and consumed within one `run_to_completion` \u2014 is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize \u2014 INSIDE the window \u2014 the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes \u2014 in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 for RFC deferred collection S5: `cycle.rs` gains guards that refuse a BUDGETED cycle's root scan and final remark at an allocation point (`frame_root_phase_refused` is false for every synchronous cycle, so `run_to_completion` is untouched); `mod.rs` adds a D2 assertion at the synchronous chokepoints that panics or returns before any phase runs, exit diagnostics, and module declarations; `policy.rs` changes trigger evaluation and poll routing, all outside a running cycle. None alters mark/sweep control flow or runs anything inside the mark-complete to sweep-entry window.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete \u2192 sweep-entry window of a synchronous full \u2014 where PASS1_MARKED is populated and consumed within one `run_to_completion` \u2014 is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize \u2014 INSIDE the window \u2014 the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes \u2014 in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 for RFC deferred collection S5: `cycle.rs` gains guards that refuse a BUDGETED cycle's root scan and final remark at an allocation point (`frame_root_phase_refused` is false for every synchronous cycle, so `run_to_completion` is untouched); `mod.rs` adds a D2 assertion at the synchronous chokepoints that panics or returns before any phase runs, exit diagnostics, and module declarations; `policy.rs` changes trigger evaluation and poll routing, all outside a running cycle. None alters mark/sweep control flow or runs anything inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 after the S5 follow-ups (formatting; perry_thread_local! for the allocation-point markers; the poll serving a parked budgeted root scan one host slice at a time): all outside a running synchronous cycle, no mark/sweep control-flow change.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -402,9 +402,9 @@ }, "sources": { "crates/perry-runtime/src/gc/census.rs": "25601f25ac70aa998f8cb5c1939d11e7c43a96235d4edf39a78e261b68709471", - "crates/perry-runtime/src/gc/cycle.rs": "2f6f6bcaf7efa1c5b9d71e88ae258108cb2467ec08d802492d4a29a0785c533a", + "crates/perry-runtime/src/gc/cycle.rs": "bed1e2239ad8f2d787ff798f6d55c214e9f2f0e3b181e7aa33d90936daf4a07a", "crates/perry-runtime/src/gc/mod.rs": "94a4c38a96cef565544f6f17971fd32c57763a98bc447826d14ec7df4f327ea8", - "crates/perry-runtime/src/gc/policy.rs": "35097e8c66f4581c969ab551d37e5996640ac2d94c2b4af86ce83b547871ed8a", + "crates/perry-runtime/src/gc/policy.rs": "7312e3603961d574995632c0901c346e4a1d04fde022a52a6839890c0b939d1b", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } }