From 48a30a7f3e471ab7d06d415382eedfa71b23349b Mon Sep 17 00:00:00 2001 From: Perry Bot Date: Tue, 29 Sep 2026 11:11:11 +0000 Subject: [PATCH 1/2] perf(gc): the barrier-arming reconstruct walk skips wholly-nursery blocks --- crates/perry-runtime/src/arena/mod.rs | 7 +- .../perry-runtime/src/arena/page_meta/mod.rs | 17 ++++ crates/perry-runtime/src/arena/walk.rs | 18 ++++ crates/perry-runtime/src/gc/barrier_arming.rs | 13 +++ .../src/gc/tests/barrier_arming.rs | 90 +++++++++++++++++++ crates/perry-runtime/src/gc/verify.rs | 64 ++++++++++++- 6 files changed, 204 insertions(+), 5 deletions(-) diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 896220dae9..e26b675b49 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -163,9 +163,10 @@ pub(crate) use page_meta::{ old_page_account_dirty_slot, old_page_account_dirty_slots, old_page_account_promoted_object, old_page_account_swept_object, old_page_account_swept_tally, old_page_clear_dirty, old_page_mark_dirty, old_page_meta_snapshot, old_page_summary, old_pages_begin_gc_cycle, - old_pages_reset_sweep_accounting, record_arena_object_start, unregister_old_object_pages, - unregister_old_objects_batch, HeapGeneration, HeapSpace, OldArenaPageObjectCursor, - OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, OldPageSweepTally, + old_pages_reset_sweep_accounting, record_arena_object_start, uniform_heap_generation, + unregister_old_object_pages, unregister_old_objects_batch, HeapGeneration, HeapSpace, + OldArenaPageObjectCursor, OldArenaSourceBlockSelection, OldPageMeta, OldPageSummary, + OldPageSweepTally, }; #[cfg(test)] diff --git a/crates/perry-runtime/src/arena/page_meta/mod.rs b/crates/perry-runtime/src/arena/page_meta/mod.rs index 0ceb53c148..e04c6462e5 100644 --- a/crates/perry-runtime/src/arena/page_meta/mod.rs +++ b/crates/perry-runtime/src/arena/page_meta/mod.rs @@ -920,6 +920,23 @@ pub(crate) fn classify_heap_generation(addr: usize) -> HeapGeneration { classify_heap_generation_uncached(addr, key) } +/// The generation EVERY address of `[base, end)` classifies as, when a single +/// registered range covers all of it; `None` when none does (or it is split +/// across registrations). A whole arena block is registered as one range, so +/// this answers "what generation is this block" without classifying each +/// object on it. +pub(crate) fn uniform_heap_generation(base: usize, end: usize) -> Option { + if base == 0 || end <= base { + return None; + } + let key = generation_class_key_for_addr(base); + let range = hot_page_generations() + .borrow() + .get(&key) + .and_then(|slot| slot.find(base))?; + (range.base <= base && range.end >= end).then_some(range.generation) +} + /// Cache-miss arm of [`classify_heap_generation`]: consult the page map and /// re-prime the one-entry cache. #[inline(never)] diff --git a/crates/perry-runtime/src/arena/walk.rs b/crates/perry-runtime/src/arena/walk.rs index 17be1ebddf..4cf650834d 100644 --- a/crates/perry-runtime/src/arena/walk.rs +++ b/crates/perry-runtime/src/arena/walk.rs @@ -296,6 +296,24 @@ impl ArenaObjectCursor { self.skip_blocks = skip; } + /// Never enter a snapshotted block for which `skip(data, end)` holds, + /// where `[data, end)` is the block's allocated extent. Replaces any + /// earlier skip set. Block-index order only; an address-ordered cursor is + /// left as it is. + pub(crate) fn skip_blocks_where(&mut self, mut skip: impl FnMut(usize, usize) -> bool) { + let ArenaObjectCursorBlocks::BlockIndex(blocks) = &self.blocks else { + return; + }; + let Some(max_idx) = blocks.iter().map(|block| block.block_idx).max() else { + return; + }; + let mut set = vec![false; max_idx + 1]; + for block in blocks { + set[block.block_idx] = skip(block.data, block.data + block.offset); + } + self.skip_blocks = set; + } + /// `(global block index, data, offset)` of the block the last yielded /// object came from, as snapshotted when the cursor was built. pub(crate) fn current_block_extent(&self) -> Option<(usize, usize, usize)> { diff --git a/crates/perry-runtime/src/gc/barrier_arming.rs b/crates/perry-runtime/src/gc/barrier_arming.rs index 16dda2f84d..405b2211ea 100644 --- a/crates/perry-runtime/src/gc/barrier_arming.rs +++ b/crates/perry-runtime/src/gc/barrier_arming.rs @@ -83,6 +83,10 @@ pub(super) struct RememberedReconstructCensus { pub(super) recovered_old_pages: u64, /// External (malloc-backed) slot-page entries re-derived from the heap. pub(super) recovered_external_pages: u64, + /// Heap objects the reconstruct walk visited. Wholly-nursery blocks are + /// skipped (`rebuild_minor_old_to_young_remembered_set`), so this is the + /// retained non-nursery population, not the whole heap. + pub(super) objects_walked: u64, } impl RememberedReconstructCensus { @@ -91,6 +95,7 @@ impl RememberedReconstructCensus { reconstructs: 0, recovered_old_pages: 0, recovered_external_pages: 0, + objects_walked: 0, } } } @@ -124,6 +129,14 @@ pub(super) fn remembered_reconstruct_census() -> RememberedReconstructCensus { RECONSTRUCT_CENSUS.with(Cell::get) } +pub(super) fn note_reconstruct_objects_walked(objects: usize) { + RECONSTRUCT_CENSUS.with(|cell| { + let mut census = cell.get(); + census.objects_walked = census.objects_walked.saturating_add(objects as u64); + cell.set(census); + }); +} + /// Arm the barrier and rebuild this thread's remembered set from the heap, if /// that has not happened yet. Called by [`remembered_dirty_snapshot`] — i.e. /// on the read side, before any collector observes the log. diff --git a/crates/perry-runtime/src/gc/tests/barrier_arming.rs b/crates/perry-runtime/src/gc/tests/barrier_arming.rs index 52c948f156..8233abe9f1 100644 --- a/crates/perry-runtime/src/gc/tests/barrier_arming.rs +++ b/crates/perry-runtime/src/gc/tests/barrier_arming.rs @@ -223,3 +223,93 @@ fn test_7187_minor_in_the_unarmed_window_has_complete_old_young_coverage() { reset_remembered_set(); clear_marks(); } + +// ── #11549: the arming walk skips wholly-nursery blocks ──────────────────── +// +// The same triad shape. The positive test proves the skip ENGAGED (the walk +// visited fewer objects than the nursery alone holds) and that the edge is +// still recovered; the sabotage arm skips old blocks too and must lose the +// edge, which proves the positive test's recovery comes from the walk under +// test and not from some other coverage. + +const ARMING_WALK_YOUNG_OBJECTS: usize = 4096; + +/// The born-old fixture plus a young generation far larger than the old one. +fn arming_walk_fixture() -> (usize, Vec) { + let (_old, fields, young) = unsafe { born_old_parent_with_young_child() }; + let mut keep = vec![young]; + for _ in 0..ARMING_WALK_YOUNG_OBJECTS { + keep.push(crate::arena::arena_alloc_gc(40, 8, GC_TYPE_OBJECT) as usize); + } + ( + crate::arena::generation_page_for_addr(fields as usize), + keep, + ) +} + +#[test] +fn test_11549_arming_walk_skips_nursery_blocks_and_recovers_the_edge() { + let _guard = GcTestIsolationGuard::new(); + reset_remembered_set(); + clear_marks(); + let _window = UnarmedWindowGuard::open(); + + let (slot_page, young) = arming_walk_fixture(); + assert!( + young.iter().all(|&addr| matches!( + crate::arena::classify_heap_generation(addr), + crate::arena::HeapGeneration::Nursery + )), + "the fixture's young objects must be nursery-resident for the skip to have a subject" + ); + let valid_ptrs = build_valid_pointer_set(); + let _ = mark_remembered_set_roots(&valid_ptrs); + + let census = remembered_reconstruct_census(); + assert_eq!(census.reconstructs, 1); + assert!( + old_page_dirty_for(slot_page), + "skipping nursery blocks must not lose the born-old parent's edge" + ); + assert!( + census.objects_walked < ARMING_WALK_YOUNG_OBJECTS as u64, + "the walk visited {} objects while the nursery alone holds {} — the \ + nursery-block skip did not engage", + census.objects_walked, + ARMING_WALK_YOUNG_OBJECTS + ); + assert!( + census.objects_walked >= 1, + "the old parent must have been walked" + ); + + reset_remembered_set(); + clear_marks(); +} + +/// Sabotage arm: a skip that also drops OLD blocks must lose the edge. +#[test] +fn test_11549_arming_walk_that_skips_old_blocks_loses_the_edge() { + let _guard = GcTestIsolationGuard::new(); + reset_remembered_set(); + clear_marks(); + let _window = UnarmedWindowGuard::open(); + + let (slot_page, _young) = arming_walk_fixture(); + let _sabotage = crate::gc::verify::arming_walk_sabotage::Guard::arm(); + let valid_ptrs = build_valid_pointer_set(); + let _ = mark_remembered_set_roots(&valid_ptrs); + + let census = remembered_reconstruct_census(); + assert_eq!(census.reconstructs, 1, "the reconstruct must still run"); + assert_eq!(census.objects_walked, 0, "sabotage must skip every block"); + assert!( + !old_page_dirty_for(slot_page), + "with every block skipped the edge must be uncovered — if the page is \ + dirty, something other than the walk covers it and the positive test \ + proves nothing" + ); + + reset_remembered_set(); + clear_marks(); +} diff --git a/crates/perry-runtime/src/gc/verify.rs b/crates/perry-runtime/src/gc/verify.rs index e94b8fec94..f06d46a33a 100644 --- a/crates/perry-runtime/src/gc/verify.rs +++ b/crates/perry-runtime/src/gc/verify.rs @@ -759,9 +759,69 @@ pub(super) fn rebuild_live_old_to_young_remembered_set() -> StickyRememberedSet rebuild_retained_old_to_young_remembered_set(true) } -#[allow(dead_code)] +/// The arming reconstruct's walk (#7187): every retained old parent's +/// old→young edges. +/// +/// It skips every arena block that is wholly NURSERY. The walk keeps a parent +/// only if `barrier_parent_needs_remembering` says so — an Old-generation +/// object, or a malloc object — and an object on a nursery block is neither, +/// so each object there was a classification and a rejection. Skipping the +/// block is therefore exact, not a heuristic. It matters because the walk runs +/// at the first minor of every thread, when the young generation is at its +/// fullest: on binary-trees at n = 3 it visited 131 k young objects to find no +/// parent, about 20 M instructions. pub(super) fn rebuild_minor_old_to_young_remembered_set() -> StickyRememberedSet { - rebuild_retained_old_to_young_remembered_set(false) + let mut state = OldToYoungRememberedRebuildState::new(false); + if let Some(cursor) = state.arena_cursor.as_mut() { + cursor.skip_blocks_where(arming_walk_skips_block); + } + while !state.step(usize::MAX) {} + super::barrier_arming::note_reconstruct_objects_walked(state.objects_scanned()); + state.finish() +} + +/// Can the arming walk skip the block `[data, end)` without looking? Only when +/// one registered range covers it and that range is the nursery. +fn arming_walk_skips_block(data: usize, end: usize) -> bool { + #[cfg(test)] + if arming_walk_sabotage::skipping_every_block() { + return true; + } + matches!( + crate::arena::uniform_heap_generation(data, end), + Some(crate::arena::HeapGeneration::Nursery) + ) +} + +/// Test-only sabotage for [`arming_walk_skips_block`]: skip EVERY block, old +/// ones included, so the witness can prove that the old→young edge it recovers +/// comes from the walk it is testing. +#[cfg(test)] +pub(crate) mod arming_walk_sabotage { + use std::cell::Cell; + + thread_local! { + static SKIP_ALL: Cell = const { Cell::new(false) }; + } + + pub(crate) fn skipping_every_block() -> bool { + SKIP_ALL.with(Cell::get) + } + + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(SKIP_ALL.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + let prior = self.0; + SKIP_ALL.with(|s| s.set(prior)); + } + } } #[inline] From e26749df6e051a5a71520c3e24808a1c25df286c Mon Sep 17 00:00:00 2001 From: Perry Bot Date: Tue, 29 Sep 2026 14:15:16 +0000 Subject: [PATCH 2/2] changelog: #11668 the first collection's barrier-arming walk skips wholly-nursery blocks --- changelog.d/11668-gc-arming-walk-skips-nursery.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/11668-gc-arming-walk-skips-nursery.md diff --git a/changelog.d/11668-gc-arming-walk-skips-nursery.md b/changelog.d/11668-gc-arming-walk-skips-nursery.md new file mode 100644 index 0000000000..33d965d5a2 --- /dev/null +++ b/changelog.d/11668-gc-arming-walk-skips-nursery.md @@ -0,0 +1 @@ +- **gc: the first collection's barrier-arming walk skips the young generation (Part of #11549).** When a thread first collects, the lazily-armed write barrier (#7187) rebuilds its old→young remembered set by walking every arena object, keeping only old or malloc parents. It now skips arena blocks that are wholly nursery, whose objects it could only classify and reject. The skip is exact: a block is skipped only when one registered generation range covers it. Against main: gc_ratchet 01 −17.6% instructions, 02 −10.7%, binary-trees n=20 −12.5% and n=10→40 −12.9%, jsonwebtoken −3.0%, 12_large_live_set −0.9%. No row gets slower, and peak RSS is unchanged.