perf(gc): cut a copying minor's fixed cost (intern young log, skip empty array-tail tables, young-only prunes)

changelog.d/11634-gc-minor-fixed-cost.md

@@ -0,0 +1 @@
+perf(gc): a copying minor's fixed cost drops from ~753k to ~98k instructions on an allocation-bound loop (Part of #11549). The intern table now carries a young-entry log, so a minor visits only slots naming young strings instead of all 8192 twice. The two array-tail transition tables are skipped while no entry has ever been published on the thread. Built-in closure metadata gets a young-only dead-owner prune. A minor skips the small-int / ASCII-char string caches, which only hold longlived pinned strings. Each skip is backed by a debug/test completeness check and a sabotage test. With this, a 4 MB nursery costs 466.0 instructions/iteration on the alloc loop against main's 469.2 at 16 MB, where it used to cost +3.0%. The nursery default is unchanged: qs/parse_nested and qs/stringify_nested still regress +2.5% / +7.2% at 4 MB from survivor copying, which is not fixed cost.

crates/perry-runtime/src/gc/dead_owner.rs

@@ -475,7 +475,7 @@ pub(super) const DEAD_KEY_PRUNES: &[DeadKeyPrune] = &[
         table: "BUILTIN_CLOSURE_LENGTH + BUILTIN_CLOSURE_NON_CONSTRUCTABLE",
         owner: DeadKeyOwner::Closure,
         prune: crate::object::prune_dead_builtin_closure_metadata_owners,
-        young_prune: None,
+        young_prune: Some(crate::object::prune_dead_builtin_closure_metadata_owners_young),
     },
     // #8040: `FUNCTION_CLASS_IDS` is keyed by a synthetic-class function
     // value's closure address, and is REKEYED (not re-derived) when that

crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs

@@ -0,0 +1,180 @@
+//! #11549 direction 2: the fixed per-minor cost of two tables that a copying
+//! minor used to walk whole on every pass.
+//!
+//! * The intern table (8192 slots) now carries a young-entry log
+//!   (`gc/young_log.rs`): a minor visits only slots that may name a young
+//!   string.
+//! * The two array-tail transition tables (2 x 8192 slots) are skipped
+//!   outright while no entry has ever been published on the thread.
+//!
+//! Each gets the young-log proof shape: the young entry still MOVES through
+//! the narrowed walk, the walk really was narrowed (a skip needs a counter),
+//! and a writer that forgets to arm is caught (sabotage).
+
+use super::super::*;
+use super::support::*;
+
+const INTERN_LOG: &str = "string.intern_table";
+
+/// Leaves both tables empty however the test exits, including by the
+/// expected panic of a sabotage test.
+struct ClearTablesOnDrop;
+
+impl Drop for ClearTablesOnDrop {
+    fn drop(&mut self) {
+        crate::string::test_clear_intern_table();
+        crate::object::array_tail_transition::test_clear();
+    }
+}
+
+fn fnv(bytes: &[u8]) -> u64 {
+    let mut hash = 0xcbf2_9ce4_8422_2325u64;
+    for &b in bytes {
+        hash ^= b as u64;
+        hash = hash.wrapping_mul(0x0100_0000_01b3);
+    }
+    hash
+}
+
+/// A young string interned through the production writer and reachable ONLY
+/// through the intern table is evacuated by a copying minor, and the table
+/// slot is rewritten to the new address — through the log, not a whole-table
+/// walk.
+#[test]
+fn young_interned_string_is_rewritten_through_the_log() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _clear = ClearTablesOnDrop;
+    gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut);
+    crate::string::test_clear_intern_table();
+
+    let bytes = b"minor-fixed-cost-young-intern";
+    let hash = fnv(bytes);
+    let young = crate::string::js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32);
+    assert!(crate::arena::pointer_in_nursery(young as usize));
+    assert_eq!(crate::string::js_string_intern(young, hash), young);
+
+    let _ = gc_collect_minor();
+
+    let after = crate::string::test_intern_slot_ptr(hash);
+    assert_ne!(after, 0, "the interned string must still be tabled");
+    assert_ne!(
+        after, young as usize,
+        "the slot must name the evacuated copy, not from-space"
+    );
+    unsafe {
+        assert_string_bytes(after as *const crate::StringHeader, bytes);
+    }
+    let row = young_log::last_walk(INTERN_LOG).expect("intern walk recorded");
+    assert!(
+        row.partial,
+        "a copying minor must take the logged walk: {row:?}"
+    );
+    assert!(row.visited >= 1, "the young slot must be visited: {row:?}");
+    assert!(
+        row.visited < row.table_len,
+        "an 8192-slot table must not be walked whole: {row:?}"
+    );
+}
+
+/// An OLD interned string notes nothing, so a minor visits no intern slot at
+/// all even though the table is not empty — the skip fired.
+#[test]
+fn old_interned_string_is_not_visited_by_a_minor() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _clear = ClearTablesOnDrop;
+    gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut);
+    crate::string::test_clear_intern_table();
+
+    let bytes = b"minor-fixed-cost-old-intern";
+    let hash = fnv(bytes);
+    let old = crate::arena::arena_alloc_gc_old(64, 8, GC_TYPE_STRING) as *mut crate::StringHeader;
+    unsafe {
+        crate::string::test_init_string_bytes(old, bytes);
+    }
+    assert!(!young_log::addr_is_minor_relevant(old as usize));
+    assert_eq!(crate::string::js_string_intern(old, hash), old);
+
+    let _ = gc_collect_minor();
+
+    assert_eq!(crate::string::test_intern_slot_ptr(hash), old as usize);
+    let row = young_log::last_walk(INTERN_LOG).expect("intern walk recorded");
+    assert!(row.partial, "{row:?}");
+    assert_eq!(
+        row.visited, 0,
+        "an old interned string must not be visited: {row:?}"
+    );
+}
+
+/// SABOTAGE (rule 2): a writer that publishes a young string into a slot
+/// without noting it is caught by the log-completeness check the minor-scoped
+/// walk runs first. This is the check that turns "someone added an intern
+/// writer and forgot `arm_intern_young`" into a red test instead of a
+/// from-space pointer left in the table.
+#[test]
+#[should_panic(expected = "young log for string.intern_table does not name")]
+fn intern_writer_that_skips_the_log_is_caught() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _clear = ClearTablesOnDrop;
+    crate::string::test_clear_intern_table();
+    let young = young_leaf();
+    crate::string::test_write_intern_slot_without_logging(7, young);
+    crate::string::test_check_intern_young_logged();
+}
+
+/// SABOTAGE for the array-tail skip: an entry published without arming
+/// `array_tail_occupied` is caught where the skip relies on the flag.
+#[test]
+#[should_panic(expected = "without arming the flag first")]
+fn array_tail_publish_without_arming_is_caught() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _clear = ClearTablesOnDrop;
+    crate::object::array_tail_transition::test_clear();
+    crate::object::array_tail_transition::test_publish_without_arming(3);
+    crate::object::array_tail_transition::test_prune();
+}
+
+/// The skip's positive half: with nothing published the scan and the prune
+/// visit nothing, and once the production writer has armed the flag they walk
+/// the tables again.
+#[test]
+fn array_tail_tables_are_skipped_only_while_unarmed() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _clear = ClearTablesOnDrop;
+    crate::object::array_tail_transition::test_clear();
+    assert!(!crate::object::array_tail_transition::test_tables_may_hold_entries());
+    let _ = gc_collect_minor();
+    assert!(!crate::object::array_tail_transition::test_tables_may_hold_entries());
+    crate::object::array_tail_transition::test_arm_and_publish(3);
+    assert!(crate::object::array_tail_transition::test_tables_may_hold_entries());
+    crate::object::array_tail_transition::test_clear();
+}
+
+/// The small-int / ASCII-char caches are skipped by a minor outright. Their
+/// real entries (longlived, pinned) satisfy the skip's precondition.
+#[test]
+fn small_string_caches_hold_only_pinned_non_young_strings() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    // Fill a few entries through the production writers.
+    let _ = crate::string::js_number_to_string(7.0);
+    let _ = crate::string::js_number_to_string(200.0);
+    let _ = gc_collect_minor();
+    crate::string::debug_assert_small_string_caches_not_minor_relevant();
+}
+
+/// SABOTAGE: a writer that publishes a young string into the cache breaks
+/// the precondition of the minor skip, and the check says so.
+#[test]
+#[should_panic(expected = "not a pinned non-young string")]
+fn small_int_cache_writer_publishing_a_young_string_is_caught() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    struct Restore;
+    impl Drop for Restore {
+        fn drop(&mut self) {
+            crate::string::test_write_small_int_cache_slot(255, std::ptr::null_mut());
+        }
+    }
+    let _restore = Restore;
+    let young = young_leaf() as *mut crate::StringHeader;
+    crate::string::test_write_small_int_cache_slot(255, young);
+    crate::string::debug_assert_small_string_caches_not_minor_relevant();
+}

crates/perry-runtime/src/gc/tests/mod.rs

@@ -62,6 +62,7 @@ mod lazy_tape_side_alloc;
 mod leaf_marks;
 mod map_store;
 mod mark_slot_hoists;
+mod minor_fixed_cost;
 mod noncollecting_root_lock;
 mod object_create;
 mod old_free_intrusive;

crates/perry-runtime/src/object/array_tail_transition.rs

@@ -383,6 +383,8 @@ pub(crate) fn record_numeric_tail_transition(
     // chain: the generic fallback mints a different predecessor, after which
     // no historical edge can match. Preserve colliding entries with bounded
     // open addressing instead of overwriting one direct-mapped slot.
+    // Arm before publish: once set, the collector walks both tables.
+    object_hot_for_owner(owner).array_tail_occupied.set(true);
     let forward = with_forward_for_owner(owner, |table| unsafe { insert_forward(table, entry) });
     let reverse = with_reverse_for_owner(owner, |table| unsafe { insert_reverse(table, entry) });
     if forward.is_none() && reverse.is_none() {
@@ -502,7 +504,55 @@ unsafe fn scan_table(
     }
 }
 
+/// May either tail table hold a non-`EMPTY` slot on this thread?
+///
+/// `false` is exact, not a heuristic: the flag is set by the only production
+/// writer (`record_numeric_tail_transition`) before its first `publish_entry`,
+/// and nothing but `test_clear` ever returns a slot to `EMPTY` (a pruned entry
+/// becomes a tombstone). So while it is clear, every slot of both tables is
+/// `EMPTY` and the GC scan and prune below would visit nothing — they were
+/// 2 x 8192 slot reads per pass, ~265k instructions per copying minor, on
+/// every program that never extends `Array`.
+#[inline]
+fn tables_may_hold_entries() -> bool {
+    let occupied = crate::state::state().object_hot.array_tail_occupied.get();
+    #[cfg(any(debug_assertions, test))]
+    if !occupied {
+        debug_assert_tables_empty();
+    }
+    occupied
+}
+
+/// The proof obligation behind a skipped walk, checked where it is relied on:
+/// a slot published without arming `array_tail_occupied` fails here instead of
+/// leaving an unvisited keys-array root in the table.
+#[cfg(any(debug_assertions, test))]
+fn debug_assert_tables_empty() {
+    for (name, table) in [
+        (
+            "forward",
+            &crate::state::state().object_hot.array_tail_forward,
+        ),
+        (
+            "reverse",
+            &crate::state::state().object_hot.array_tail_reverse,
+        ),
+    ] {
+        let table = unsafe { &*table.get() };
+        if let Some(index) = table.iter().position(|entry| !entry.is_empty()) {
+            panic!(
+                "array-tail {name} table slot {index} is occupied but \
+                 `array_tail_occupied` is clear: a writer published an entry \
+                 without arming the flag first, so the GC would skip its roots"
+            );
+        }
+    }
+}
+
 pub(crate) fn scan_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) {
+    if !tables_may_hold_entries() {
+        return;
+    }
     with_forward(|table| unsafe { scan_table(table, visitor) });
     with_reverse(|table| unsafe { scan_table(table, visitor) });
 }
@@ -547,12 +597,19 @@ unsafe fn note_live_entries(
 
 #[cold]
 pub(crate) fn prune_invalid_entries() {
+    if !tables_may_hold_entries() {
+        return;
+    }
     with_forward(|table| unsafe { prune_table(table) });
     with_reverse(|table| unsafe { prune_table(table) });
 }
 
 #[cfg(test)]
 pub(crate) fn test_clear() {
+    crate::state::state()
+        .object_hot
+        .array_tail_occupied
+        .set(false);
     with_forward(|table| unsafe {
         for entry in (*table).iter_mut() {
             *entry = ArrayTailTransitionEntry::EMPTY;
@@ -565,6 +622,45 @@ pub(crate) fn test_clear() {
     });
 }
 
+#[cfg(test)]
+pub(crate) fn test_tables_may_hold_entries() -> bool {
+    tables_may_hold_entries()
+}
+
+#[cfg(test)]
+pub(crate) fn test_prune() {
+    prune_invalid_entries();
+}
+
+#[cfg(test)]
+fn test_entry(shape_id: u32) -> ArrayTailTransitionEntry {
+    ArrayTailTransitionEntry {
+        predecessor_keys: 0,
+        successor_keys: 0,
+        predecessor_shape_id: shape_id,
+        successor_shape_id: shape_id + 1,
+        slot: 0,
+        array_index: 0,
+        predecessor_live_inline_slots: 0,
+        successor_live_inline_slots: 0,
+    }
+}
+
+/// A writer that forgets to arm: publish straight into the forward table.
+#[cfg(test)]
+pub(crate) fn test_publish_without_arming(shape_id: u32) {
+    with_forward(|table| unsafe { insert_forward(table, test_entry(shape_id)) });
+}
+
+/// Arm exactly as `record_numeric_tail_transition` does, then publish.
+#[cfg(test)]
+pub(crate) fn test_arm_and_publish(shape_id: u32) {
+    object_hot_for_owner(std::ptr::null())
+        .array_tail_occupied
+        .set(true);
+    with_forward(|table| unsafe { insert_forward(table, test_entry(shape_id)) });
+}
+
 #[cfg(test)]
 mod zeroed_table_tests {
     use super::*;

crates/perry-runtime/src/object/mod.rs

@@ -648,6 +648,10 @@ pub(crate) struct ObjectHotTables {
     /// entry and fall back to the complete open-addressed tables.
     pub(crate) array_tail_direct:
         std::cell::UnsafeCell<Box<[array_tail_transition::ArrayTailDirectIndex]>>,
+    /// Set before the first entry is published into either tail table; while
+    /// false every slot of both is `EMPTY`, so the GC scan and the prune have
+    /// nothing to visit (see `array_tail_transition::tables_may_hold_entries`).
+    pub(crate) array_tail_occupied: Cell<bool>,
 }
 
 impl ObjectHotTables {
@@ -681,6 +685,7 @@ impl ObjectHotTables {
             array_tail_direct: std::cell::UnsafeCell::new(crate::zeroed_cache::new_zeroed_cache(
                 array_tail_transition::ARRAY_TAIL_TRANSITION_CACHE_SIZE,
             )),
+            array_tail_occupied: Cell::new(false),
         }
     }
 }

crates/perry-runtime/src/object/native_module.rs

@@ -46,12 +46,12 @@ pub(crate) use callable_exports::{
     module_cjs_cache_value, module_cjs_extensions_value, module_cjs_global_paths_value,
     module_cjs_path_cache_value, module_cjs_prototype_for_instance, module_constants_value,
     native_string_value, prune_dead_builtin_closure_metadata_owners,
-    scan_builtin_closure_metadata_roots_mut, scan_tls_derived_prototype_roots_mut,
-    set_bound_native_closure_name, set_builtin_closure_length,
-    set_builtin_closure_non_constructable, sqlite_session_constructor_value,
-    sqlite_statement_sync_constructor_value, timers_promises_parent_namespace,
-    tls_constructor_prototype_is_instance_of, util_inspect_default_options_value,
-    zlib_codes_object,
+    prune_dead_builtin_closure_metadata_owners_young, scan_builtin_closure_metadata_roots_mut,
+    scan_tls_derived_prototype_roots_mut, set_bound_native_closure_name,
+    set_builtin_closure_length, set_builtin_closure_non_constructable,
+    sqlite_session_constructor_value, sqlite_statement_sync_constructor_value,
+    timers_promises_parent_namespace, tls_constructor_prototype_is_instance_of,
+    util_inspect_default_options_value, zlib_codes_object,
 };
 pub(crate) use constants::get_native_module_constant;
 pub(crate) use constructor_exports::{