From e23cdc1e2d59455964ad8d58df16e4f9c61a7e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 15:16:17 +0000 Subject: [PATCH 1/5] perf(runtime): read sites answer inherited and absent keys from holder-shape facts (A1, WIP) The read site cache gains a holder entry (receiver ShapeId, holder root, holder ShapeId, slot or absent, up to three hop roots and their ShapeIds), primed from get_field_ic_miss_impl after the generic getter confirms it and checked by the emitted tower where the MRU word and the ways miss: depth 1 inline, depth 2-4 through the GC-leaf stub js_read_site_holder_hit. Exotic read receivers now carry a per-object prototype identity in their shape. The emitted call to js_inherited_read_cache_hit_f64 is gone; the table itself stays until A2. --- changelog.d/inherited-read-holder-entry.md | 12 + crates/perry-abi/src/lib.rs | 13 + .../src/expr/property_get/generic_dispatch.rs | 160 +++++-- .../src/expr/property_get/tests.rs | 270 ++++------- .../src/gc_effects/linux-x86_64.tsv | 1 + crates/perry-codegen/src/root_reload.rs | 1 + .../src/runtime_decls/objects.rs | 1 + .../perry-codegen/src/wasm32/runtime_abi.tsv | 1 + crates/perry-runtime/src/gc/mod.rs | 4 + .../src/gc/tests/arguments_objects.rs | 19 + .../src/object/field_get_set/ic_miss.rs | 26 +- .../perry-runtime/src/object/method_site.rs | 6 +- .../src/object/method_site/read_holder.rs | 426 ++++++++++++++++++ .../src/object/proto_validity.rs | 18 +- crates/perry-runtime/src/object/shapes.rs | 9 +- crates/perry/tests/read_holder_entry.rs | 255 +++++++++++ scripts/gc_root_dominance_check.py | 4 + 17 files changed, 982 insertions(+), 244 deletions(-) create mode 100644 changelog.d/inherited-read-holder-entry.md create mode 100644 crates/perry-runtime/src/object/method_site/read_holder.rs create mode 100644 crates/perry/tests/read_holder_entry.rs diff --git a/changelog.d/inherited-read-holder-entry.md b/changelog.d/inherited-read-holder-entry.md new file mode 100644 index 0000000000..c4d980de41 --- /dev/null +++ b/changelog.d/inherited-read-holder-entry.md @@ -0,0 +1,12 @@ +An inherited or absent property read is now answered by the read site itself, +from facts of two shapes, instead of the inherited-read side table. When `o.k` +misses because `k` is not own, the miss handler records in the site cache the +receiver ShapeId, the object that holds `k` (a strong GC root), that +object ShapeId and the slot; an absent key records the terminal object +instead. The emitted read compares the receiver ShapeId and the holder +ShapeId and loads the slot inline (depth 1), or calls a GC-leaf stub that also +compares the intermediate hops (depth 2 to 4). A key added, deleted or +redefined on any object on the chain, or a `setPrototypeOf`, moves a ShapeId +the entry compares; a value store is seen because the slot is loaded. No global +validity word is involved. `process.env` and `arguments` now carry a per-object +prototype identity in their shape, so no shape-keyed memo admits them. diff --git a/crates/perry-abi/src/lib.rs b/crates/perry-abi/src/lib.rs index 61ffa65514..b0ed24e5e6 100644 --- a/crates/perry-abi/src/lib.rs +++ b/crates/perry-abi/src/lib.rs @@ -59,6 +59,19 @@ pub const METHOD_SITE_GEN_OFFSET: usize = 32; /// Entries per method site, and one entry's size. pub const METHOD_SITE_WAYS: usize = 2; pub const METHOD_SITE_ENTRY_SIZE: usize = 40; + +/// `object::method_site::read_holder` — the property-read cache words +/// (`PicCache`) holding the read site's holder entry, which the emitted read +/// tower checks where the MRU word and the ways miss +/// (`perry-codegen/src/expr/property_get/generic_dispatch.rs`). +pub const PIC_HOLDER_RECV_WORD: usize = 12; +pub const PIC_HOLDER_OBJ_WORD: usize = 13; +pub const PIC_HOLDER_SHAPE_WORD: usize = 14; +pub const PIC_HOLDER_KIND_WORD: usize = 15; +/// The kind word of a depth-1 ABSENT entry: the answer is `undefined`. +pub const PIC_HOLDER_ABSENT_DEPTH1: i64 = 1 << 62; +/// Words in a property-read cache: MRU, way state, four ways, the holder entry. +pub const PIC_CACHE_WORDS: usize = 21; /// A method site calls a body with its argument count padded by `undefined` /// up to this many extra arguments (never past 16), and admits bodies that /// declare up to that many parameters. diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 8c4f3e0074..30c15a256b 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -26,7 +26,7 @@ use crate::types::{DOUBLE, I1, I32, I64, I8, PTR}; /// here and `pic_cache_words_match_codegen` in the runtime: change one and both /// fail. #[cfg(test)] -pub(crate) const PIC_CACHE_WORDS: usize = 12; +pub(crate) const PIC_CACHE_WORDS: usize = 21; /// First word of the polymorphic way array (words 0..2 are the MRU entry and /// word 3 is the gate). Mirrors the runtime's `PIC_WAY_BASE`. pub(crate) const PIC_WAY_BASE: usize = 4; @@ -1021,13 +1021,8 @@ pub(crate) fn lower_generic_property_get( // signal byte-identical and go without the hook. ctx.current_block = ways_entry_idx; let token_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); - let inherited_idx = (!crate::expr::typed_feedback_emission_enabled()) - .then(|| ctx.new_block("pic.miss.inherited")); - let never_primed_label = inherited_idx - .map(|idx| ctx.block_label(idx)) - .unwrap_or_else(|| cold_label.clone()); ctx.block() - .cond_br(&token_cache.present, &miss_label, &never_primed_label); + .cond_br(&token_cache.present, &miss_label, &cold_label); // `js_object_get_field_ic_miss` primes only slots below the descriptor's // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact @@ -1179,7 +1174,9 @@ pub(crate) fn lower_generic_property_get( let ways_live = ctx.block().icmp_sgt(I64, &way_state, "0"); let ways_idx = ctx.new_block("pic.ways"); let ways_label = ctx.block_label(ways_idx); - ctx.block().cond_br(&ways_live, &ways_label, &call_label); + let holder_idx = ctx.new_block("pic.holder"); + let holder_label = ctx.block_label(holder_idx); + ctx.block().cond_br(&ways_live, &ways_label, &holder_label); ctx.current_block = ways_idx; // `is_object` is not ANDed in any more: it is statically true on every edge @@ -1229,7 +1226,8 @@ pub(crate) fn lower_generic_property_get( .expect("PIC_WAYS is non-zero, so the reduction leaves exactly one lane"); let way_load_idx = ctx.new_block("pic.way.load"); let way_load_label = ctx.block_label(way_load_idx); - ctx.block().cond_br(&way_any, &way_load_label, &call_label); + ctx.block() + .cond_br(&way_any, &way_load_label, &holder_label); ctx.current_block = way_load_idx; if fused_recv.is_some() { @@ -1287,45 +1285,111 @@ pub(crate) fn lower_generic_property_get( ctx.block().br(&call_label); } - // The inherited-read hook, on the never-primed edge only (see the branch - // that reaches it, in `pic.token.ways`). A read whose key lives on the - // prototype chain can never take the own-slot hit — the receiver's shape - // says the key is not own — so before this block it paid the slow entry's - // prologue and dispatch (79 of an inherited read's 204 instructions, - // measured by the inherited-reads lane) just to reach the same lookup - // inside `get_field_ic_miss_impl`. `js_inherited_read_cache_hit_f64` is - // a pure state read — it allocates nothing, triggers no GC and runs no - // user code — so it is a leaf in `gc_call_effects.rs` and - // `root_reload.rs`: no spill, no reload around it. `TAG_HOLE` is its - // decline sentinel, which no ordinary value can be, so the answer is one - // compare, with the SERVED edge as the true edge like every guard-passing - // edge in this tower (#7883); a decline continues to the one exit exactly - // as the never-primed edge did before. Nothing is primed from here: - // priming stays in the miss handler, the one place that already knows - // the key is not own without a second search. The versioned-loop deopt - // note is emitted here as it is on the exit, so entering either cold arm - // still records the bailout. - let inherited_arm = inherited_idx.map(|idx| { - ctx.current_block = idx; - crate::expr::emit_versioned_loop_callback_deopt(ctx); - let inh_key_handle = emit_key_handle(ctx, &key_handle_global); - let handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); - let recv_ptr = ctx.block().inttoptr(I64, &handle); - let key_ptr = ctx.block().inttoptr(I64, &inh_key_handle); - let val_inherited = ctx.block().call( + // The read site's HOLDER entry (`object::method_site::read_holder` in the + // runtime), on the two edges where the MRU word and the ways have missed: + // the answer for a key that is NOT own on the receiver, as facts of two + // shapes. The receiver's ShapeId says the key is not own and which object + // is its [[Prototype]]; the holder's ShapeId says the key is an own inline + // data slot there (or, for an ABSENT entry, that the terminal object lacks + // it). Both are compared here, the holder is a strong root in the site, and + // the value is LOADED, so no global word and no invalidation exist. A + // stable tombstone (#9064) can clear the holder's slot without moving its + // ShapeId, so a loaded `TAG_HOLE` goes to the call. + // + // [cache + RECV] == token else call + // kind = [cache + KIND] ; kind stub (depth 2..4, deep absent) + // h = [cache + OBJ] ; [h + 4] == low32([cache + SHAPE]) else call + // kind == ABSENT_DEPTH1 -> undefined + // v = [h + HDR + 8*kind] ; v != TAG_HOLE else call + // + // A site that never primed has no cache and so no entry; its edge goes + // straight to the call, which primes (`get_field_ic_miss_impl`). + let holder_arm: Vec<(String, String)> = { + let word = |ctx: &mut FnCtx<'_>, w: usize| { + let p = ctx.block().gep(I64, &cache_ref, &[(I64, &w.to_string())]); + ctx.block().load(I64, &p) + }; + let mut arms = Vec::with_capacity(3); + ctx.current_block = holder_idx; + let recv_word = word(ctx, crate::runtime_abi::PIC_HOLDER_RECV_WORD); + let recv_eq = ctx.block().icmp_eq(I64, &recv_word, &token); + let kind_idx = ctx.new_block("pic.holder.kind"); + let kind_label = ctx.block_label(kind_idx); + ctx.block().cond_br(&recv_eq, &kind_label, &call_label); + + ctx.current_block = kind_idx; + let kind = word(ctx, crate::runtime_abi::PIC_HOLDER_KIND_WORD); + let is_stub = ctx.block().icmp_slt(I64, &kind, "0"); + let stub_idx = ctx.new_block("pic.holder.stub"); + let stub_label = ctx.block_label(stub_idx); + let inline_idx = ctx.new_block("pic.holder.inline"); + let inline_label = ctx.block_label(inline_idx); + ctx.block().cond_br(&is_stub, &stub_label, &inline_label); + + ctx.current_block = inline_idx; + let holder = word(ctx, crate::runtime_abi::PIC_HOLDER_OBJ_WORD); + let holder_shape = word(ctx, crate::runtime_abi::PIC_HOLDER_SHAPE_WORD); + let holder_shape32 = ctx.block().trunc(I64, &holder_shape, I32); + let hsid_addr = ctx.block().add(I64, &holder, "4"); + let hsid_ptr = ctx.block().inttoptr(I64, &hsid_addr); + let hsid = ctx.block().load(I32, &hsid_ptr); + let holder_eq = ctx.block().icmp_eq(I32, &hsid, &holder_shape32); + let answer_idx = ctx.new_block("pic.holder.answer"); + let answer_label = ctx.block_label(answer_idx); + ctx.block().cond_br(&holder_eq, &answer_label, &call_label); + + ctx.current_block = answer_idx; + let absent = ctx.block().icmp_eq( + I64, + &kind, + &crate::runtime_abi::PIC_HOLDER_ABSENT_DEPTH1.to_string(), + ); + let absent_idx = ctx.new_block("pic.holder.absent"); + let absent_label = ctx.block_label(absent_idx); + let load_idx = ctx.new_block("pic.holder.load"); + let load_label = ctx.block_label(load_idx); + ctx.block().cond_br(&absent, &absent_label, &load_label); + + ctx.current_block = absent_idx; + let undef = ctx + .block() + .bitcast_i64_to_double(crate::nanbox::TAG_UNDEFINED_I64); + let absent_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + arms.push((undef, absent_end)); + + ctx.current_block = load_idx; + let offset = ctx.block().shl(I64, &kind, "3"); + let base = ctx.block().add(I64, &holder, &obj_header_size); + let field_addr = ctx.block().add(I64, &base, &offset); + let field_ptr = ctx.block().inttoptr(I64, &field_addr); + let val = ctx.block().load(DOUBLE, &field_ptr); + let bits = ctx.block().bitcast_double_to_i64(&val); + let live = ctx.block().icmp_ne(I64, &bits, crate::nanbox::TAG_HOLE_I64); + let load_end = ctx.block().label.clone(); + ctx.block().cond_br(&live, &merge_label, &call_label); + arms.push((val, load_end)); + + // Depth 2..4 and a deep absent entry: the hop words are compared by a + // GC-leaf stub (no allocation, no collection, no user code — a leaf in + // `gc_call_effects` and `root_reload.rs`); `TAG_HOLE` declines. + ctx.current_block = stub_idx; + let stub_handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); + let stub_recv = ctx.block().inttoptr(I64, &stub_handle); + let stub_val = ctx.block().call( DOUBLE, - "js_inherited_read_cache_hit_f64", - &[(PTR, &recv_ptr), (PTR, &key_ptr)], + "js_read_site_holder_hit", + &[(PTR, &stub_recv), (PTR, &cache_ref)], ); - let inherited_bits = ctx.block().bitcast_double_to_i64(&val_inherited); - let inherited_served = - ctx.block() - .icmp_ne(I64, &inherited_bits, crate::nanbox::TAG_HOLE_I64); - let inherited_end_label = ctx.block().label.clone(); - ctx.block() - .cond_br(&inherited_served, &merge_label, &cold_label); - (val_inherited, inherited_end_label) - }); + let stub_bits = ctx.block().bitcast_double_to_i64(&stub_val); + let served = ctx + .block() + .icmp_ne(I64, &stub_bits, crate::nanbox::TAG_HOLE_I64); + let stub_end = ctx.block().label.clone(); + ctx.block().cond_br(&served, &merge_label, &call_label); + arms.push((stub_val, stub_end)); + arms + }; // The object exit: one call reproducing every pointer-path arm this tower // used to expand. @@ -1381,8 +1445,8 @@ pub(crate) fn lower_generic_property_get( (&val_miss, &miss_end_label), (&val_nonptr, &nonptr_end_label), ]; - if let Some((val_inherited, inherited_end_label)) = inherited_arm.as_ref() { - incoming.push((val_inherited, inherited_end_label)); + for (val, label) in holder_arm.iter() { + incoming.push((val, label)); } if let Some((sso_val, sso_end_label)) = sso_arm.as_ref() { incoming.push((sso_val, sso_end_label)); diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index bf11d52055..2d101d86c0 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -359,14 +359,16 @@ fn fs_promises_native_module_value_uses_submodule_singleton() { fn pic_cache_layout_matches_runtime() { use crate::expr::property_get::generic_dispatch::{PIC_CACHE_WORDS, PIC_WAYS, PIC_WAY_BASE}; assert_eq!( - PIC_CACHE_WORDS, 12, - "perry-runtime's PIC_CACHE_WORDS is 12; update both sides together" + PIC_CACHE_WORDS, 21, + "perry-runtime's PIC_CACHE_WORDS is 21; update both sides together" ); assert_eq!( PIC_WAY_BASE + PIC_WAYS * 2, - PIC_CACHE_WORDS, - "the ways must fill the emitted global exactly" + crate::runtime_abi::PIC_HOLDER_RECV_WORD, + "the holder entry starts where the ways end" ); + assert!(crate::runtime_abi::PIC_HOLDER_KIND_WORD < PIC_CACHE_WORDS); + assert_eq!(crate::runtime_abi::PIC_CACHE_WORDS, PIC_CACHE_WORDS); let ir = emit(false, None); let ic_defs: Vec<&str> = ir .lines() @@ -1059,13 +1061,12 @@ fn generic_property_get_slot_load_is_reached_only_through_every_guard() { "the overflow-bit test must not gate the inline slot load — a spill \ entry is refused by the ShapeId compare itself:\n{chain}" ); - // The inherited-read hook (#10834/#10842) lives on the DECLINED edge. Its - // answer must never be a condition on the way to the own slot load: if it - // were, an own read would pay a call, and this walk would have collected - // the call's result in the chain. + // The holder entry lives past the ways. Its stub's answer must never be a + // condition on the way to the own slot load: if it were, an own read would + // pay a call, and this walk would have collected the call's result. assert!( - !chain.contains("js_inherited_read_cache_hit_f64"), - "the inherited-read hook must not gate the inline slot load:\n{chain}" + !chain.contains("js_read_site_holder_hit"), + "the holder stub must not gate the inline slot load:\n{chain}" ); // The GC header is not read on the way to the slot load at all: neither @@ -1616,10 +1617,17 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { // `pic.way.live` is GONE with the way path's `TAG_HOLE` compare: the // load block has nothing left to decide and branches to the merge. "pic.way.load", - // the inherited-read hook, on the never-primed edge out of - // `pic.token.ways` and nowhere else (`js_inherited_read_cache_hit_f64`, - // a leaf); a decline continues to the one exit - "pic.miss.inherited", + // the holder entry (`method_site::read_holder`), past the ways: the + // receiver word, the kind, the inline depth-1 compare and load (or + // `undefined` for an absent entry), and the GC-leaf stub for depth + // 2..4; every decline continues to the one exit + "pic.holder", + "pic.holder.kind", + "pic.holder.inline", + "pic.holder.answer", + "pic.holder.absent", + "pic.holder.load", + "pic.holder.stub", // the one exit, and the join "pic.miss.call", "pget.recv_merge", @@ -1644,199 +1652,87 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { ); } -/// The inherited-read cache (#10834/#10842) is asked on the NEVER-PRIMED edge -/// and nowhere else. A read whose key lives on the prototype chain is never an -/// own slot on the receiver's shape, so a site that only reads such a key never -/// resolves its per-site cache, and every read of it reaches `pic.token.ways` -/// with `present` false. That edge — which used to go straight to the exit — -/// now asks the cache before calling out. The first placement asked on EVERY -/// path into the exit and charged each own-key miss a declining probe (+88 on -/// a megamorphic site, +89 on a spill read, measured); this one costs every -/// other path zero instructions. +/// The read site's holder entry is checked where the MRU word and the ways +/// have missed, and nowhere else: /// -/// Five things are pinned, each of which would otherwise fail silently (the -/// program still computes the right value through the slow entry): -/// -/// 1. the hook call sits in `pic.miss.inherited` and in no other block, in -/// particular NOT on any path to the inline slot load (the CFG-walk test -/// asserts the same from the other side); -/// 2. that block is reached from `pic.token.ways` on the FALSE edge of the -/// cache-present test, and from nowhere else; -/// 3. its result is branched on with the SERVED edge as the true edge, the -/// tower's rule for every guard-passing edge, and the false edge is the -/// one exit; -/// 4. the slow entry is still called from `pic.miss.call` only, with the same -/// four operands; -/// 5. the merge phi takes the served value from `pic.miss.inherited`. +/// 1. the never-primed edge (`present` false) goes straight to the one exit — +/// a site with no cache has no entry; +/// 2. both edges out of the ways (`pic.miss` with no live way, `pic.ways` with +/// no matching way) go to `pic.holder`; +/// 3. the stub is called from `pic.holder.stub` only, and its `TAG_HOLE` +/// decline and the inline load's hole compare both continue to the exit. #[test] -fn the_inherited_read_cache_is_asked_on_the_never_primed_edge_only() { +fn the_holder_entry_is_checked_past_the_ways_only() { let ir = emit(false, None); let func = ir .split("\ndefine ") .find(|f| f.contains("\npic.miss.call")) .unwrap_or_else(|| panic!("no function contains the generic tower:\n{ir}")); let mut blocks: Vec<(String, Vec)> = Vec::new(); - let mut cur: Option<(String, Vec)> = None; for line in func.lines() { if !line.starts_with(' ') && line.ends_with(':') { - if let Some(b) = cur.take() { - blocks.push(b); - } - cur = Some((line.trim_end_matches(':').to_string(), Vec::new())); - continue; - } - if let Some((_, body)) = cur.as_mut() { + blocks.push((line.trim_end_matches(':').to_string(), Vec::new())); + } else if let Some((_, body)) = blocks.last_mut() { body.push(line.trim().to_string()); } } - if let Some(b) = cur.take() { - blocks.push(b); - } - // 1. one caller block, and it is the inherited arm. - let holders: Vec<&str> = blocks - .iter() - .filter(|(_, body)| { - body.iter() - .any(|l| l.contains("call double @js_inherited_read_cache_hit_f64(")) - }) - .map(|(l, _)| l.as_str()) - .collect(); - assert_eq!( - holders.len(), - 1, - "the inherited hook must be called from exactly one block: {holders:?}\n{func}" - ); - let inh_label = holders[0]; - assert!( - inh_label.starts_with("pic.miss.inherited"), - "the hook belongs on the never-primed edge, found it in `{inh_label}`:\n{func}" - ); - let (_, inh_body) = blocks.iter().find(|(l, _)| l == inh_label).unwrap(); - let hook_line = inh_body - .iter() - .find(|l| l.contains("@js_inherited_read_cache_hit_f64(")) - .unwrap(); - assert!( - hook_line.contains("(ptr %") && hook_line.matches(", ptr %").count() == 1, - "the hook takes the masked receiver and the interned key as two \ - pointers:\n{hook_line}" - ); - // 2. reached only from `pic.token.ways`, on the FALSE edge of `present`. - let preds: Vec<(&str, &str)> = blocks - .iter() - .flat_map(|(l, body)| { - body.iter() - .filter(|t| t.starts_with("br ") && t.contains(&format!("label %{inh_label}"))) - .map(move |t| (l.as_str(), t.as_str())) - }) - .collect(); - assert_eq!( - preds.len(), - 1, - "exactly one edge may reach the hook: {preds:?}\n{func}" - ); - let (pred_label, pred_term) = preds[0]; - assert!( - pred_label.starts_with("pic.token.ways"), - "the hook's one predecessor must be the cache-present test: {pred_label}" - ); - let parts: Vec<&str> = pred_term - .strip_prefix("br i1 ") - .unwrap() - .split(", ") - .collect(); - assert!( - parts[1].starts_with("label %pic.miss") && !parts[1].starts_with("label %pic.miss.inh"), - "the TRUE edge of `present` must still be the way compares: {pred_term}" - ); - assert!( - parts[2].starts_with(&format!("label %{inh_label}")), - "the hook must sit on the FALSE (never-primed) edge: {pred_term}" - ); - let present_def = blocks - .iter() - .find(|(l, _)| l == pred_label) - .and_then(|(_, body)| { - body.iter() - .find(|l| l.starts_with(&format!("{} = ", parts[0]))) - }) - .unwrap_or_else(|| { - panic!( - "the branch condition {} must be defined in {pred_label}", - parts[0] - ) - }); - assert!( - present_def.contains("icmp ne ptr ") && present_def.ends_with(", null"), - "`present` is the cache slot's non-null test:\n{present_def}" - ); - // 3. polarity: `icmp ne , TAG_HOLE` is "served", served is the TRUE - // edge and lands on the merge; the false edge is the one exit. - let served = inh_body - .iter() - .find(|l| l.contains("icmp ne i64 ") && l.ends_with(crate::nanbox::TAG_HOLE_I64)) - .unwrap_or_else(|| panic!("the decline compare against TAG_HOLE:\n{func}")); - let cond = served.split_once(" = ").map(|(c, _)| c).unwrap(); - let term = inh_body - .iter() - .rev() - .find(|l| l.starts_with("br ")) - .unwrap(); - let parts: Vec<&str> = term - .strip_prefix("br i1 ") - .unwrap_or_else(|| panic!("the arm must branch on the hook's answer: {term}")) - .split(", ") - .collect(); - assert_eq!( - parts[0], cond, - "the branch must be on the served predicate: {term}" - ); - assert!( - parts[1].starts_with("label %pget.recv_merge"), - "the SERVED edge must be the true edge and land on the merge: {term}" - ); + let term = |prefix: &str| -> String { + let (_, body) = blocks + .iter() + .find(|(l, _)| { + l.strip_prefix(prefix).is_some_and(|r| { + r.strip_prefix('.') + .is_some_and(|n| n.parse::().is_ok()) + }) + }) + .unwrap_or_else(|| panic!("no block {prefix}:\n{func}")); + body.iter() + .rev() + .find(|l| l.starts_with("br ")) + .unwrap() + .clone() + }; + // 1. + let ways_entry = term("pic.token.ways"); assert!( - parts[2].starts_with("label %pic.miss.call"), - "the decline must be the false edge into the one exit: {term}" + ways_entry.contains("label %pic.miss.") && ways_entry.contains("label %pic.miss.call"), + "the never-primed edge goes straight to the exit: {ways_entry}" ); - // 4. the slow entry: one caller, the exit, same operands. - let slow_callers: Vec<&str> = blocks + // 2. + for b in ["pic.miss", "pic.ways"] { + let t = term(b); + assert!( + t.contains("label %pic.holder."), + "{b} must fall to the holder entry: {t}" + ); + assert!( + !t.contains("label %pic.miss.call"), + "{b} must not skip the holder entry: {t}" + ); + } + // 3. + let callers: Vec<&str> = blocks .iter() - .filter(|(_, body)| { - body.iter() - .any(|l| l.contains("@js_object_get_field_ic_slow(")) - }) + .filter(|(_, body)| body.iter().any(|l| l.contains("@js_read_site_holder_hit("))) .map(|(l, _)| l.as_str()) .collect(); - assert_eq!(slow_callers.len(), 1, "{slow_callers:?}"); assert!( - slow_callers[0].starts_with("pic.miss.call"), - "the slow entry must be called from the one exit: {slow_callers:?}" - ); - let (_, slow_body) = blocks.iter().find(|(l, _)| l == slow_callers[0]).unwrap(); - let slow_line = slow_body - .iter() - .find(|l| l.contains("@js_object_get_field_ic_slow(")) - .unwrap(); - assert!( - slow_line.contains("ptr @perry_ic_") && slow_line.contains("_packed_get"), - "the slow entry must still receive the cache slot and the packed \ - word:\n{slow_line}" - ); - // 5. the merge takes the served value from the inherited arm. - let (_, merge_body) = blocks - .iter() - .find(|(l, _)| l.starts_with("pget.recv_merge")) - .unwrap(); - let phi = merge_body - .iter() - .find(|l| l.contains(" = phi double ")) - .unwrap(); - let served_value = hook_line.split_once(" = ").map(|(v, _)| v).unwrap(); - assert!( - phi.contains(&format!("[ {served_value}, %{inh_label} ]")), - "the merge must take the hook's value from `{inh_label}`:\n{phi}" + callers.len() == 1 && callers[0].starts_with("pic.holder.stub"), + "the stub is called from pic.holder.stub only: {callers:?}" ); + for b in ["pic.holder.stub", "pic.holder.load"] { + let (_, body) = blocks.iter().find(|(l, _)| l.starts_with(b)).unwrap(); + assert!( + body.iter() + .any(|l| l.contains("icmp ne i64 ") && l.ends_with(crate::nanbox::TAG_HOLE_I64)), + "{b} must decline on TAG_HOLE:\n{func}" + ); + let t = term(b); + assert!( + t.contains("label %pget.recv_merge") && t.contains("label %pic.miss.call"), + "{b}: served to the merge, declined to the exit: {t}" + ); + } } #[path = "array_length_tests.rs"] diff --git a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv index 9d6ab0aeef..f48998ed63 100644 --- a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv @@ -2613,6 +2613,7 @@ js_queue_microtask Reenters js_queue_next_tick Reenters js_queue_next_tick_args Reenters js_rangeerror_new Reenters +js_read_site_holder_hit Leaf js_readable_stream_cancel Reenters js_readable_stream_controller_byob_request Reenters js_readable_stream_controller_close Reenters diff --git a/crates/perry-codegen/src/root_reload.rs b/crates/perry-codegen/src/root_reload.rs index 49ea2345a5..5aea9bdaaa 100644 --- a/crates/perry-codegen/src/root_reload.rs +++ b/crates/perry-codegen/src/root_reload.rs @@ -209,6 +209,7 @@ const NON_COLLECTING: &[&str] = &[ "perry_transition_cache_base", "js_transition_ic_note_hit", "js_inherited_read_cache_hit_f64", + "js_read_site_holder_hit", // S2 GC-leaf IC hits; proven `Leaf` by the generated call-effects table. "js_object_get_field_ic_fast", "js_class_field_get_ic_fast", diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index 5228f32a4e..dee6b2f418 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -621,6 +621,7 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // value, or `TAG_HOLE` for a decline. A pure state read (see // `gc_call_effects.rs`). module.declare_function("js_inherited_read_cache_hit_f64", DOUBLE, &[PTR, PTR]); + module.declare_function("js_read_site_holder_hit", DOUBLE, &[PTR, PTR]); // The per-agent pointer block (`expr/agent_ptr.rs`), read inline on ELF // executables through the initial-exec TLS model; its slot-1 accessor, // and the method-call site's miss entry (`expr/method_site.rs`). diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index 4a061bf583..a26633f433 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -3090,6 +3090,7 @@ js_queue_microtask void i64 js_queue_next_tick void i64 js_queue_next_tick_args void i64,ptr,i32s js_rangeerror_new ptr ptr +js_read_site_holder_hit f64 ptr,ptr js_readable_stream_cancel ptr f64,f64 js_readable_stream_controller_byob_request f64 f64 js_readable_stream_controller_close f64 f64 diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 5536fa94a9..51aa9abe40 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -1083,6 +1083,10 @@ pub fn gc_init() { // Method-calls lane: an inherited method-site entry holds the method // closure it calls, so the closure is a STRONG root (`object::method_site`). reg_scanner!(crate::object::method_site::scan_method_site_roots_mut); + // A read site's holder entry names the object that holds the answer (and + // the hops to it); the emitted hit loads through it, so each is a STRONG + // root (`object::method_site::read_holder`). + reg_scanner!(crate::object::method_site::read_holder::scan_read_holder_roots_mut); reg_scanner!(crate::map::scan_map_iterator_array_roots_mut); reg_scanner!(crate::set::scan_set_iterator_array_roots_mut); reg_scanner!(crate::perf_hooks::scan_perf_entries_roots_mut); diff --git a/crates/perry-runtime/src/gc/tests/arguments_objects.rs b/crates/perry-runtime/src/gc/tests/arguments_objects.rs index 834044f21a..e8985835b3 100644 --- a/crates/perry-runtime/src/gc/tests/arguments_objects.rs +++ b/crates/perry-runtime/src/gc/tests/arguments_objects.rs @@ -252,3 +252,22 @@ fn only_the_state_word_identifies_an_arguments_object() { assert_eq!(test_arguments_mapped_box(restricted, 0), None); assert!(test_arguments_mapping_array(restricted).is_none()); } + +/// The exotic flag is a SHAPE fact: marking moves the receiver to a shape +/// whose [[Prototype]] identity is its own, which no shape-keyed read memo +/// admits (`object::method_site::read_holder`), and a key added afterwards +/// keeps it. +#[test] +fn the_exotic_flag_moves_the_receiver_to_a_per_object_identity() { + let _guard = GcTestIsolationGuard::with_realm_bootstrapped(); + let obj = js_object_alloc(0, 1); + let before = unsafe { crate::object::shapes::object_shape_stamp(obj) }; + unsafe { crate::object::proto_validity::mark_exotic_read_receiver(obj as usize) }; + let after = unsafe { crate::object::shapes::object_shape_stamp(obj) }; + assert_ne!(before, after, "marking must move the receiver's ShapeId"); + assert_eq!( + crate::object::shapes::shape_proto_id(after), + Some(crate::object::shapes::PROTO_ID_PER_OBJECT), + "the marked receiver's shape must carry a per-object identity" + ); +} diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index c04a742c8b..e2e451343a 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -208,7 +208,7 @@ pub(crate) fn set_method_value_name(key: &[u8]) -> Option<&'static [u8]> { /// Words in a per-site property-read cache global (`@perry_ic_N`). Codegen /// emits `[PIC_CACHE_WORDS x i64] zeroinitializer`; this type is the runtime's /// view of the same memory. -pub const PIC_CACHE_WORDS: usize = 12; +pub const PIC_CACHE_WORDS: usize = crate::codegen_abi::PIC_CACHE_WORDS; /// The runtime view of a `@perry_ic_N` property-read cache. /// @@ -221,7 +221,8 @@ pub const PIC_CACHE_WORDS: usize = 12; /// | 1 | `slot0` — its resolved field slot | /// | 2 | unused — was the Array-subclass named-prefix token, retired by S6 (site state must derive from one shape) | /// | 3,4 / 5,6 / 7,8 / 9,10 | `(tok, slot)` ways | -/// | 11 | round-robin victim index for the ways | +/// | 3 | way state ([`PIC_WAY_STATE`]) | +/// | 12..=20 | the holder entry for a key that is not own (`method_site::read_holder`) | pub type PicCache = [i64; PIC_CACHE_WORDS]; /// The value a per-site compact MRU word (`@perry_ic_N_packed_get`) holds @@ -1129,6 +1130,13 @@ pub(super) fn get_field_ic_miss_impl( // +106 instructions per read against the same binary with // `PERRY_INHERITED_IC=0`, i.e. the cache was pure overhead for // this shape. + // The site's holder entry: primed here, answered by the + // emitted tower from then on (`method_site::read_holder`). + if let Some(value) = + crate::object::method_site::read_holder::prime_read_holder(obj, key, cache_slot) + { + return f64::from_bits(value.bits()); + } if !inherited_declined { // Already inside this function's `unsafe` block (line 874), // so a nested one is `unused_unsafe` under -D warnings. @@ -1282,6 +1290,14 @@ pub(super) fn get_field_ic_miss_impl( // paying for a second search. Walk the chain once and record the answer. // A decline leaves the generic getter below untouched, which is today's // behaviour for every case the cache refuses. + if matches!(miss_reason, R::NotOwn) { + // The site's holder entry (`method_site::read_holder`). + if let Some(value) = unsafe { + crate::object::method_site::read_holder::prime_read_holder(obj, key, cache_slot) + } { + return f64::from_bits(value.bits()); + } + } if matches!(miss_reason, R::NotOwn) && !inherited_declined { if let Some(value) = unsafe { crate::object::inherited_read_cache::inherited_read_cache_prime(obj, key) } @@ -2425,8 +2441,8 @@ mod poly_pic_tests { #[test] fn pic_cache_words_match_codegen() { assert_eq!( - PIC_CACHE_WORDS, 12, - "codegen emits `[12 x i64]`; update both sides together" + PIC_CACHE_WORDS, 21, + "codegen's PIC_CACHE_WORDS is 21; update both sides together" ); assert!( PIC_WAY_STATE < PIC_CACHE_WORDS, @@ -2438,7 +2454,7 @@ mod poly_pic_tests { ); assert_eq!( PIC_WAY_BASE + PIC_WAYS * 2, - PIC_CACHE_WORDS, + crate::codegen_abi::PIC_HOLDER_RECV_WORD, "the ways must fill the global exactly" ); } diff --git a/crates/perry-runtime/src/object/method_site.rs b/crates/perry-runtime/src/object/method_site.rs index 2170bc2b21..cfea8ff7d0 100644 --- a/crates/perry-runtime/src/object/method_site.rs +++ b/crates/perry-runtime/src/object/method_site.rs @@ -86,6 +86,8 @@ //! a worker never reads a primary-heap closure through a site. use crate::object::ObjectHeader; + +pub(crate) mod read_holder; use std::sync::atomic::{AtomicU64, Ordering}; /// `word` of a site no prime has touched: no receiver word is all-ones. @@ -196,6 +198,7 @@ static WORKER_AGENTS_EXIST: std::sync::atomic::AtomicBool = pub fn note_worker_agent() { if !WORKER_AGENTS_EXIST.swap(true, Ordering::SeqCst) { super::proto_validity::bump_proto_validity(); + read_holder::empty_read_holder_entries(); } } @@ -280,8 +283,9 @@ fn stats_report_enabled() -> bool { refused.push_str(&format!(" refused.{}={n}", REFUSALS[i])); } } + let (hd, ha, hr) = read_holder::read_holder_stats(); eprintln!( - "[method-site] primes_own={a} primes_inherited={b} primes_function={} misses={c} marked_value_write_bumps={}{refused}", + "[method-site] primes_own={a} primes_inherited={b} primes_function={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_holder_refused={hr} marked_value_write_bumps={}{refused}", method_site_function_primes(), crate::object::proto_validity::marked_value_write_bumps() ); diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs new file mode 100644 index 0000000000..3c56147fe0 --- /dev/null +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -0,0 +1,426 @@ +//! The read site's HOLDER entry: `o.k` where `k` is not an own key of the +//! receiver, answered by facts of two shapes. +//! +//! * The receiver's ShapeId `S` vouches that `k` is not own, that the receiver +//! is an ordinary object, and its [[Prototype]] identity. Only a serial +//! identity or `PROTO_ID_DEFAULT` (the realm's `Object.prototype`) pins ONE +//! object, so only those admit. +//! * The holder's ShapeId `SH` vouches that `k` is an own inline data slot of +//! the holder `H` — or, for an ABSENT entry, that the terminal object lacks +//! `k` and has a null [[Prototype]]. +//! * For a holder deeper than the direct prototype, each intermediate hop's +//! ShapeId vouches that the hop lacks `k` and still links to the next hop. +//! +//! Every fact is compared on use, so there is no invalidation and no global +//! word: a key add, delete, descriptor change or `setPrototypeOf` on any object +//! the entry names moves that object's ShapeId, and a value store to the +//! holder's slot is seen because the hit LOADS the slot. A stable tombstone +//! (#9064) can clear the holder's slot without moving its ShapeId, so a loaded +//! `TAG_HOLE` is a miss. +//! +//! The entry lives in the read site's own cache (`PicCache` words +//! [`HOLDER_RECV`]..=[`HOLDER_REGISTERED`]). The holder and the hops are +//! STRONG roots, rewritten when they move ([`scan_read_holder_roots_mut`]). +//! +//! # Emitted form +//! +//! Codegen (`generic_dispatch.rs`) checks the entry on the edges where the +//! MRU word and the polymorphic ways have missed. A depth-1 entry is compared +//! and loaded inline; a deeper one calls [`js_read_site_holder_hit`], a +//! GC leaf that compares the hop words and answers `TAG_HOLE` to decline. +//! +//! # Priming +//! +//! Only from the read miss handler, which already knows the key is not own, +//! and only after the generic getter has produced the answer: the entry is +//! recorded only when what the shapes say equals what the getter returned +//! (names the runtime synthesizes, lazily materialized intrinsics and +//! `constructor` refuse there). Primary agent only; a worker agent's start +//! empties every entry. + +use super::{key_may_be_accessor, next_prototype, ordinary_receiver, WORKER_AGENTS_EXIST}; +use crate::object::shapes::{ + object_proto_id, object_shape_descriptor, object_shape_stamp, shape_proto_id, ShapeObjectKind, + PIC_ID_TOKEN_BIT, PROTO_ID_DEFAULT, PROTO_ID_NULL, +}; +use crate::object::{ObjectHeader, PicCache, PicCacheSlot}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// The receiver's ShapeId as a PIC token (`ShapeId | PIC_ID_TOKEN_BIT`), or 0 +/// for an empty entry. A zeroed cache is therefore an empty one: no token is 0. +pub const HOLDER_RECV: usize = crate::codegen_abi::PIC_HOLDER_RECV_WORD; +/// The holder's (or, for an absent entry, the terminal object's) address. +pub const HOLDER_OBJ: usize = crate::codegen_abi::PIC_HOLDER_OBJ_WORD; +/// Low 32 bits: the holder's ShapeId. High 32 bits: the third hop's ShapeId. +pub const HOLDER_SHAPE: usize = crate::codegen_abi::PIC_HOLDER_SHAPE_WORD; +/// The answer's kind, laid out for the emitted test: +/// +/// | value | meaning | +/// |---|---| +/// | `0 ..= u32::MAX` | depth 1, the value is the holder's inline slot | +/// | [`HOLDER_ABSENT_DEPTH1`] | depth 1, absent: the answer is `undefined` | +/// | negative | [`HOLDER_STUB`] set: call [`js_read_site_holder_hit`] | +pub const HOLDER_KIND: usize = crate::codegen_abi::PIC_HOLDER_KIND_WORD; +/// First of three intermediate hop addresses (depth 2..=4). +pub const HOLDER_HOPS: usize = HOLDER_KIND + 1; +/// Low 32 bits: the first hop's ShapeId. High 32 bits: the second hop's. +pub const HOLDER_HOP_SHAPES: usize = HOLDER_HOPS + 3; +/// Nonzero once the cache is on the root list. +pub const HOLDER_REGISTERED: usize = HOLDER_HOP_SHAPES + 1; + +pub const HOLDER_ABSENT_DEPTH1: i64 = crate::codegen_abi::PIC_HOLDER_ABSENT_DEPTH1; +pub const HOLDER_STUB: u64 = 1 << 63; +const HOLDER_ABSENT_BIT: u64 = 1 << 62; +const HOLDER_DEPTH_SHIFT: u32 = 32; +const HOLDER_MAX_DEPTH: usize = 4; + +/// Every cache that holds (or held) a holder entry, for the root scan and for +/// emptying the entries when the first worker agent starts. The entries are +/// in the per-site caches; this is only where the scan finds them. +static HOLDER_SITES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + +per_test_global! { + static PRIMES_HOLDER: AtomicU64 = AtomicU64::new(0); + static PRIMES_ABSENT: AtomicU64 = AtomicU64::new(0); + static REFUSED_HOLDER: AtomicU64 = AtomicU64::new(0); +} + +/// `(data primes, absent primes, refusals)`. +pub fn read_holder_stats() -> (u64, u64, u64) { + ( + PRIMES_HOLDER.load(Ordering::Relaxed), + PRIMES_ABSENT.load(Ordering::Relaxed), + REFUSED_HOLDER.load(Ordering::Relaxed), + ) +} + +#[inline] +fn refuse() { + REFUSED_HOLDER.fetch_add(1, Ordering::Relaxed); +} + +#[inline] +unsafe fn shape_word(addr: usize) -> u32 { + object_shape_stamp(addr as *const ObjectHeader) +} + +#[inline] +unsafe fn slot_bits(addr: usize, slot: u32) -> u64 { + std::ptr::read( + (addr as *const u8).add(std::mem::size_of::() + slot as usize * 8) + as *const u64, + ) +} + +/// Does `name` belong to the read fast path at all? Index-like names live in +/// elements, and the refused names are synthesized or special-cased by the +/// getter. +fn holder_name_admitted(name: &[u8]) -> bool { + !super::name_refused(name) && name != b"__proto__" && !name.iter().all(u8::is_ascii_digit) +} + +/// The answer the shapes give, found by a walk that allocates nothing. +struct Walk { + holder: usize, + holder_shape: u32, + /// `None` = absent. + slot: Option, + hops: [(usize, u32); HOLDER_MAX_DEPTH - 1], + depth: usize, +} + +/// A hop the entry may name: an ordinary, shaped, non-exotic object whose +/// ShapeId records the prototype identity it really has. +unsafe fn hop_admitted(addr: usize, name: &[u8]) -> bool { + if !crate::value::addr_class::is_above_handle_band(addr) + || !super::address_is_prime_stable(addr) + { + return false; + } + let Some(header) = crate::value::addr_class::try_read_gc_header(addr) else { + return false; + }; + let obj = addr as *const ObjectHeader; + if header.obj_type != crate::gc::GC_TYPE_OBJECT + || header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || header._reserved & crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO != 0 + || crate::closure::is_closure_ptr(addr) + || crate::object::dictionary::is_dictionary(obj) + { + return false; + } + let meta = (*obj).meta; + if !meta.is_null() + && ((*meta).elements != 0 + || (*meta).flags & crate::object::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER != 0) + { + return false; + } + !key_may_be_accessor(obj, name) +} + +/// The prototype identity `obj`'s shape records, if it admits: a serial, the +/// default link or null — and equal to what the object says it is. +unsafe fn admitted_proto_id(obj: *const ObjectHeader) -> Option { + let pid = shape_proto_id(object_shape_stamp(obj))?; + let serial = pid != PROTO_ID_DEFAULT && pid < crate::object::shapes::PROTO_ID_CLASS; + if !(serial || pid == PROTO_ID_DEFAULT || pid == PROTO_ID_NULL) { + return None; + } + (object_proto_id(obj) == pid).then_some(pid) +} + +unsafe fn walk(recv: *const ObjectHeader, name: &[u8]) -> Option { + let mut w = Walk { + holder: 0, + holder_shape: 0, + slot: None, + hops: [(0, 0); HOLDER_MAX_DEPTH - 1], + depth: 0, + }; + let object_prototype = crate::array::object_prototype_addr_if_resolved(); + let mut current = recv; + for depth in 1..=HOLDER_MAX_DEPTH { + // `%Object.prototype%` is an immutable-prototype exotic object: its + // [[Prototype]] is null for its whole life, whatever its shape's + // identity word says, so reaching it ends the chain. + let terminal = depth > 1 && current as usize == object_prototype; + let pid = if terminal { + PROTO_ID_NULL + } else { + admitted_proto_id(current)? + }; + if pid == PROTO_ID_NULL { + // `current` is the terminal object, and it lacks `name`. + if depth == 1 { + return None; + } + let (h, sh) = w.hops[depth - 2]; + w.hops[depth - 2] = (0, 0); + w.holder = h; + w.holder_shape = sh; + w.depth = depth - 1; + return Some(w); + } + let next = if pid == PROTO_ID_DEFAULT { + object_prototype as *const ObjectHeader + } else { + next_prototype(current) + }; + if next.is_null() || next == current || next == recv || !hop_admitted(next as usize, name) { + return None; + } + let shape = object_shape_descriptor(next)?; + if shape.object_kind != ShapeObjectKind::Ordinary || object_shape_stamp(next) == 0 { + return None; + } + let keys = shape.keys as usize as *const crate::array::ArrayHeader; + if !keys.is_null() { + if let Some(s) = + crate::object::keys_find_slot_by_bytes_resolved(keys, shape.logical_key_count, name) + { + if s >= shape.live_inline_slot_count { + return None; + } + w.holder = next as usize; + w.holder_shape = object_shape_stamp(next); + w.slot = Some(s); + w.depth = depth; + return Some(w); + } + } + if depth == HOLDER_MAX_DEPTH { + // A fifth object would be needed: either the holder or the null + // link past the last hop. + if next as usize != object_prototype && admitted_proto_id(next) != Some(PROTO_ID_NULL) { + return None; + } + w.holder = next as usize; + w.holder_shape = object_shape_stamp(next); + w.depth = depth; + return Some(w); + } + w.hops[depth - 1] = (next as usize, object_shape_stamp(next)); + current = next; + } + None +} + +/// Prime `cache_slot`'s holder entry for `obj.key`, whose key the caller has +/// proved is not own. Returns the answer (from the generic getter) when the +/// receiver took the generic read here; `None` when it did not, and the caller +/// reads as before. +/// +/// # Safety +/// `obj` is a live `GC_TYPE_OBJECT` receiver; `key` a live string header. +pub(crate) unsafe fn prime_read_holder( + obj: *const ObjectHeader, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, +) -> Option { + if cache_slot.is_null() + || key.is_null() + || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) + || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT + { + return None; + } + let name = crate::string::header_str_checked(key)?.as_bytes(); + let recv = ordinary_receiver(obj as usize)?; + // Cheap pre-walk: a receiver the entry could never describe keeps the + // caller's path and pays nothing for the getter below. + if !holder_name_admitted(name) || key_may_be_accessor(recv, name) || walk(recv, name).is_none() + { + refuse(); + return None; + } + + // The answer, from the generic getter. It can run user code and collect, + // so the receiver is rooted across it and everything is re-read after. + let scope = crate::gc::RuntimeHandleScope::new(); + let handle = scope.root_raw_mut_ptr(obj as *mut ObjectHeader); + let (value, obj) = handle.across_mut::(|| { + crate::object::field_get_set::get_field_by_name_past_inherited_cache(obj, key) + }); + let name = crate::string::header_str_checked(key)?.as_bytes(); + let Some(recv) = ordinary_receiver(obj as usize) else { + return Some(value); + }; + let Some(w) = walk(recv, name) else { + refuse(); + return Some(value); + }; + // Confirm: what the shapes say must be what the getter returned. + let bits = value.bits(); + let confirmed = match w.slot { + None => bits == crate::value::TAG_UNDEFINED, + Some(s) => bits == slot_bits(w.holder, s) && bits != crate::value::TAG_HOLE, + }; + if !confirmed { + refuse(); + return Some(value); + } + let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); + if cache.is_null() { + return Some(value); + } + publish(cache, recv, &w); + Some(value) +} + +unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk) { + let c = &mut *cache; + c[HOLDER_RECV] = 0; + c[HOLDER_OBJ] = w.holder as i64; + c[HOLDER_SHAPE] = (u64::from(w.holder_shape) | u64::from(w.hops[2].1) << 32) as i64; + c[HOLDER_KIND] = match (w.depth, w.slot) { + (1, Some(s)) => i64::from(s), + (1, None) => HOLDER_ABSENT_DEPTH1, + (d, s) => { + (HOLDER_STUB + | if s.is_none() { HOLDER_ABSENT_BIT } else { 0 } + | (d as u64) << HOLDER_DEPTH_SHIFT + | u64::from(s.unwrap_or(0))) as i64 + } + }; + for i in 0..HOLDER_MAX_DEPTH - 1 { + c[HOLDER_HOPS + i] = w.hops[i].0 as i64; + } + c[HOLDER_HOP_SHAPES] = (u64::from(w.hops[0].1) | u64::from(w.hops[1].1) << 32) as i64; + if c[HOLDER_REGISTERED] == 0 { + c[HOLDER_REGISTERED] = 1; + if let Ok(mut sites) = HOLDER_SITES.lock() { + sites.push(cache as usize); + } + } + // Last: the entry is live only once every other word is written. + c[HOLDER_RECV] = (u64::from(object_shape_stamp(recv)) | PIC_ID_TOKEN_BIT) as i64; + if w.slot.is_some() { + PRIMES_HOLDER.fetch_add(1, Ordering::Relaxed); + } else { + PRIMES_ABSENT.fetch_add(1, Ordering::Relaxed); + } + super::stats_report_enabled(); +} + +/// The emitted form's call for an entry it does not answer inline (depth 2..4, +/// or absent past depth 1). A GC leaf: it reads site words and object words +/// only. `TAG_HOLE` declines, and the caller continues to the miss call. +/// +/// # Safety +/// `recv` is the receiver the emitted tower validated as a plain object; +/// `cache` the site's resolved `PicCache`. +#[no_mangle] +pub unsafe extern "C" fn js_read_site_holder_hit(recv: *const u8, cache: *const PicCache) -> f64 { + let hole = f64::from_bits(crate::value::TAG_HOLE); + if cache.is_null() || recv.is_null() { + return hole; + } + let c = &*cache; + let token = (u64::from(std::ptr::read((recv as *const u32).add(1))) | PIC_ID_TOKEN_BIT) as i64; + if c[HOLDER_RECV] != token { + return hole; + } + let kind = c[HOLDER_KIND]; + let (depth, absent, slot) = if kind >= 0 { + (1, kind == HOLDER_ABSENT_DEPTH1, kind as u32) + } else { + let k = kind as u64; + ( + ((k >> HOLDER_DEPTH_SHIFT) & 0xF) as usize, + k & HOLDER_ABSENT_BIT != 0, + k as u32, + ) + }; + let hop_shapes = c[HOLDER_HOP_SHAPES] as u64; + let shape_words = [ + hop_shapes as u32, + (hop_shapes >> 32) as u32, + (c[HOLDER_SHAPE] as u64 >> 32) as u32, + ]; + for i in 0..depth.saturating_sub(1).min(HOLDER_MAX_DEPTH - 1) { + if shape_word(c[HOLDER_HOPS + i] as usize) != shape_words[i] { + return hole; + } + } + let holder = c[HOLDER_OBJ] as usize; + if shape_word(holder) != c[HOLDER_SHAPE] as u32 { + return hole; + } + if absent { + return f64::from_bits(crate::value::TAG_UNDEFINED); + } + f64::from_bits(slot_bits(holder, slot)) +} + +/// Root scan: every live entry's holder and hops are marked and rewritten. +pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + let Ok(sites) = HOLDER_SITES.lock() else { + return; + }; + for &site in sites.iter() { + // SAFETY: registered caches are arena allocations that are never freed. + let c = unsafe { &mut *(site as *mut PicCache) }; + if c[HOLDER_RECV] == 0 { + continue; + } + visitor.visit_i64_slot(&mut c[HOLDER_OBJ]); + for i in 0..HOLDER_MAX_DEPTH - 1 { + visitor.visit_i64_slot(&mut c[HOLDER_HOPS + i]); + } + } +} + +/// The first worker agent's start: a holder entry names a primary-heap object, +/// so every entry is emptied, and none is primed again (`WORKER_AGENTS_EXIST`). +pub(crate) fn empty_read_holder_entries() { + let Ok(sites) = HOLDER_SITES.lock() else { + return; + }; + for &site in sites.iter() { + // SAFETY: as in `scan_read_holder_roots_mut`; one aligned word store. + unsafe { + std::ptr::write_volatile(&mut (*(site as *mut PicCache))[HOLDER_RECV], 0); + } + } +} diff --git a/crates/perry-runtime/src/object/proto_validity.rs b/crates/perry-runtime/src/object/proto_validity.rs index 21c57db274..2848fb72d2 100644 --- a/crates/perry-runtime/src/object/proto_validity.rs +++ b/crates/perry-runtime/src/object/proto_validity.rs @@ -239,11 +239,25 @@ pub(crate) const NULL_PROTOTYPE_SERIAL: u64 = u64::MAX; /// # Safety /// As [`mark_object_as_prototype`]: allocates, and may move the owner. pub(crate) unsafe fn mark_exotic_read_receiver(obj: usize) { - if let Some(meta) = + if obj == 0 || !crate::value::addr_class::is_plausible_heap_addr(obj) { + return; + } + let scope = crate::gc::RuntimeHandleScope::new(); + let handle = scope.root_raw_mut_ptr(obj as *mut crate::object::ObjectHeader); + let (meta, object) = handle.across_mut::(|| { ensure_meta_for_mark(obj, crate::object::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER) - { + }); + if let Some(meta) = meta { // GC_STORE_AUDIT(POINTER_FREE): scalar classification bit. (*meta).flags |= crate::object::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER; + // The flag makes the receiver's [[Prototype]] identity its own + // (`shapes::object_proto_id`), and the identity is part of the shape: + // move it to a shape that says so. That makes "its reads are not + // answered by its shape" a SHAPE fact for a memo keyed on the + // receiver's ShapeId alone (`method_site::read_holder`). + let _ = handle.across_mut::(|| { + crate::object::shapes::restamp_object_proto_id(object) + }); } } diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 341842987c..a1b6fd95b6 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -3059,7 +3059,7 @@ pub(crate) const PROTO_ID_DEFAULT: u64 = 0; /// A null [[Prototype]]. pub(crate) const PROTO_ID_NULL: u64 = u64::MAX; const PROTO_ID_TAG_SHIFT: u32 = 62; -const PROTO_ID_CLASS: u64 = 1 << PROTO_ID_TAG_SHIFT; +pub(crate) const PROTO_ID_CLASS: u64 = 1 << PROTO_ID_TAG_SHIFT; const PROTO_ID_MIXED: u64 = 2 << PROTO_ID_TAG_SHIFT; const PROTO_ID_UNIQUE: u64 = 3 << PROTO_ID_TAG_SHIFT; /// The prototype identity of a shape that answers nothing about its receiver @@ -3141,6 +3141,13 @@ pub(crate) unsafe fn object_proto_id(obj: *const crate::object::ObjectHeader) -> let class_id = (*obj).class_id; let class = vtable_class(class_id); let meta = (*obj).meta; + // An exotic read receiver (`process.env`, `arguments`) is answered by no + // shape: its identity is its own, so every lineage it mints keeps it and + // no shape-keyed memo admits it (`proto_validity::mark_exotic_read_receiver`). + if !meta.is_null() && (*meta).flags & crate::object::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER != 0 + { + return PROTO_ID_PER_OBJECT; + } if !meta.is_null() && (*meta).prototype != 0 { let bits = (*meta).prototype; if bits == crate::value::TAG_NULL { diff --git a/crates/perry/tests/read_holder_entry.rs b/crates/perry/tests/read_holder_entry.rs new file mode 100644 index 0000000000..ccb75b7ff4 --- /dev/null +++ b/crates/perry/tests/read_holder_entry.rs @@ -0,0 +1,255 @@ +//! The read site's holder entry (`object::method_site::read_holder`): a key +//! that is not own on the receiver is answered from facts of the receiver's +//! and the holder's shapes, compared on every use. +//! +//! One program, run at two trip counts taken from argv (a fixed small loop is +//! unrolled and a literal receiver refined, so neither would reach the site), +//! changes the chain MID-loop in every way the entry must see: a shadowing own +//! key, a value store to the holder, a key added to and deleted from the +//! holder, a getter defined on it, `setPrototypeOf` on the receiver and on an +//! intermediate hop, a key added to a hop at depth 2 and 3, `F.prototype` +//! replaced per iteration, a key added to `Object.prototype` under an absent +//! entry, a collection that moves a holder that was young at prime time, the +//! exotic receivers (`process.env`, `arguments`) and a class accessor. Each +//! line prints the value read and the value node prints; the entry must have +//! primed (data and absent), or the program proves nothing about it. + +use std::path::PathBuf; +use std::process::Command; + +const SOURCE: &str = r#"// Runtime trip counts (argv), every chain change MID-loop, parameter +// receivers: a fixed small loop would be unrolled and the receiver refined. +const N = Number(process.argv[2] ?? "40"); +const H = N >> 1; +const lines: string[] = []; +function check(name: string, got: number, want: number): void { + lines.push(name + " " + got + " " + want + (got === want ? " ok" : " FAIL")); +} +function num(v: any, missing: number): number { + return v === undefined ? missing : v; +} + +// 1. shadow: an own key added to the receiver +function t1(o: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { if (i === H) o.k = 100; s += o.k; } + return s; +} +check("shadow", t1(Object.create({ k: 1 }), N), H + (N - H) * 100); + +// 2. a value store to the holder's slot +function t2(o: any, p: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { if (i === H) p.k = 5; s += o.k; } + return s; +} +const P2: any = { k: 1 }; +check("holder-value", t2(Object.create(P2), P2, N), H + (N - H) * 5); + +// 3. another key added to the holder, then a value store +function t3(o: any, p: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { + if (i === H) p.other = 9; + if (i === H + 1) p.k = 7; + s += o.k; + } + return s; +} +const P3: any = { k: 1 }; +check("holder-restamp", t3(Object.create(P3), P3, N), H + 1 + (N - H - 1) * 7); + +// 4. delete from the holder; a getter defined on the holder +function t4(o: any, p: any, n: number, mode: number): number { + let s = 0; + for (let i = 0; i < n; i++) { + if (i === H) { + if (mode === 0) delete p.k; + else Object.defineProperty(p, "k", { get() { return 50; }, configurable: true }); + } + s += num(o.k, 1000); + } + return s; +} +const P4a: any = { k: 1, j: 2 }; +check("holder-delete", t4(Object.create(P4a), P4a, N, 0), H + (N - H) * 1000); +const P4b: any = { k: 1 }; +check("holder-getter", t4(Object.create(P4b), P4b, N, 1), H + (N - H) * 50); + +// 5. setPrototypeOf on the receiver; on the intermediate hop (depth 2) +function t5(o: any, target: any, to: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { if (i === H) Object.setPrototypeOf(target, to); s += num(o.k, 1000); } + return s; +} +const o5a = Object.create({ k: 1 }); +check("recv-setproto", t5(o5a, o5a, { k: 3 }, N), H + (N - H) * 3); +const mid5 = Object.create({ k: 1 }); +check("hop-setproto", t5(Object.create(mid5), mid5, { k: 4 }, N), H + (N - H) * 4); + +// 6. depth 3: the key added to an intermediate hop shadows the holder's +function t6(o: any, hop: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { if (i === H) hop.k = 6; s += o.k; } + return s; +} +const C6: any = { k: 1 }; +const B6 = Object.create(C6); +const A6 = Object.create(B6); +check("depth3-hop", t6(Object.create(A6), B6, N), H + (N - H) * 6); +const C6b: any = { k: 1 }; +const A6b = Object.create(C6b); +check("depth2-hop", t6(Object.create(A6b), A6b, N), H + (N - H) * 6); + +// 7. F.prototype replaced every iteration +function F7(this: any) {} +function read7(o: any): number { return o.k; } +function t7(n: number): number { + const PA = { k: 1 }, PB = { k: 2 }; + let s = 0; + for (let i = 0; i < n; i++) { + (F7 as any).prototype = (i & 1) ? PB : PA; + s += read7(new (F7 as any)()); + } + return s; +} +check("ctor-prototype", t7(N), (N - (N >> 1)) * 1 + (N >> 1) * 2); + +// 8. absent: a key added to Object.prototype mid-loop (depth 1: a literal; +// depth 2: an Object.create child) +function t8(o: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { + if (i === H) (Object.prototype as any).zz8 = 8; + s += num(o.zz8, 0); + } + delete (Object.prototype as any).zz8; + return s; +} +check("absent-d1", t8({ a: 1 }, N), (N - H) * 8); +check("absent-d2", t8(Object.create({ a: 1 }), N), (N - H) * 8); +function t8b(o: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) s += num(o.zz, 1); + return s; +} +check("absent-steady", t8b({ a: 1, b: 2 }, N), N); + +// 9. GC moves the holder: it is young at prime time +function t9(n: number): number { + const p: any = { k: 1 }; + const o = Object.create(p); + let s = 0; + let keep: any[] = []; + for (let i = 0; i < n; i++) { + if (i === H) { + for (let j = 0; j < 20000; j++) keep.push({ j }); + (globalThis as any).gc(); + keep = []; + } + if (i === H + 1) p.k = 11; + s += o.k; + } + return s; +} +check("gc-moves-holder", t9(N), H + 1 + (N - H - 1) * 11); + +// Exotic receivers: process.env, arguments +function t10(e: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { + if (i === H) e.PERRY_HOLDER_TEST_KEY = "v"; + s += e.PERRY_HOLDER_TEST_KEY === undefined ? 0 : 1; + } + return s; +} +check("exotic-env", t10(process.env, N), N - H); +function t11(n: number, ...rest: any[]): number { + let s = 0; + const a: any = (function (this: any) { return arguments; })(1, 2); + for (let i = 0; i < n; i++) { + if (i === H) (Object.prototype as any).zz11 = 3; + s += num(a.zz11, 0); + } + delete (Object.prototype as any).zz11; + return s; +} +check("exotic-arguments", t11(N), (N - H) * 3); + +// Class accessor: never answered as a slot +class C12 { n = 1; get k(): number { return this.n * 2; } } +function t12(o: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { if (i === H) o.n = 5; s += o.k; } + return s; +} +check("class-accessor", t12(new C12(), N), H * 2 + (N - H) * 10); + +console.log(lines.join("\n")); +"#; + +fn run(n: &str) -> (String, String) { + let dir = tempfile::tempdir().expect("tempdir"); + let entry = dir.path().join("main.ts"); + let output = dir.path().join("main_bin"); + std::fs::write(&entry, SOURCE).expect("write entry"); + let compile = Command::new(PathBuf::from(env!("CARGO_BIN_EXE_perry"))) + .current_dir(dir.path()) + .arg("compile") + .arg(&entry) + .arg("-o") + .arg(&output) + .env("PERRY_NO_CACHE", "1") + .output() + .expect("run perry compile"); + assert!( + compile.status.success(), + "perry compile failed\nstderr:\n{}", + String::from_utf8_lossy(&compile.stderr) + ); + let run = Command::new(&output) + .arg(n) + .current_dir(dir.path()) + .env("PERRY_METHOD_SITE_STATS", "1") + .env("PERRY_GC_FORCE_EVACUATE", "1") + .env("PERRY_GC_POISON_FROMSPACE", "1") + .output() + .expect("run compiled binary"); + let stderr = String::from_utf8_lossy(&run.stderr).into_owned(); + assert!( + run.status.success(), + "binary failed ({:?})\nstderr:\n{stderr}", + run.status + ); + (String::from_utf8_lossy(&run.stdout).into_owned(), stderr) +} + +fn stat(stderr: &str, name: &str) -> u64 { + let line = stderr + .lines() + .find(|l| l.starts_with("[method-site]")) + .unwrap_or_else(|| panic!("no [method-site] line in:\n{stderr}")); + line.split_whitespace() + .find_map(|w| w.strip_prefix(name).and_then(|v| v.strip_prefix('='))) + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| panic!("no {name} in {line}")) +} + +#[test] +fn holder_entry_follows_every_chain_change() { + for n in ["40", "41"] { + let (stdout, stderr) = run(n); + let lines: Vec<&str> = stdout.lines().collect(); + assert_eq!(lines.len(), 17, "n={n}: expected 17 checks:\n{stdout}"); + let failed: Vec<&&str> = lines.iter().filter(|l| !l.ends_with(" ok")).collect(); + assert!(failed.is_empty(), "n={n}: wrong reads {failed:?}\n{stdout}"); + assert!( + stat(&stderr, "read_holder_primes") > 0, + "n={n}: no data entry primed\n{stderr}" + ); + assert!( + stat(&stderr, "read_absent_primes") > 0, + "n={n}: no absent entry primed\n{stderr}" + ); + } +} diff --git a/scripts/gc_root_dominance_check.py b/scripts/gc_root_dominance_check.py index 21db88dec4..a9624e2d14 100755 --- a/scripts/gc_root_dominance_check.py +++ b/scripts/gc_root_dominance_check.py @@ -485,6 +485,10 @@ def build_cfg(f): # per-thread table probe plus one load through the holder; no allocation, # no user code, no chain walk (declines answer TAG_HOLE). "js_inherited_read_cache_hit_f64", + # object/method_site/read_holder.rs `js_read_site_holder_hit`: compares the + # site-held hop and holder ShapeIds and loads one slot; no allocation, no + # user code (declines answer TAG_HOLE). + "js_read_site_holder_hit", # S2 GC-leaf IC hits (`expr/ic_fast_split.rs`); audit in gc_call_effects.rs. "js_object_get_field_ic_fast", "js_class_field_get_ic_fast", From 9183bca091922472642e0cfa46f85a0e24c1d329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 15:50:58 +0000 Subject: [PATCH 2/5] perf(runtime): the holder entry is asked first, refused sites keep the inherited-read hook - The class-field read miss arm (`this.k` in a literal method) asks the site holder entry after the MRU word, and a miss the live entry answers primes nothing: the `this` cells re-primed on every read (292 -> 4383). - The inherited-read hook is kept on the never-primed edge and on a miss at a site whose ways never primed, for receivers the entry does not describe (a compiled-class instance: absent 334 -> 434). - A site that refused once, or was re-primed for four receiver shapes, is latched in its cache state word and never walks or primes again. - The emitted tower asks the holder entry first on the MRU miss edge, before the way state and the ways; the absent kind is a bit test and the holder load has no TAG_HOLE compare (a delete is a shape transition, #10826). - Codegen tests follow the new block graph. - gc_runtime_root_holders: PASS1_MARKED re-audited and re-pinned for the new reg_scanner!; thread_exit_address_globals: HOLDER_SITES holds PIC-arena addresses (process_global_allocation). --- changelog.d/inherited-read-holder-entry.md | 9 + .../src/expr/property_get/generic_dispatch.rs | 125 +++++++---- .../src/expr/property_get/tests.rs | 149 +++++++++----- .../src/object/method_site/read_holder.rs | 194 +++++++++++++----- .../src/typed_feedback/guards.rs | 14 +- scripts/gc_runtime_root_holders.json | 4 +- scripts/thread_exit_address_globals.json | 8 + 7 files changed, 366 insertions(+), 137 deletions(-) diff --git a/changelog.d/inherited-read-holder-entry.md b/changelog.d/inherited-read-holder-entry.md index c4d980de41..a9b492ca40 100644 --- a/changelog.d/inherited-read-holder-entry.md +++ b/changelog.d/inherited-read-holder-entry.md @@ -10,3 +10,12 @@ redefined on any object on the chain, or a `setPrototypeOf`, moves a ShapeId the entry compares; a value store is seen because the slot is loaded. No global validity word is involved. `process.env` and `arguments` now carry a per-object prototype identity in their shape, so no shape-keyed memo admits them. + +The entry is asked first where the site own-slot word misses, before the +polymorphic ways. A receiver it does not describe (a compiled-class instance, +a second receiver shape) keeps the inherited-read hook it had before, on the +never-primed edge and on a miss at a site whose ways never primed. A miss the +live entry answers primes nothing (the class-field read miss arm asks the +entry directly), and a site that refused once, or was re-primed for four +different receiver shapes, stops priming, so its misses do not walk the chain +and run the getter again. diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 30c15a256b..84905e0461 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -1021,8 +1021,20 @@ pub(crate) fn lower_generic_property_get( // signal byte-identical and go without the hook. ctx.current_block = ways_entry_idx; let token_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); + // Without typed feedback the site's cache is asked, in order: the holder + // entry (`pic.holder`, first — the one answer a non-own key can have), + // then the ways; a site with no cache yet goes to the inherited-read hook. + let holder_on = !crate::expr::typed_feedback_emission_enabled(); + let inherited_idx = holder_on.then(|| ctx.new_block("pic.miss.inherited")); + let holder_idx = holder_on.then(|| ctx.new_block("pic.holder")); + let never_primed_label = inherited_idx + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| cold_label.clone()); + let present_label = holder_idx + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| miss_label.clone()); ctx.block() - .cond_br(&token_cache.present, &miss_label, &cold_label); + .cond_br(&token_cache.present, &present_label, &never_primed_label); // `js_object_get_field_ic_miss` primes only slots below the descriptor's // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact @@ -1174,9 +1186,22 @@ pub(crate) fn lower_generic_property_get( let ways_live = ctx.block().icmp_sgt(I64, &way_state, "0"); let ways_idx = ctx.new_block("pic.ways"); let ways_label = ctx.block_label(ways_idx); - let holder_idx = ctx.new_block("pic.holder"); - let holder_label = ctx.block_label(holder_idx); - ctx.block().cond_br(&ways_live, &ways_label, &holder_label); + // A site whose ways never primed (state 0) has only ever seen receivers + // the MRU word or the holder entry named, or none it could learn: its + // miss asks the inherited-read hook first, as a never-primed site does. + // A megamorphic site (-1) goes straight to the call. + match inherited_idx { + Some(inh_idx) => { + let fresh_idx = ctx.new_block("pic.ways.fresh"); + let fresh_label = ctx.block_label(fresh_idx); + ctx.block().cond_br(&ways_live, &ways_label, &fresh_label); + ctx.current_block = fresh_idx; + let fresh = ctx.block().icmp_eq(I64, &way_state, "0"); + let inh_label = ctx.block_label(inh_idx); + ctx.block().cond_br(&fresh, &inh_label, &call_label); + } + None => ctx.block().cond_br(&ways_live, &ways_label, &call_label), + } ctx.current_block = ways_idx; // `is_object` is not ANDed in any more: it is statically true on every edge @@ -1226,8 +1251,7 @@ pub(crate) fn lower_generic_property_get( .expect("PIC_WAYS is non-zero, so the reduction leaves exactly one lane"); let way_load_idx = ctx.new_block("pic.way.load"); let way_load_label = ctx.block_label(way_load_idx); - ctx.block() - .cond_br(&way_any, &way_load_label, &holder_label); + ctx.block().cond_br(&way_any, &way_load_label, &call_label); ctx.current_block = way_load_idx; if fused_recv.is_some() { @@ -1286,36 +1310,37 @@ pub(crate) fn lower_generic_property_get( } // The read site's HOLDER entry (`object::method_site::read_holder` in the - // runtime), on the two edges where the MRU word and the ways have missed: - // the answer for a key that is NOT own on the receiver, as facts of two - // shapes. The receiver's ShapeId says the key is not own and which object - // is its [[Prototype]]; the holder's ShapeId says the key is an own inline - // data slot there (or, for an ABSENT entry, that the terminal object lacks - // it). Both are compared here, the holder is a strong root in the site, and - // the value is LOADED, so no global word and no invalidation exist. A - // stable tombstone (#9064) can clear the holder's slot without moving its - // ShapeId, so a loaded `TAG_HOLE` goes to the call. + // runtime), asked first on the MRU word's miss edge: the answer for a key + // that is NOT own on the receiver, as facts of two shapes. The receiver's + // ShapeId says the key is not own and which object is its [[Prototype]]; + // the holder's ShapeId says the key is an own inline data slot there (or, + // for an ABSENT entry, that the terminal object lacks it). Both are + // compared here, the holder is a strong root in the site, and the value is + // LOADED, so no global word and no invalidation exist. A delete is a shape + // transition (#10826), so a matching holder ShapeId proves the slot live: + // no `TAG_HOLE` test, as the MRU hit has none. // - // [cache + RECV] == token else call + // [cache + RECV] == token else the ways // kind = [cache + KIND] ; kind stub (depth 2..4, deep absent) // h = [cache + OBJ] ; [h + 4] == low32([cache + SHAPE]) else call - // kind == ABSENT_DEPTH1 -> undefined - // v = [h + HDR + 8*kind] ; v != TAG_HOLE else call + // kind & ABSENT_DEPTH1 -> undefined + // v = [h + HDR + 8*kind] // - // A site that never primed has no cache and so no entry; its edge goes - // straight to the call, which primes (`get_field_ic_miss_impl`). - let holder_arm: Vec<(String, String)> = { + // A site that never primed has no cache and so no entry; its edge goes to + // the inherited-read hook, then the call, which primes + // (`get_field_ic_miss_impl`). + let mut holder_arm: Vec<(String, String)> = Vec::with_capacity(4); + if let Some(holder_idx) = holder_idx { let word = |ctx: &mut FnCtx<'_>, w: usize| { let p = ctx.block().gep(I64, &cache_ref, &[(I64, &w.to_string())]); ctx.block().load(I64, &p) }; - let mut arms = Vec::with_capacity(3); ctx.current_block = holder_idx; let recv_word = word(ctx, crate::runtime_abi::PIC_HOLDER_RECV_WORD); let recv_eq = ctx.block().icmp_eq(I64, &recv_word, &token); let kind_idx = ctx.new_block("pic.holder.kind"); let kind_label = ctx.block_label(kind_idx); - ctx.block().cond_br(&recv_eq, &kind_label, &call_label); + ctx.block().cond_br(&recv_eq, &kind_label, &miss_label); ctx.current_block = kind_idx; let kind = word(ctx, crate::runtime_abi::PIC_HOLDER_KIND_WORD); @@ -1338,12 +1363,15 @@ pub(crate) fn lower_generic_property_get( let answer_label = ctx.block_label(answer_idx); ctx.block().cond_br(&holder_eq, &answer_label, &call_label); + // The absent kind is one bit no slot index has: a bit test, not a + // compare against a 64-bit immediate. ctx.current_block = answer_idx; - let absent = ctx.block().icmp_eq( + let absent_bit = ctx.block().and( I64, &kind, &crate::runtime_abi::PIC_HOLDER_ABSENT_DEPTH1.to_string(), ); + let absent = ctx.block().icmp_ne(I64, &absent_bit, "0"); let absent_idx = ctx.new_block("pic.holder.absent"); let absent_label = ctx.block_label(absent_idx); let load_idx = ctx.new_block("pic.holder.load"); @@ -1356,19 +1384,16 @@ pub(crate) fn lower_generic_property_get( .bitcast_i64_to_double(crate::nanbox::TAG_UNDEFINED_I64); let absent_end = ctx.block().label.clone(); ctx.block().br(&merge_label); - arms.push((undef, absent_end)); + holder_arm.push((undef, absent_end)); ctx.current_block = load_idx; - let offset = ctx.block().shl(I64, &kind, "3"); let base = ctx.block().add(I64, &holder, &obj_header_size); - let field_addr = ctx.block().add(I64, &base, &offset); - let field_ptr = ctx.block().inttoptr(I64, &field_addr); + let base_ptr = ctx.block().inttoptr(I64, &base); + let field_ptr = ctx.block().gep(DOUBLE, &base_ptr, &[(I64, &kind)]); let val = ctx.block().load(DOUBLE, &field_ptr); - let bits = ctx.block().bitcast_double_to_i64(&val); - let live = ctx.block().icmp_ne(I64, &bits, crate::nanbox::TAG_HOLE_I64); let load_end = ctx.block().label.clone(); - ctx.block().cond_br(&live, &merge_label, &call_label); - arms.push((val, load_end)); + ctx.block().br(&merge_label); + holder_arm.push((val, load_end)); // Depth 2..4 and a deep absent entry: the hop words are compared by a // GC-leaf stub (no allocation, no collection, no user code — a leaf in @@ -1387,9 +1412,39 @@ pub(crate) fn lower_generic_property_get( .icmp_ne(I64, &stub_bits, crate::nanbox::TAG_HOLE_I64); let stub_end = ctx.block().label.clone(); ctx.block().cond_br(&served, &merge_label, &call_label); - arms.push((stub_val, stub_end)); - arms - }; + holder_arm.push((stub_val, stub_end)); + } + + // The inherited-read hook, on the never-primed edge and on a fresh-ways + // miss: a receiver whose non-own key the holder entry does not describe + // (a compiled-class instance, a second receiver shape, a latched site) is + // answered by `js_inherited_read_cache_hit_f64` as before the entry + // existed. It is a pure state read — it allocates nothing, triggers no GC + // and runs no user code — so it is a leaf in `gc_call_effects.rs` and + // `root_reload.rs`. `TAG_HOLE` is its decline sentinel; a decline goes to + // the call. The versioned-loop deopt note is emitted here as it is on the + // exit, so entering either cold arm still records the bailout. + if let Some(idx) = inherited_idx { + ctx.current_block = idx; + crate::expr::emit_versioned_loop_callback_deopt(ctx); + let inh_key_handle = emit_key_handle(ctx, &key_handle_global); + let handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); + let recv_ptr = ctx.block().inttoptr(I64, &handle); + let key_ptr = ctx.block().inttoptr(I64, &inh_key_handle); + let val_inherited = ctx.block().call( + DOUBLE, + "js_inherited_read_cache_hit_f64", + &[(PTR, &recv_ptr), (PTR, &key_ptr)], + ); + let inherited_bits = ctx.block().bitcast_double_to_i64(&val_inherited); + let inherited_served = + ctx.block() + .icmp_ne(I64, &inherited_bits, crate::nanbox::TAG_HOLE_I64); + let inherited_end_label = ctx.block().label.clone(); + ctx.block() + .cond_br(&inherited_served, &merge_label, &call_label); + holder_arm.push((val_inherited, inherited_end_label)); + } // The object exit: one call reproducing every pointer-path arm this tower // used to expand. diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 2d101d86c0..9b6ef0bb90 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -552,7 +552,8 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { ); // T1: the landing block is now the single slow exit itself, and the // dominance is structural — `pic.miss` has exactly ONE predecessor, - // `pic.token.miss`, which `pic.token` dominates. Assert that directly: + // `pic.holder` (whose only predecessor is `pic.token.ways`, reached from + // `pic.token.miss`), which `pic.token` dominates. Assert that directly: // routing any receiver-validation failure back into `pic.miss` would add a // predecessor and immediately re-introduce the phis #7907 removed. // `pic.miss` carries a numeric suffix and `pic.miss.call` starts with the @@ -580,7 +581,7 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { .count(); assert_eq!( preds, 1, - "pic.miss must have exactly one predecessor (pic.token.miss), or it is \ + "pic.miss must have exactly one predecessor (pic.holder), or it is \ no longer dominated by pic.token:\n{ir}" ); assert!( @@ -597,14 +598,16 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { "the small-handle sentinel select only existed because an invalid \ receiver could reach the way compares; it must be gone:\n{ir}" ); - // The receiver predicates, exactly once each. `icmp eq i32 %` is two: the - // ShapeId identity compare on the hit path and the spill compare in - // `pic.token.miss` that replaced the hit path's overflow-bit test. There - // is no GC-kind compare at all any more (#10828), so a single `icmp eq - // i8` would mean the header load has crept back somewhere. + // The receiver predicates, exactly once each. `icmp eq i32 %` is three: + // the ShapeId identity compare on the hit path, the spill compare in + // `pic.token.miss` that replaced the hit path's overflow-bit test, and the + // HOLDER's ShapeId compare in `pic.holder.inline` (an object other than + // the receiver, so not a re-derivation). There is no GC-kind compare at + // all any more (#10828), so a single `icmp eq i8` would mean the header + // load has crept back somewhere. for (needle, what, expect) in [ ("icmp eq i8 ", "the GC_TYPE_OBJECT compare", 0), - ("icmp eq i32 %", "the ShapeId identity compare", 2), + ("icmp eq i32 %", "the ShapeId identity compare", 3), ] { let n = main.matches(needle).count(); assert_eq!( @@ -1617,10 +1620,14 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { // `pic.way.live` is GONE with the way path's `TAG_HOLE` compare: the // load block has nothing left to decide and branches to the merge. "pic.way.load", - // the holder entry (`method_site::read_holder`), past the ways: the - // receiver word, the kind, the inline depth-1 compare and load (or - // `undefined` for an absent entry), and the GC-leaf stub for depth - // 2..4; every decline continues to the one exit + // a site whose ways never primed asks the inherited-read hook before + // the call, as a never-primed site does + "pic.ways.fresh", + "pic.miss.inherited", + // the holder entry (`method_site::read_holder`), first on the MRU + // miss edge: the receiver word, the kind, the inline depth-1 compare + // and load (or `undefined` for an absent entry), and the GC-leaf stub + // for depth 2..4 "pic.holder", "pic.holder.kind", "pic.holder.inline", @@ -1652,17 +1659,23 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { ); } -/// The read site's holder entry is checked where the MRU word and the ways -/// have missed, and nowhere else: +/// The read site's holder entry is the first answer asked where the MRU word +/// missed, and the inherited-read hook is kept for what it does not describe: /// -/// 1. the never-primed edge (`present` false) goes straight to the one exit — -/// a site with no cache has no entry; -/// 2. both edges out of the ways (`pic.miss` with no live way, `pic.ways` with -/// no matching way) go to `pic.holder`; -/// 3. the stub is called from `pic.holder.stub` only, and its `TAG_HOLE` -/// decline and the inline load's hole compare both continue to the exit. +/// 1. `pic.token.ways`: a present cache goes to `pic.holder`; the never-primed +/// edge goes to the inherited-read hook (`pic.miss.inherited`); +/// 2. `pic.holder`: a receiver word that is not the entry's goes on to the +/// ways (`pic.miss`), never straight to the exit; +/// 3. `pic.miss`: live ways go to `pic.ways`, otherwise `pic.ways.fresh`, +/// which asks the hook only for a site whose ways never primed (a +/// megamorphic site goes to the call); +/// 4. the stub is called from `pic.holder.stub` only, declines on `TAG_HOLE` +/// to the exit; the inline load has no hole compare (a delete is a shape +/// transition, #10826) and goes to the merge only; +/// 5. the hook calls `js_inherited_read_cache_hit_f64` and declines to the +/// exit. #[test] -fn the_holder_entry_is_checked_past_the_ways_only() { +fn the_holder_entry_is_asked_first_and_the_hook_is_kept() { let ir = emit(false, None); let func = ir .split("\ndefine ") @@ -1676,8 +1689,8 @@ fn the_holder_entry_is_checked_past_the_ways_only() { body.push(line.trim().to_string()); } } - let term = |prefix: &str| -> String { - let (_, body) = blocks + let body = |prefix: &str| -> Vec { + blocks .iter() .find(|(l, _)| { l.strip_prefix(prefix).is_some_and(|r| { @@ -1685,32 +1698,60 @@ fn the_holder_entry_is_checked_past_the_ways_only() { .is_some_and(|n| n.parse::().is_ok()) }) }) - .unwrap_or_else(|| panic!("no block {prefix}:\n{func}")); - body.iter() + .unwrap_or_else(|| panic!("no block {prefix}:\n{func}")) + .1 + .clone() + }; + let term = |prefix: &str| -> String { + body(prefix) + .iter() .rev() .find(|l| l.starts_with("br ")) .unwrap() .clone() }; + let targets = |t: &str| -> Vec { + t.split("label %") + .skip(1) + .map(|x| { + let x = x.trim_end_matches(&[',', ' '][..]); + let mut parts: Vec<&str> = x.split('.').collect(); + if parts.last().is_some_and(|p| p.parse::().is_ok()) { + parts.pop(); + } + parts.join(".") + }) + .collect() + }; // 1. - let ways_entry = term("pic.token.ways"); - assert!( - ways_entry.contains("label %pic.miss.") && ways_entry.contains("label %pic.miss.call"), - "the never-primed edge goes straight to the exit: {ways_entry}" + assert_eq!( + targets(&term("pic.token.ways")), + ["pic.holder", "pic.miss.inherited"], + "{func}" ); // 2. - for b in ["pic.miss", "pic.ways"] { - let t = term(b); - assert!( - t.contains("label %pic.holder."), - "{b} must fall to the holder entry: {t}" - ); - assert!( - !t.contains("label %pic.miss.call"), - "{b} must not skip the holder entry: {t}" - ); - } + assert_eq!( + targets(&term("pic.holder")), + ["pic.holder.kind", "pic.miss"], + "{func}" + ); // 3. + assert_eq!( + targets(&term("pic.miss")), + ["pic.ways", "pic.ways.fresh"], + "{func}" + ); + assert_eq!( + targets(&term("pic.ways.fresh")), + ["pic.miss.inherited", "pic.miss.call"], + "{func}" + ); + assert_eq!( + targets(&term("pic.ways")), + ["pic.way.load", "pic.miss.call"], + "{func}" + ); + // 4. let callers: Vec<&str> = blocks .iter() .filter(|(_, body)| body.iter().any(|l| l.contains("@js_read_site_holder_hit("))) @@ -1720,19 +1761,33 @@ fn the_holder_entry_is_checked_past_the_ways_only() { callers.len() == 1 && callers[0].starts_with("pic.holder.stub"), "the stub is called from pic.holder.stub only: {callers:?}" ); - for b in ["pic.holder.stub", "pic.holder.load"] { - let (_, body) = blocks.iter().find(|(l, _)| l.starts_with(b)).unwrap(); + for b in ["pic.holder.stub", "pic.miss.inherited"] { assert!( - body.iter() + body(b) + .iter() .any(|l| l.contains("icmp ne i64 ") && l.ends_with(crate::nanbox::TAG_HOLE_I64)), "{b} must decline on TAG_HOLE:\n{func}" ); - let t = term(b); - assert!( - t.contains("label %pget.recv_merge") && t.contains("label %pic.miss.call"), - "{b}: served to the merge, declined to the exit: {t}" + assert_eq!( + targets(&term(b)), + ["pget.recv_merge", "pic.miss.call"], + "{b}: served to the merge, declined to the exit" ); } + assert!( + !body("pic.holder.load") + .iter() + .any(|l| l.contains(crate::nanbox::TAG_HOLE_I64)), + "the holder load has no hole compare:\n{func}" + ); + assert_eq!(targets(&term("pic.holder.load")), ["pget.recv_merge"]); + // 5. + assert!( + body("pic.miss.inherited") + .iter() + .any(|l| l.contains("@js_inherited_read_cache_hit_f64(")), + "the hook calls the inherited-read cache:\n{func}" + ); } #[path = "array_length_tests.rs"] diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 3c56147fe0..0d405538c3 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -14,9 +14,9 @@ //! Every fact is compared on use, so there is no invalidation and no global //! word: a key add, delete, descriptor change or `setPrototypeOf` on any object //! the entry names moves that object's ShapeId, and a value store to the -//! holder's slot is seen because the hit LOADS the slot. A stable tombstone -//! (#9064) can clear the holder's slot without moving its ShapeId, so a loaded -//! `TAG_HOLE` is a miss. +//! holder's slot is seen because the hit LOADS the slot. A delete is a shape +//! transition (#10826), so a holder whose ShapeId matches still has the slot: +//! the hit needs no `TAG_HOLE` test, as the emitted MRU hit needs none. //! //! The entry lives in the read site's own cache (`PicCache` words //! [`HOLDER_RECV`]..=[`HOLDER_REGISTERED`]). The holder and the hops are @@ -37,6 +37,14 @@ //! (names the runtime synthesizes, lazily materialized intrinsics and //! `constructor` refuse there). Primary agent only; a worker agent's start //! empties every entry. +//! +//! A miss whose receiver the live entry already answers is served from the +//! entry and primes nothing (a caller that does not emit the check, such as +//! the class-field read's miss arm, reaches here on every read). A site that +//! refused once, or whose entry was replaced for a different receiver shape +//! [`MAX_REPRIMES`] times, is LATCHED in its own state word +//! ([`HOLDER_STATE`]): it never walks or primes again, and its misses take the +//! path they took before the entry existed. use super::{key_may_be_accessor, next_prototype, ordinary_receiver, WORKER_AGENTS_EXIST}; use crate::object::shapes::{ @@ -65,8 +73,17 @@ pub const HOLDER_KIND: usize = crate::codegen_abi::PIC_HOLDER_KIND_WORD; pub const HOLDER_HOPS: usize = HOLDER_KIND + 1; /// Low 32 bits: the first hop's ShapeId. High 32 bits: the second hop's. pub const HOLDER_HOP_SHAPES: usize = HOLDER_HOPS + 3; -/// Nonzero once the cache is on the root list. -pub const HOLDER_REGISTERED: usize = HOLDER_HOP_SHAPES + 1; +/// The site's holder state: [`STATE_REGISTERED`], [`STATE_LATCHED`] and the +/// count of re-primes for a different receiver shape. +pub const HOLDER_STATE: usize = HOLDER_HOP_SHAPES + 1; +/// The cache is on the root list. +const STATE_REGISTERED: i64 = 1; +/// The site refused, or is polymorphic in its non-own receivers: no walk and +/// no prime from here on. +const STATE_LATCHED: i64 = 2; +const STATE_REPRIME_SHIFT: u32 = 8; +/// Re-primes for a different receiver shape a site takes before it latches. +const MAX_REPRIMES: i64 = 4; pub const HOLDER_ABSENT_DEPTH1: i64 = crate::codegen_abi::PIC_HOLDER_ABSENT_DEPTH1; pub const HOLDER_STUB: u64 = 1 << 63; @@ -99,6 +116,83 @@ fn refuse() { REFUSED_HOLDER.fetch_add(1, Ordering::Relaxed); } +/// Refuse, and latch `cache` (when the site has one) so it never walks again. +#[inline] +unsafe fn refuse_and_latch(cache: *mut PicCache) { + refuse(); + if !cache.is_null() { + (*cache)[HOLDER_STATE] |= STATE_LATCHED; + } +} + +/// The entry's answer (value bits) for a receiver whose PIC token is +/// `token`, or `None` when the entry is empty, names another receiver shape, +/// or any hop or holder word it recorded has moved. Reads site words and +/// object words only: a GC leaf. +/// +/// The loaded slot needs no `TAG_HOLE` test, for the reason the emitted MRU +/// hit needs none: every delete is a shape transition (#10826), so a holder +/// whose ShapeId still matches has not had the slot cleared. +#[inline(always)] +unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { + if c[HOLDER_RECV] != token || token == 0 { + return None; + } + let kind = c[HOLDER_KIND]; + let (depth, absent, slot) = if kind >= 0 { + (1, kind == HOLDER_ABSENT_DEPTH1, kind as u32) + } else { + let k = kind as u64; + ( + ((k >> HOLDER_DEPTH_SHIFT) & 0xF) as usize, + k & HOLDER_ABSENT_BIT != 0, + k as u32, + ) + }; + let hop_shapes = c[HOLDER_HOP_SHAPES] as u64; + let shape_words = [ + hop_shapes as u32, + (hop_shapes >> 32) as u32, + (c[HOLDER_SHAPE] as u64 >> 32) as u32, + ]; + for i in 0..depth.saturating_sub(1).min(HOLDER_MAX_DEPTH - 1) { + if shape_word(c[HOLDER_HOPS + i] as usize) != shape_words[i] { + return None; + } + } + let holder = c[HOLDER_OBJ] as usize; + if shape_word(holder) != c[HOLDER_SHAPE] as u32 { + return None; + } + if absent { + return Some(crate::value::TAG_UNDEFINED); + } + Some(slot_bits(holder, slot)) +} + +/// The site's holder entry asked for `handle`, without priming: what the +/// emitted tower's holder check answers, for a runtime caller that asks the +/// site's words itself (`typed_feedback::guards`' class-field miss arm). +/// +/// # Safety +/// `handle` is a pointer above the handle band; `cache_slot` null or the +/// site's live read cache slot. +#[inline] +pub(crate) unsafe fn read_holder_hit( + handle: *const ObjectHeader, + cache_slot: *mut PicCacheSlot, +) -> Option { + let cache = crate::object::field_get_set::pic_slot_peek::(cache_slot); + if cache.is_null() { + return None; + } + let stamp = object_shape_stamp(handle); + if stamp == 0 { + return None; + } + entry_answer(&*cache, (u64::from(stamp) | PIC_ID_TOKEN_BIT) as i64).map(f64::from_bits) +} + #[inline] unsafe fn shape_word(addr: usize) -> u32 { object_shape_stamp(addr as *const ObjectHeader) @@ -265,13 +359,30 @@ pub(crate) unsafe fn prime_read_holder( { return None; } + // A receiver the live entry answers is served from it: nothing to prime. + // A latched site keeps the caller's path. + let existing = crate::object::field_get_set::pic_slot_peek::(cache_slot); + if !existing.is_null() { + let stamp = object_shape_stamp(obj); + if stamp != 0 { + let token = (u64::from(stamp) | PIC_ID_TOKEN_BIT) as i64; + if let Some(bits) = entry_answer(&*existing, token) { + return Some(crate::value::JSValue::from_bits(bits)); + } + } + if (*existing)[HOLDER_STATE] & STATE_LATCHED != 0 { + return None; + } + } let name = crate::string::header_str_checked(key)?.as_bytes(); let recv = ordinary_receiver(obj as usize)?; // Cheap pre-walk: a receiver the entry could never describe keeps the - // caller's path and pays nothing for the getter below. + // caller's path and pays nothing for the getter below. A site with no + // cache yet stays without one, so the emitted never-primed edge keeps its + // inherited-read hook. if !holder_name_admitted(name) || key_may_be_accessor(recv, name) || walk(recv, name).is_none() { - refuse(); + refuse_and_latch(existing); return None; } @@ -282,12 +393,16 @@ pub(crate) unsafe fn prime_read_holder( let (value, obj) = handle.across_mut::(|| { crate::object::field_get_set::get_field_by_name_past_inherited_cache(obj, key) }); + // From here a refusal has already run the getter, so the site latches: + // the next miss must not walk and run it again only to refuse again. + let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); let name = crate::string::header_str_checked(key)?.as_bytes(); let Some(recv) = ordinary_receiver(obj as usize) else { + refuse_and_latch(cache); return Some(value); }; let Some(w) = walk(recv, name) else { - refuse(); + refuse_and_latch(cache); return Some(value); }; // Confirm: what the shapes say must be what the getter returned. @@ -297,10 +412,9 @@ pub(crate) unsafe fn prime_read_holder( Some(s) => bits == slot_bits(w.holder, s) && bits != crate::value::TAG_HOLE, }; if !confirmed { - refuse(); + refuse_and_latch(cache); return Some(value); } - let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); if cache.is_null() { return Some(value); } @@ -310,6 +424,19 @@ pub(crate) unsafe fn prime_read_holder( unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk) { let c = &mut *cache; + let token = (u64::from(object_shape_stamp(recv)) | PIC_ID_TOKEN_BIT) as i64; + if c[HOLDER_RECV] != 0 && c[HOLDER_RECV] != token { + // The site's non-own receivers take more than one shape. One entry + // cannot hold them; after a few replacements the site stops priming. + let n = (c[HOLDER_STATE] >> STATE_REPRIME_SHIFT) + 1; + c[HOLDER_STATE] = (c[HOLDER_STATE] & ((1 << STATE_REPRIME_SHIFT) - 1)) + | (n << STATE_REPRIME_SHIFT) + | if n >= MAX_REPRIMES { STATE_LATCHED } else { 0 }; + if n >= MAX_REPRIMES { + refuse(); + return; + } + } c[HOLDER_RECV] = 0; c[HOLDER_OBJ] = w.holder as i64; c[HOLDER_SHAPE] = (u64::from(w.holder_shape) | u64::from(w.hops[2].1) << 32) as i64; @@ -327,14 +454,14 @@ unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk) { c[HOLDER_HOPS + i] = w.hops[i].0 as i64; } c[HOLDER_HOP_SHAPES] = (u64::from(w.hops[0].1) | u64::from(w.hops[1].1) << 32) as i64; - if c[HOLDER_REGISTERED] == 0 { - c[HOLDER_REGISTERED] = 1; + if c[HOLDER_STATE] & STATE_REGISTERED == 0 { + c[HOLDER_STATE] |= STATE_REGISTERED; if let Ok(mut sites) = HOLDER_SITES.lock() { sites.push(cache as usize); } } // Last: the entry is live only once every other word is written. - c[HOLDER_RECV] = (u64::from(object_shape_stamp(recv)) | PIC_ID_TOKEN_BIT) as i64; + c[HOLDER_RECV] = token; if w.slot.is_some() { PRIMES_HOLDER.fetch_add(1, Ordering::Relaxed); } else { @@ -352,45 +479,11 @@ unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk) { /// `cache` the site's resolved `PicCache`. #[no_mangle] pub unsafe extern "C" fn js_read_site_holder_hit(recv: *const u8, cache: *const PicCache) -> f64 { - let hole = f64::from_bits(crate::value::TAG_HOLE); if cache.is_null() || recv.is_null() { - return hole; + return f64::from_bits(crate::value::TAG_HOLE); } - let c = &*cache; let token = (u64::from(std::ptr::read((recv as *const u32).add(1))) | PIC_ID_TOKEN_BIT) as i64; - if c[HOLDER_RECV] != token { - return hole; - } - let kind = c[HOLDER_KIND]; - let (depth, absent, slot) = if kind >= 0 { - (1, kind == HOLDER_ABSENT_DEPTH1, kind as u32) - } else { - let k = kind as u64; - ( - ((k >> HOLDER_DEPTH_SHIFT) & 0xF) as usize, - k & HOLDER_ABSENT_BIT != 0, - k as u32, - ) - }; - let hop_shapes = c[HOLDER_HOP_SHAPES] as u64; - let shape_words = [ - hop_shapes as u32, - (hop_shapes >> 32) as u32, - (c[HOLDER_SHAPE] as u64 >> 32) as u32, - ]; - for i in 0..depth.saturating_sub(1).min(HOLDER_MAX_DEPTH - 1) { - if shape_word(c[HOLDER_HOPS + i] as usize) != shape_words[i] { - return hole; - } - } - let holder = c[HOLDER_OBJ] as usize; - if shape_word(holder) != c[HOLDER_SHAPE] as u32 { - return hole; - } - if absent { - return f64::from_bits(crate::value::TAG_UNDEFINED); - } - f64::from_bits(slot_bits(holder, slot)) + f64::from_bits(entry_answer(&*cache, token).unwrap_or(crate::value::TAG_HOLE)) } /// Root scan: every live entry's holder and hops are marked and rewritten. @@ -399,7 +492,8 @@ pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVis return; }; for &site in sites.iter() { - // SAFETY: registered caches are arena allocations that are never freed. + // SAFETY: registered caches are PIC-arena allocations + // (`pic_arena_alloc`), which are never freed. let c = unsafe { &mut *(site as *mut PicCache) }; if c[HOLDER_RECV] == 0 { continue; diff --git a/crates/perry-runtime/src/typed_feedback/guards.rs b/crates/perry-runtime/src/typed_feedback/guards.rs index f72fbb0743..bb746ccfaa 100644 --- a/crates/perry-runtime/src/typed_feedback/guards.rs +++ b/crates/perry-runtime/src/typed_feedback/guards.rs @@ -899,9 +899,10 @@ fn class_field_get_one_path( ) } -/// The two answers the receiver's shape gives without the ladder, in the -/// order the emitted generic read asks them: the site's own word, then (on -/// its declined edge) the inherited-read cache. `None` for everything else. +/// The answers the receiver's shape gives without the ladder, in the order +/// the emitted generic read asks them: the site's own word, the site's holder +/// entry, then (on its declined edge) the inherited-read cache. `None` for +/// everything else. /// /// `leaf`: the caller is the S2 GC-leaf entry, so an inherited ACCESSOR entry /// (which runs a getter) is declined, as `js_inherited_read_cache_hit_f64` @@ -932,6 +933,13 @@ unsafe fn class_field_get_from_shape( crate::hot_diag::recv_route_note_runtime(crate::hot_diag::RT_ROUTE_CLASS_MISS_SHAPE); return Some(value); } + // The site's holder entry, which the emitted generic read asks next. + if let Some(value) = + crate::object::method_site::read_holder::read_holder_hit(handle, cache_slot) + { + crate::hot_diag::recv_route_note_runtime(crate::hot_diag::RT_ROUTE_CLASS_MISS_SHAPE); + return Some(value); + } let value = if leaf { let value = crate::object::inherited_read_cache::js_inherited_read_cache_hit_f64(handle, key); diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 37a24eb8fe..fc2a2eb06b 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -398,7 +398,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete \u2192 sweep-entry window of a synchronous full \u2014 where PASS1_MARKED is populated and consumed within one `run_to_completion` \u2014 is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize \u2014 INSIDE the window \u2014 the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes \u2014 in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 (charter step 5, P0): the field-representation census adds `mod census_field_repr;` to `gc/mod.rs` and, in `census.rs`, one Rust-owned accumulator fed from `visit_object` inside `take_census`, which runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED; it reads slots and the per-object layout maps (try_borrow), allocates nothing on the JS heap and alters no mark/sweep control flow. The window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 (class constructors as function objects, #11414): `gc/mod.rs` gains one `reg_scanner!` registration (`object::class_value::scan_class_value_roots_mut`, the per-agent class function-object table, visited and rewritten) \u2014 a root-scanner registration that alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 after #11659 added `verify::verify_array_hole_tails_at_collection()` to `gc/mod.rs` and `gc/policy.rs`. Every call sits at a collection or budgeted-cycle ENTRY, beside `roots::ensure_stack_maps_built()`, i.e. before mark begins and so before `census_pass1_if_armed` opens the window. The call is a read-only walk of array headers (debug builds, or release with PERRY_GC_VERIFY_ARRAY_HOLES): it neither allocates GC memory, relocates, runs JS, nor reads or writes PASS1_MARKED, and it panics rather than continuing on a violation. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-29 after the pinned-roots fix touched `gc/cycle.rs` and `gc/mod.rs`: the block-persistence live-block predicate now reads GC_FLAG_MARKED alone (pinned objects are marked as roots, so the set of live blocks is unchanged), and `gc/mod.rs` adds one `reg_scanner!` registration. Neither relocates, runs JS, or moves the mark-complete to sweep-entry window. Re-audited again 2026-09-29 (same fix, copying-minor follow-up): `BlockPersistCycleState`'s force-mark in `gc/cycle.rs` no longer skips an unmarked pinned header, so it marks and pushes it like any other object of a live block. That is marking inside the budgeted cycle's persistence step: nothing relocates, no JS runs, and the budgeted path still skips both census boundaries. The window is unchanged.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete \u2192 sweep-entry window of a synchronous full \u2014 where PASS1_MARKED is populated and consumed within one `run_to_completion` \u2014 is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize \u2014 INSIDE the window \u2014 the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes \u2014 in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 (charter step 5, P0): the field-representation census adds `mod census_field_repr;` to `gc/mod.rs` and, in `census.rs`, one Rust-owned accumulator fed from `visit_object` inside `take_census`, which runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED; it reads slots and the per-object layout maps (try_borrow), allocates nothing on the JS heap and alters no mark/sweep control flow. The window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 (class constructors as function objects, #11414): `gc/mod.rs` gains one `reg_scanner!` registration (`object::class_value::scan_class_value_roots_mut`, the per-agent class function-object table, visited and rewritten) \u2014 a root-scanner registration that alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 after #11659 added `verify::verify_array_hole_tails_at_collection()` to `gc/mod.rs` and `gc/policy.rs`. Every call sits at a collection or budgeted-cycle ENTRY, beside `roots::ensure_stack_maps_built()`, i.e. before mark begins and so before `census_pass1_if_armed` opens the window. The call is a read-only walk of array headers (debug builds, or release with PERRY_GC_VERIFY_ARRAY_HOLES): it neither allocates GC memory, relocates, runs JS, nor reads or writes PASS1_MARKED, and it panics rather than continuing on a violation. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-29 after the pinned-roots fix touched `gc/cycle.rs` and `gc/mod.rs`: the block-persistence live-block predicate now reads GC_FLAG_MARKED alone (pinned objects are marked as roots, so the set of live blocks is unchanged), and `gc/mod.rs` adds one `reg_scanner!` registration. Neither relocates, runs JS, or moves the mark-complete to sweep-entry window. Re-audited again 2026-09-29 (same fix, copying-minor follow-up): `BlockPersistCycleState`'s force-mark in `gc/cycle.rs` no longer skips an unmarked pinned header, so it marks and pushes it like any other object of a live block. That is marking inside the budgeted cycle's persistence step: nothing relocates, no JS runs, and the budgeted path still skips both census boundaries. The window is unchanged. Re-audited 2026-09-29 for the read-site holder entry: `gc/mod.rs` adds one `reg_scanner!` registration, `read_holder::scan_read_holder_roots_mut`, which visits the holder and hop words of registered read-site caches during ROOT SCAN, before mark propagation completes. It rewrites root slots only through the visitor (as every registered scanner does), runs no JS, and nothing it does executes between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. The window is unchanged.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -415,7 +415,7 @@ "sources": { "crates/perry-runtime/src/gc/census.rs": "b2b60a124ea0f1131108940a8e675dae92f24192ecb482f87b6183e7641a2677", "crates/perry-runtime/src/gc/cycle.rs": "aad8d71901d78a81998bf6af53ac058a4d56228af6fb1b18614de0a677010938", - "crates/perry-runtime/src/gc/mod.rs": "99fc1ae6ee60476713e4d9d2d3641a65de200d26e5345986c4263f5904af02b6", + "crates/perry-runtime/src/gc/mod.rs": "f96a5c31db59785b1ce3394739deaef9ce63d1798b26ce0b8bea49bf66de02cf", "crates/perry-runtime/src/gc/policy.rs": "84a869e0aa09e932a4d2b6601186260129ba851b4482f29579047713c60fc95e", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } diff --git a/scripts/thread_exit_address_globals.json b/scripts/thread_exit_address_globals.json index ab0c291b1c..336e1de04d 100644 --- a/scripts/thread_exit_address_globals.json +++ b/scripts/thread_exit_address_globals.json @@ -4059,6 +4059,14 @@ "verdict": "no_heap_address", "why": "Diagnostic refusal counters indexed by a reason code: the only writes are `SITE_REFUSED[reason].fetch_add(1, ..)` and the only reads are loads in the counter dump; never stores an address." }, + { + "file": "crates/perry-runtime/src/object/method_site/read_holder.rs", + "names": [ + "HOLDER_SITES" + ], + "verdict": "process_global_allocation", + "why": "The addresses of read-site `PicCache`s, each allocated by `field_get_set::ic_slot::pic_arena_alloc` from `std::alloc::alloc_zeroed` chunks that are never freed and belong to no thread's arena, so thread exit cannot free or reuse them. The primary-heap holder and hop addresses the caches hold are written only by the primary agent (`prime_read_holder` checks `current_agent() == PRIMARY_AGENT` and `WORKER_AGENTS_EXIST`), are strong roots visited by `scan_read_holder_roots_mut`, and are emptied by `empty_read_holder_entries` when the first worker agent starts." + }, { "file": "crates/perry-stdlib/src/common/handle_lifecycle.rs", "names": [ From e0a2feeb07721a063fc0739d5c2d805ee51442c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 16:29:14 +0000 Subject: [PATCH 3/5] perf(codegen): the holder entry sits behind the way state; only a latched site asks the inherited-read hook Asking the holder before the way-state branch cost every polymorphic way hit a load and a compare, and sending every fresh-ways miss to the hook cost sites whose receivers the miss handler never learns an extra lookup: Zod +0.66% median. Now a fresh site asks the holder, a way miss asks it, a megamorphic site goes straight to the call, and a holder miss asks the hook only when the site's state word says LATCHED (perry-abi PIC_HOLDER_STATE_WORD / PIC_HOLDER_STATE_LATCHED). Codegen tests follow. --- changelog.d/inherited-read-holder-entry.md | 17 +++-- crates/perry-abi/src/lib.rs | 5 ++ .../src/expr/property_get/generic_dispatch.rs | 66 +++++++++++------ .../src/expr/property_get/tests.rs | 74 +++++++++++-------- .../src/object/method_site/read_holder.rs | 5 +- 5 files changed, 102 insertions(+), 65 deletions(-) diff --git a/changelog.d/inherited-read-holder-entry.md b/changelog.d/inherited-read-holder-entry.md index a9b492ca40..3cc3b66075 100644 --- a/changelog.d/inherited-read-holder-entry.md +++ b/changelog.d/inherited-read-holder-entry.md @@ -11,11 +11,12 @@ the entry compares; a value store is seen because the slot is loaded. No global validity word is involved. `process.env` and `arguments` now carry a per-object prototype identity in their shape, so no shape-keyed memo admits them. -The entry is asked first where the site own-slot word misses, before the -polymorphic ways. A receiver it does not describe (a compiled-class instance, -a second receiver shape) keeps the inherited-read hook it had before, on the -never-primed edge and on a miss at a site whose ways never primed. A miss the -live entry answers primes nothing (the class-field read miss arm asks the -entry directly), and a site that refused once, or was re-primed for four -different receiver shapes, stops priming, so its misses do not walk the chain -and run the getter again. +The entry is asked where the site own-slot word and its polymorphic ways +miss (a megamorphic site goes straight to the call, as before). A receiver it +does not describe (a compiled-class instance) keeps the inherited-read hook +it had before on the never-primed edge, and a site that refused once, or was +re-primed for four different receiver shapes, is latched: it stops priming, +so its misses do not walk the chain and run the getter again, and it asks the +inherited-read hook as a never-primed site does. A miss the live entry +answers primes nothing (the class-field read miss arm asks the entry +directly). diff --git a/crates/perry-abi/src/lib.rs b/crates/perry-abi/src/lib.rs index b0ed24e5e6..cb53dffc49 100644 --- a/crates/perry-abi/src/lib.rs +++ b/crates/perry-abi/src/lib.rs @@ -68,6 +68,11 @@ pub const PIC_HOLDER_RECV_WORD: usize = 12; pub const PIC_HOLDER_OBJ_WORD: usize = 13; pub const PIC_HOLDER_SHAPE_WORD: usize = 14; pub const PIC_HOLDER_KIND_WORD: usize = 15; +/// The site's holder state word, and its bit for a LATCHED site: one that +/// refused, or whose non-own receivers took several shapes. Its misses ask the +/// inherited-read hook, as a never-primed site's do. +pub const PIC_HOLDER_STATE_WORD: usize = 20; +pub const PIC_HOLDER_STATE_LATCHED: i64 = 2; /// The kind word of a depth-1 ABSENT entry: the answer is `undefined`. pub const PIC_HOLDER_ABSENT_DEPTH1: i64 = 1 << 62; /// Words in a property-read cache: MRU, way state, four ways, the holder entry. diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 84905e0461..9328bff9be 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -1021,20 +1021,17 @@ pub(crate) fn lower_generic_property_get( // signal byte-identical and go without the hook. ctx.current_block = ways_entry_idx; let token_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); - // Without typed feedback the site's cache is asked, in order: the holder - // entry (`pic.holder`, first — the one answer a non-own key can have), - // then the ways; a site with no cache yet goes to the inherited-read hook. + // Without typed feedback, a site with no cache yet goes to the + // inherited-read hook; a site with one asks its ways (when live) and then + // its holder entry (`pic.holder`). let holder_on = !crate::expr::typed_feedback_emission_enabled(); let inherited_idx = holder_on.then(|| ctx.new_block("pic.miss.inherited")); let holder_idx = holder_on.then(|| ctx.new_block("pic.holder")); let never_primed_label = inherited_idx .map(|idx| ctx.block_label(idx)) .unwrap_or_else(|| cold_label.clone()); - let present_label = holder_idx - .map(|idx| ctx.block_label(idx)) - .unwrap_or_else(|| miss_label.clone()); ctx.block() - .cond_br(&token_cache.present, &present_label, &never_primed_label); + .cond_br(&token_cache.present, &miss_label, &never_primed_label); // `js_object_get_field_ic_miss` primes only slots below the descriptor's // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact @@ -1186,19 +1183,22 @@ pub(crate) fn lower_generic_property_get( let ways_live = ctx.block().icmp_sgt(I64, &way_state, "0"); let ways_idx = ctx.new_block("pic.ways"); let ways_label = ctx.block_label(ways_idx); - // A site whose ways never primed (state 0) has only ever seen receivers - // the MRU word or the holder entry named, or none it could learn: its - // miss asks the inherited-read hook first, as a never-primed site does. - // A megamorphic site (-1) goes straight to the call. - match inherited_idx { - Some(inh_idx) => { - let fresh_idx = ctx.new_block("pic.ways.fresh"); - let fresh_label = ctx.block_label(fresh_idx); - ctx.block().cond_br(&ways_live, &ways_label, &fresh_label); - ctx.current_block = fresh_idx; + // A fresh site (state 0) asks the holder entry at once, and a way miss + // asks it too. The holder sits BEHIND the way-state branch, not before + // it, so a polymorphic own-key site's way hit pays nothing for it (Zod + // reads through ways far more often than through a holder), and a + // megamorphic site (-1) goes to the call as before. + let past_ways_label = holder_idx + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| call_label.clone()); + match holder_idx { + Some(_) => { + let quiet_idx = ctx.new_block("pic.ways.quiet"); + let quiet_label = ctx.block_label(quiet_idx); + ctx.block().cond_br(&ways_live, &ways_label, &quiet_label); + ctx.current_block = quiet_idx; let fresh = ctx.block().icmp_eq(I64, &way_state, "0"); - let inh_label = ctx.block_label(inh_idx); - ctx.block().cond_br(&fresh, &inh_label, &call_label); + ctx.block().cond_br(&fresh, &past_ways_label, &call_label); } None => ctx.block().cond_br(&ways_live, &ways_label, &call_label), } @@ -1251,7 +1251,8 @@ pub(crate) fn lower_generic_property_get( .expect("PIC_WAYS is non-zero, so the reduction leaves exactly one lane"); let way_load_idx = ctx.new_block("pic.way.load"); let way_load_label = ctx.block_label(way_load_idx); - ctx.block().cond_br(&way_any, &way_load_label, &call_label); + ctx.block() + .cond_br(&way_any, &way_load_label, &past_ways_label); ctx.current_block = way_load_idx; if fused_recv.is_some() { @@ -1310,7 +1311,7 @@ pub(crate) fn lower_generic_property_get( } // The read site's HOLDER entry (`object::method_site::read_holder` in the - // runtime), asked first on the MRU word's miss edge: the answer for a key + // runtime), asked where the MRU word and the ways missed: the answer for a key // that is NOT own on the receiver, as facts of two shapes. The receiver's // ShapeId says the key is not own and which object is its [[Prototype]]; // the holder's ShapeId says the key is an own inline data slot there (or, @@ -1320,7 +1321,7 @@ pub(crate) fn lower_generic_property_get( // transition (#10826), so a matching holder ShapeId proves the slot live: // no `TAG_HOLE` test, as the MRU hit has none. // - // [cache + RECV] == token else the ways + // [cache + RECV] == token else pic.holder.miss // kind = [cache + KIND] ; kind stub (depth 2..4, deep absent) // h = [cache + OBJ] ; [h + 4] == low32([cache + SHAPE]) else call // kind & ABSENT_DEPTH1 -> undefined @@ -1340,7 +1341,26 @@ pub(crate) fn lower_generic_property_get( let recv_eq = ctx.block().icmp_eq(I64, &recv_word, &token); let kind_idx = ctx.new_block("pic.holder.kind"); let kind_label = ctx.block_label(kind_idx); - ctx.block().cond_br(&recv_eq, &kind_label, &miss_label); + let holder_miss_idx = ctx.new_block("pic.holder.miss"); + let holder_miss_label = ctx.block_label(holder_miss_idx); + ctx.block() + .cond_br(&recv_eq, &kind_label, &holder_miss_label); + + // A LATCHED site (it refused, or its non-own receivers took several + // shapes) asks the inherited-read hook, as a never-primed site does: + // the entry will not describe its receivers. Any other miss goes to + // the call, which primes. + ctx.current_block = holder_miss_idx; + let state = word(ctx, crate::runtime_abi::PIC_HOLDER_STATE_WORD); + let latched_bit = ctx.block().and( + I64, + &state, + &crate::runtime_abi::PIC_HOLDER_STATE_LATCHED.to_string(), + ); + let latched = ctx.block().icmp_ne(I64, &latched_bit, "0"); + let inh_label = + ctx.block_label(inherited_idx.expect("the hook exists whenever the holder arm does")); + ctx.block().cond_br(&latched, &inh_label, &call_label); ctx.current_block = kind_idx; let kind = word(ctx, crate::runtime_abi::PIC_HOLDER_KIND_WORD); diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 9b6ef0bb90..69b152c50c 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -495,10 +495,13 @@ fn generic_property_get_tries_ways_before_calling_the_miss_handler() { .filter(|&x| x > ways) .min() .unwrap()]; + // The way-miss edge goes on to the site's holder entry (`pic.holder`), + // whose own misses reach the call. assert!( - ways_body.contains("pic.way.load") && ways_body.contains("pic.miss.call"), + ways_body.contains("pic.way.load") && ways_body.contains("label %pic.holder."), "pic.ways must end in a branch choosing between the way load and the \ - miss call — otherwise the compares are not gating anything:\n{ways_body}" + holder entry on the way to the miss call — otherwise the compares are \ + not gating anything:\n{ways_body}" ); assert!( !ways_body.contains("call double @js_object_get_field_ic"), @@ -552,8 +555,8 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { ); // T1: the landing block is now the single slow exit itself, and the // dominance is structural — `pic.miss` has exactly ONE predecessor, - // `pic.holder` (whose only predecessor is `pic.token.ways`, reached from - // `pic.token.miss`), which `pic.token` dominates. Assert that directly: + // `pic.token.ways` (reached from `pic.token.miss`), which `pic.token` + // dominates. Assert that directly: // routing any receiver-validation failure back into `pic.miss` would add a // predecessor and immediately re-introduce the phis #7907 removed. // `pic.miss` carries a numeric suffix and `pic.miss.call` starts with the @@ -581,7 +584,7 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { .count(); assert_eq!( preds, 1, - "pic.miss must have exactly one predecessor (pic.holder), or it is \ + "pic.miss must have exactly one predecessor (pic.token.ways), or it is \ no longer dominated by pic.token:\n{ir}" ); assert!( @@ -1620,15 +1623,16 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { // `pic.way.live` is GONE with the way path's `TAG_HOLE` compare: the // load block has nothing left to decide and branches to the merge. "pic.way.load", - // a site whose ways never primed asks the inherited-read hook before - // the call, as a never-primed site does - "pic.ways.fresh", - "pic.miss.inherited", - // the holder entry (`method_site::read_holder`), first on the MRU - // miss edge: the receiver word, the kind, the inline depth-1 compare - // and load (or `undefined` for an absent entry), and the GC-leaf stub - // for depth 2..4 + // the holder entry (`method_site::read_holder`), past the ways: the + // receiver word, the kind, the inline depth-1 compare and load (or + // `undefined` for an absent entry), and the GC-leaf stub for depth + // 2..4; a receiver it does not name at a LATCHED site asks the + // inherited-read hook before the call. `pic.ways.quiet` splits a + // fresh site (to the holder) from a megamorphic one (to the call). + "pic.ways.quiet", "pic.holder", + "pic.holder.miss", + "pic.miss.inherited", "pic.holder.kind", "pic.holder.inline", "pic.holder.answer", @@ -1659,23 +1663,24 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { ); } -/// The read site's holder entry is the first answer asked where the MRU word +/// The read site's holder entry is asked where the MRU word and the ways /// missed, and the inherited-read hook is kept for what it does not describe: /// -/// 1. `pic.token.ways`: a present cache goes to `pic.holder`; the never-primed -/// edge goes to the inherited-read hook (`pic.miss.inherited`); -/// 2. `pic.holder`: a receiver word that is not the entry's goes on to the -/// ways (`pic.miss`), never straight to the exit; -/// 3. `pic.miss`: live ways go to `pic.ways`, otherwise `pic.ways.fresh`, -/// which asks the hook only for a site whose ways never primed (a -/// megamorphic site goes to the call); +/// 1. `pic.token.ways`: a present cache goes to the ways (`pic.miss`); the +/// never-primed edge goes to the inherited-read hook (`pic.miss.inherited`); +/// 2. `pic.miss` with no live way goes to `pic.ways.quiet`, which sends a +/// fresh site to `pic.holder` and a megamorphic one to the call; +/// `pic.ways` with no matching way goes to `pic.holder`; +/// 3. `pic.holder`: a receiver word that is not the entry's goes to +/// `pic.holder.miss`, which asks the hook only for a LATCHED site (any +/// other miss goes to the call, which primes); /// 4. the stub is called from `pic.holder.stub` only, declines on `TAG_HOLE` /// to the exit; the inline load has no hole compare (a delete is a shape /// transition, #10826) and goes to the merge only; /// 5. the hook calls `js_inherited_read_cache_hit_f64` and declines to the /// exit. #[test] -fn the_holder_entry_is_asked_first_and_the_hook_is_kept() { +fn the_holder_entry_is_asked_past_the_ways_and_the_hook_is_kept() { let ir = emit(false, None); let func = ir .split("\ndefine ") @@ -1726,29 +1731,34 @@ fn the_holder_entry_is_asked_first_and_the_hook_is_kept() { // 1. assert_eq!( targets(&term("pic.token.ways")), - ["pic.holder", "pic.miss.inherited"], + ["pic.miss", "pic.miss.inherited"], "{func}" ); // 2. assert_eq!( - targets(&term("pic.holder")), - ["pic.holder.kind", "pic.miss"], + targets(&term("pic.miss")), + ["pic.ways", "pic.ways.quiet"], "{func}" ); - // 3. assert_eq!( - targets(&term("pic.miss")), - ["pic.ways", "pic.ways.fresh"], + targets(&term("pic.ways.quiet")), + ["pic.holder", "pic.miss.call"], "{func}" ); assert_eq!( - targets(&term("pic.ways.fresh")), - ["pic.miss.inherited", "pic.miss.call"], + targets(&term("pic.ways")), + ["pic.way.load", "pic.holder"], "{func}" ); + // 3. assert_eq!( - targets(&term("pic.ways")), - ["pic.way.load", "pic.miss.call"], + targets(&term("pic.holder")), + ["pic.holder.kind", "pic.holder.miss"], + "{func}" + ); + assert_eq!( + targets(&term("pic.holder.miss")), + ["pic.miss.inherited", "pic.miss.call"], "{func}" ); // 4. diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 0d405538c3..0a554d3d92 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -75,13 +75,14 @@ pub const HOLDER_HOPS: usize = HOLDER_KIND + 1; pub const HOLDER_HOP_SHAPES: usize = HOLDER_HOPS + 3; /// The site's holder state: [`STATE_REGISTERED`], [`STATE_LATCHED`] and the /// count of re-primes for a different receiver shape. -pub const HOLDER_STATE: usize = HOLDER_HOP_SHAPES + 1; +pub const HOLDER_STATE: usize = crate::codegen_abi::PIC_HOLDER_STATE_WORD; /// The cache is on the root list. const STATE_REGISTERED: i64 = 1; /// The site refused, or is polymorphic in its non-own receivers: no walk and /// no prime from here on. -const STATE_LATCHED: i64 = 2; +const STATE_LATCHED: i64 = crate::codegen_abi::PIC_HOLDER_STATE_LATCHED; const STATE_REPRIME_SHIFT: u32 = 8; +const _: () = assert!(HOLDER_STATE == HOLDER_HOP_SHAPES + 1); /// Re-primes for a different receiver shape a site takes before it latches. const MAX_REPRIMES: i64 = 4; From 0be6f0396d38403167b858cc919e80ecd3a073b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 19:09:19 +0000 Subject: [PATCH 4/5] fix(read holder): an unresolved Object.prototype does not decide the pre-walk Since function objects stopped materializing the realm, a never-primed site whose chain ends at the default link walked before %Object.prototype% existed, refused, and stayed with the inherited-read cache for good. The walk after the getter (which resolves it) decides instead. --- .../src/object/method_site/read_holder.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index f41d92057e..5b9f8697c8 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -381,9 +381,18 @@ pub(crate) unsafe fn prime_read_holder( let recv = ordinary_receiver(obj as usize)?; // Cheap pre-walk: a receiver the entry could never describe keeps the // caller's path and pays nothing for the getter below. A site with no - // cache yet stays without one, so the emitted never-primed edge keeps its - // inherited-read hook. - if !holder_name_admitted(name) || key_may_be_accessor(recv, name) || walk(recv, name).is_none() + // cache yet stays without one, so the slow entry keeps asking the + // inherited-read cache for it. + // + // A walk that ends at the default link needs `%Object.prototype%`, which + // is materialized lazily: while it is unresolved the walk cannot pin it, + // and refusing here would leave the site to the inherited-read cache for + // good (the getter below is what resolves it). So an unresolved realm + // does not decide the pre-walk; the walk after the getter does. + let realm_pending = crate::array::object_prototype_addr_if_resolved() == 0; + if !holder_name_admitted(name) + || key_may_be_accessor(recv, name) + || (walk(recv, name).is_none() && !realm_pending) { refuse_and_latch(existing); return None; From 7b49810a3c87630a9743a60e7465cca08ae90c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 19:45:38 +0000 Subject: [PATCH 5/5] fix(read holder): an OrdinaryUnmarked hop is an ordinary layout Object.prototype is minted OrdinaryUnmarked; reads treat it as Ordinary, so the walk admits every ordinary-layout kind. --- crates/perry-runtime/src/object/method_site/read_holder.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 5b9f8697c8..a740a53128 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -50,8 +50,8 @@ use super::{key_may_be_accessor, next_prototype, ordinary_receiver, WORKER_AGENTS_EXIST}; use crate::object::shapes::{ - object_proto_id, object_shape_descriptor, object_shape_stamp, shape_proto_id, ShapeObjectKind, - PIC_ID_TOKEN_BIT, PROTO_ID_DEFAULT, PROTO_ID_NULL, + object_proto_id, object_shape_descriptor, object_shape_stamp, shape_proto_id, PIC_ID_TOKEN_BIT, + PROTO_ID_DEFAULT, PROTO_ID_NULL, }; use crate::object::{ObjectHeader, PicCache, PicCacheSlot}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -308,7 +308,7 @@ unsafe fn walk(recv: *const ObjectHeader, name: &[u8]) -> Option { return None; } let shape = object_shape_descriptor(next)?; - if shape.object_kind != ShapeObjectKind::Ordinary || object_shape_stamp(next) == 0 { + if !shape.object_kind.is_ordinary_layout() || object_shape_stamp(next) == 0 { return None; } let keys = shape.keys as usize as *const crate::array::ArrayHeader;