From 2e5cd3081534cf0891dcecd8698c579897095065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 08:22:16 +0200 Subject: [PATCH 01/40] perf: validate inherited method sites by holder shape and loaded slot --- crates/perry-codegen/src/expr/method_site.rs | 66 +++++---- .../perry-runtime/src/object/method_site.rs | 125 +++++++++--------- .../src/object/method_site/read_holder.rs | 2 +- crates/perry/tests/method_site.rs | 32 ++++- 4 files changed, 132 insertions(+), 93 deletions(-) diff --git a/crates/perry-codegen/src/expr/method_site.rs b/crates/perry-codegen/src/expr/method_site.rs index 9601bd1036..f8ce8d252c 100644 --- a/crates/perry-codegen/src/expr/method_site.rs +++ b/crates/perry-codegen/src/expr/method_site.rs @@ -12,7 +12,7 @@ //! w = load [recv] ; (class_id | ShapeId) //! w == site.word else MISS //! s = site.slot -//! s < 0 (inherited): PERRY_PROTO_VALIDITY == site.gen else MISS +//! s < 0 (inherited): [site.holder] == site.holder_word else MISS //! h = site.closure ; f = site.code //! own: v = load [recv + HDR + 8*s] ; v is a heap pointer else MISS //! fn: v = load [[recv + PROPS] + HDR + 8*s] (bit 61: a function's @@ -42,7 +42,11 @@ pub(crate) fn method_site_enabled(ctx: &FnCtx<'_>, property: &str, argc: usize) // 64-bit targets only: the site addresses 8-byte slots behind a 16-byte // header and compares an 8-byte receiver word. let triple = ctx.target_triple; - if !(triple.starts_with("x86_64") || triple.starts_with("aarch64")) || triple.contains("32") { + if !(triple.starts_with("x86_64") + || triple.starts_with("aarch64") + || triple.starts_with("arm64")) + || triple.contains("32") + { return false; } // A typed-feedback (profiling) build records every method call in the @@ -110,6 +114,7 @@ pub(crate) fn emit_method_site( let own_fn_idx = ctx.new_block("msite.own_fn"); let value_idx = ctx.new_block("msite.value"); let inh_idx = ctx.new_block("msite.inherited"); + let inh_value_idx = ctx.new_block("msite.inherited_value"); let call_idx = ctx.new_block("msite.call"); let miss_idx = ctx.new_block("msite.miss"); let merge_idx = ctx.new_block("msite.merge"); @@ -119,6 +124,7 @@ pub(crate) fn emit_method_site( let own_fn_l = ctx.block_label(own_fn_idx); let value_l = ctx.block_label(value_idx); let inh_l = ctx.block_label(inh_idx); + let inh_value_l = ctx.block_label(inh_value_idx); let call_l = ctx.block_label(call_idx); let miss_l = ctx.block_label(miss_idx); let merge_l = ctx.block_label(merge_idx); @@ -318,6 +324,33 @@ pub(crate) fn emit_method_site( blk.br(&value_l); (v, end) }; + // The receiver's shape pins the direct holder. Its word pins the slot; + // the slot value itself is loaded on every hit, just like an own method. + ctx.current_block = inh_idx; + let holder = { + let blk = ctx.block(); + let hp = blk.gep(crate::types::I8, &entry, &[(I64, &abi_closure)]); + let holder = blk.load(I64, &hp); + let holder_ptr = blk.inttoptr(I64, &holder); + let word = blk.load(I64, &holder_ptr); + let wp = blk.gep(crate::types::I8, &entry, &[(I64, &abi_gen)]); + let saved = blk.load(I64, &wp); + let valid = blk.icmp_eq(I64, &word, &saved); + blk.cond_br(&valid, &inh_value_l, &miss_l); + holder + }; + ctx.current_block = inh_value_idx; + let (inh_v, inh_end) = { + let blk = ctx.block(); + let holder_ptr = blk.inttoptr(I64, &holder); + let base = blk.gep(crate::types::I8, &holder_ptr, &[(I64, &header.to_string())]); + let idx = blk.and(I64, &slot, &index_mask); + let vp = blk.gep(I64, &base, &[(I64, &idx)]); + let v = blk.load(I64, &vp); + let end = blk.label.clone(); + blk.br(&value_l); + (v, end) + }; // value: it must hold a closure running the memoized body. ctx.current_block = value_idx; let own_ub = { @@ -328,6 +361,7 @@ pub(crate) fn emit_method_site( (&inline_v, &inline_end), (&spill_v, &spill_end), (&bag_v, &bag_end), + (&inh_v, &inh_end), ], ); let u = blk.sub(I64, &v, &(RECEIVER_BIAS as i64).to_string()); @@ -336,7 +370,7 @@ pub(crate) fn emit_method_site( u }; ctx.current_block = own_fn_idx; - let (own_handle, own_func, own_end) = { + let (own_handle, own_func, _own_end) = { let blk = ctx.block(); let kp = emit_field_ptr(blk, &own_ub, kind_offset); let kind = blk.load(crate::types::I16, &kp); @@ -387,31 +421,9 @@ pub(crate) fn emit_method_site( } (h, mf, end) }; - // inherited: the memoized closure, valid while the validity word holds. - ctx.current_block = inh_idx; - let (inh_handle, inh_func, inh_end) = { - let blk = ctx.block(); - let g = blk.load(I64, "@PERRY_PROTO_VALIDITY"); - let mg_p = blk.gep(crate::types::I8, &entry, &[(I64, &abi_gen)]); - let mg = blk.load(I64, &mg_p); - let valid = blk.icmp_eq(I64, &g, &mg); - let hp = blk.gep(crate::types::I8, &entry, &[(I64, &abi_closure)]); - let h = blk.load(I64, &hp); - let fp_p = blk.gep(crate::types::I8, &entry, &[(I64, &abi_code)]); - let f = blk.load(I64, &fp_p); - let end = blk.label.clone(); - blk.cond_br(&valid, &call_l, &miss_l); - (h, f, end) - }; // call: the body directly, with the receiver as its `this` parameter. ctx.current_block = call_idx; - let handle = ctx - .block() - .phi(I64, &[(&own_handle, &own_end), (&inh_handle, &inh_end)]); - let func = ctx - .block() - .phi(I64, &[(&own_func, &own_end), (&inh_func, &inh_end)]); - let fptr = ctx.block().inttoptr(I64, &func); + let fptr = ctx.block().inttoptr(I64, &own_func); let mut call_args: Vec = lowered_args.to_vec(); // Pad with `undefined` up to the arity the prime admits, so a body that // declares a few more parameters than this call passes is entered @@ -425,7 +437,7 @@ pub(crate) fn emit_method_site( let hit_value = crate::expr::body_call::emit_js_body_call( ctx.block(), crate::expr::body_call::JsBody::Pointer(&fptr), - &handle, + &own_handle, &recv_bits, &call_args, ); diff --git a/crates/perry-runtime/src/object/method_site.rs b/crates/perry-runtime/src/object/method_site.rs index 764395480c..a0caaeaa5a 100644 --- a/crates/perry-runtime/src/object/method_site.rs +++ b/crates/perry-runtime/src/object/method_site.rs @@ -9,9 +9,9 @@ //! proves the value is a closure whose body info equals //! [`MethodEntry::info`], and calls [`MethodEntry::code`] directly with the //! receiver as `this`; -//! * **inherited entry** ([`METHOD_SITE_INHERITED`] in `slot`) — compares -//! [`MethodEntry::gen`] against `PERRY_PROTO_VALIDITY` and calls -//! [`MethodEntry::code`] on the memoized closure [`MethodEntry::closure`]. +//! * **inherited entry** ([`METHOD_SITE_INHERITED`] in `slot`) — compares the +//! direct holder's word with [`MethodEntry::gen`], loads its slot, proves +//! the loaded closure has [`MethodEntry::info`], and calls it directly. //! //! Everything else calls [`js_method_site_miss`], which primes the entry when //! the facts below hold and then performs the ordinary dispatch. @@ -30,12 +30,10 @@ //! seen at once. The body info, not the closure, is compared: a factory //! that returns fresh closures per object shares one body, and the call //! passes the LOADED closure, so each object's captures are its own. -//! * An inherited entry holds the method closure itself. The chain it was -//! found through is made of MARKED prototypes only, and -//! `PERRY_PROTO_VALIDITY` moves on every structural change of a marked -//! object AND on every write to an existing slot of one -//! (`proto_validity::note_marked_value_write`, owner decision D3(b)), so an -//! unchanged word proves the closure is still the value `m` resolves to. +//! * An inherited entry holds the direct holder as a strong root. The +//! receiver's shape pins that holder, and the holder's shape pins the +//! inline slot. A structural change invalidates one of those word compares; +//! a value overwrite is seen by loading the slot on every hit. //! //! What the prime refuses (they keep the ordinary dispatch): non-ordinary //! receivers (class objects, native-module namespaces, dictionaries, @@ -66,17 +64,14 @@ //! * an entry that holds a heap reference stores it in [`MethodEntry::closure`] //! and is registered by [`publish`], so [`scan_method_site_roots_mut`] marks //! and rewrites it; -//! * anything the ShapeId does not pin is validated by `PERRY_PROTO_VALIDITY` -//! ([`MethodEntry::gen`]), which moves on every structural change of a -//! marked prototype and every write to an existing slot of one -//! (`proto_validity::note_marked_value_write`, -//! `proto_validity::store_cache_may_learn`); +//! * an inherited entry records the direct holder's word in +//! [`MethodEntry::gen`]; deeper chains stay on ordinary dispatch; //! * primes run after the ordinary dispatch, with collection suppressed. //! //! # GC //! -//! [`MethodEntry::closure`] is a STRONG root: marked, and rewritten when the -//! closure moves (`scan_method_site_roots_mut`). Every site that ever primed an +//! [`MethodEntry::closure`] is a STRONG root for the inherited holder: marked, +//! and rewritten when it moves (`scan_method_site_roots_mut`). Every site that ever primed an //! inherited entry is registered once for the scan. //! //! # Agents @@ -117,9 +112,9 @@ pub struct MethodEntry { /// The method body's `JsFunctionInfo` (the identity an own hit compares /// the slot closure's info word with). pub info: u64, - /// Inherited entry: the method closure's address (a STRONG GC root). + /// Inherited entry: the direct holder's address (a STRONG GC root). pub closure: usize, - /// Inherited entry: `PERRY_PROTO_VALIDITY` when the entry was primed. + /// Inherited entry: the holder's full `(class_id | ShapeId)` word. pub gen: u64, /// The method body's code address, the hit's call target. pub code: u64, @@ -184,17 +179,14 @@ const _: () = { /// The emitted `@perry_ic_N = private global ptr null` for a method site. pub type MethodSiteSlot = *mut MethodSite; -crate::perry_thread_local! { - /// Every site that holds (or held) an inherited entry, for the root scan. - static METHOD_SITES: std::cell::UnsafeCell> = - const { std::cell::UnsafeCell::new(Vec::new()) }; -} +/// Every site that holds (or held) an inherited entry, for the root scan and +/// for clearing primary-heap holders when the first worker starts. +static METHOD_SITES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); /// Set when the first `perry/thread` worker agent starts. Site memos are -/// process-global and an inherited entry holds a primary-heap closure, so from -/// then on no inherited entry is primed and every existing one is dead: the -/// same call bumps `PERRY_PROTO_VALIDITY`, which no entry primed earlier can -/// match again. Own entries hold no heap reference (ShapeIds are +/// process-global and an inherited entry holds a primary-heap holder, so from +/// then on no inherited entry is primed and every existing one is emptied. +/// Own entries hold no heap reference (ShapeIds are /// process-unique; the call passes the receiver's own closure). static WORKER_AGENTS_EXIST: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); @@ -203,6 +195,20 @@ static WORKER_AGENTS_EXIST: std::sync::atomic::AtomicBool = pub fn note_worker_agent() { if !WORKER_AGENTS_EXIST.swap(true, Ordering::SeqCst) { super::proto_validity::bump_proto_validity(); + if let Ok(sites) = METHOD_SITES.lock() { + for &site in sites.iter() { + // A worker must not read a primary-heap holder through a + // process-wide site. No inherited entry is published again. + unsafe { + for entry in (*(site as *mut MethodSite)).entries.iter_mut() { + if entry.slot & METHOD_SITE_INHERITED != 0 { + entry.word = METHOD_SITE_EMPTY; + entry.closure = 0; + } + } + } + } + } read_holder::empty_read_holder_entries(); } } @@ -440,6 +446,12 @@ unsafe fn site_is_megamorphic(slot: *mut MethodSiteSlot) -> bool { /// receiver word, else into an empty one, else over the next in turn. The /// word is written LAST, so a half-written entry never matches. unsafe fn publish(slot: *mut MethodSiteSlot, entry: MethodEntry) -> bool { + let Ok(mut sites) = METHOD_SITES.lock() else { + return false; + }; + if entry.slot & METHOD_SITE_INHERITED != 0 && WORKER_AGENTS_EXIST.load(Ordering::SeqCst) { + return false; + } let site = site_of(slot); if site.is_null() { return false; @@ -449,14 +461,14 @@ unsafe fn publish(slot: *mut MethodSiteSlot, entry: MethodEntry) -> bool { // an own entry replaces only the one naming the same slot AND body, so // objects of one shape holding different bodies each get an entry (the // emitted own hit falls through to the next way on a body mismatch). - let inherited = entry.slot == METHOD_SITE_INHERITED; + let inherited = entry.slot & METHOD_SITE_INHERITED != 0; let idx = site .entries .iter() .position(|e| { e.word == entry.word && if inherited { - e.slot == METHOD_SITE_INHERITED + e.slot & METHOD_SITE_INHERITED != 0 } else { e.slot == entry.slot && e.info == entry.info } @@ -473,8 +485,7 @@ unsafe fn publish(slot: *mut MethodSiteSlot, entry: MethodEntry) -> bool { }); if entry.closure != 0 && site.registered == 0 { site.registered = 1; - let ptr = site as *mut MethodSite; - METHOD_SITES.with(|cell| (*cell.get()).push(ptr)); + sites.push(site as *mut MethodSite as usize); } let e = &mut site.entries[idx]; e.word = METHOD_SITE_EMPTY; @@ -832,8 +843,8 @@ unsafe fn direct_callable( } } -/// Prime an inherited entry: `name` is absent from the receiver and found as -/// a plain inline data slot on a chain of MARKED prototypes. +/// Prime an inherited entry when the receiver's shape pins a direct holder +/// with `name` in a plain inline data slot. Deeper chains use ordinary dispatch. unsafe fn prime_inherited( slot: *mut MethodSiteSlot, obj: *const ObjectHeader, @@ -859,18 +870,18 @@ unsafe fn prime_inherited( refuse(8); return; } - // The ShapeId must name the prototype this walk follows (#11342). - let stamp = super::shapes::object_shape_stamp(obj); - if super::shapes::shape_proto_id(stamp) != Some(super::shapes::object_proto_id(obj)) { + // Only a serial or the realm-default identity pins one direct prototype. + let Some(proto_id) = read_holder::admitted_proto_id(obj) else { refuse(7); return; - } - // Read BEFORE the walk: a change during it can only make the entry older. - let gen = super::proto_validity::proto_validity(); - let mut current = obj; - for _hop in 0..4 { - let next = next_prototype(current); - if next.is_null() || next == current || next == obj { + }; + { + let next = if proto_id == super::shapes::PROTO_ID_DEFAULT { + crate::array::object_prototype_addr_if_resolved() as *const ObjectHeader + } else { + next_prototype(obj) + }; + if next.is_null() || next == obj { refuse(9); return; } @@ -902,16 +913,9 @@ unsafe fn prime_inherited( return; } let meta = (*next).meta; - if meta.is_null() || (*meta).flags & super::OBJECT_META_FLAG_IS_PROTOTYPE == 0 { - // Only a marked hop invalidates an entry recorded through it. - // Marking allocates (the meta record), so nothing here may be - // touched afterwards: mark and abandon; the next miss primes. - let _ = super::proto_validity::mark_object_as_prototype(next_addr); - refuse(8); - return; - } - if (*meta).elements != 0 - || (*meta).flags & super::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER != 0 + if (!meta.is_null() + && ((*meta).elements != 0 + || (*meta).flags & super::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER != 0)) || key_may_be_accessor(next, name) { refuse(8); @@ -937,11 +941,11 @@ unsafe fn prime_inherited( } let entry = MethodEntry { word, - slot: METHOD_SITE_INHERITED, + slot: METHOD_SITE_INHERITED | u64::from(s), info: info as *const crate::closure::JsFunctionInfo as u64, code: info.code as u64, - closure: (value & crate::value::POINTER_MASK) as usize, - gen, + closure: next_addr, + gen: std::ptr::read(next_addr as *const u64), }; if publish(slot, entry) { PRIMES_INHERITED.fetch_add(1, Ordering::Relaxed); @@ -949,7 +953,6 @@ unsafe fn prime_inherited( return; } } - current = next; } refuse(9); } @@ -977,13 +980,13 @@ unsafe fn next_prototype(obj: *const ObjectHeader) -> *const ObjectHeader { /// Root scan: every inherited entry's closure is marked and rewritten. pub(crate) fn scan_method_site_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - METHOD_SITES.with(|cell| unsafe { - for &site in (*cell.get()).iter() { - for e in (*site).entries.iter_mut() { + if let Ok(sites) = METHOD_SITES.lock() { + for &site in sites.iter() { + for e in unsafe { (*(site as *mut MethodSite)).entries.iter_mut() } { if e.closure != 0 { visitor.visit_tagged_usize_slot(&mut e.closure, crate::value::POINTER_TAG); } } } - }); + } } diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index a740a53128..ac037f7710 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -258,7 +258,7 @@ unsafe fn hop_admitted(addr: usize, name: &[u8]) -> bool { /// The prototype identity `obj`'s shape records, if it admits: a serial, the /// default link or null — and equal to what the object says it is. -unsafe fn admitted_proto_id(obj: *const ObjectHeader) -> Option { +pub(super) unsafe fn admitted_proto_id(obj: *const ObjectHeader) -> Option { let pid = shape_proto_id(object_shape_stamp(obj))?; let serial = pid != PROTO_ID_DEFAULT && pid < crate::object::shapes::PROTO_ID_CLASS; if !(serial || pid == PROTO_ID_DEFAULT || pid == PROTO_ID_NULL) { diff --git a/crates/perry/tests/method_site.rs b/crates/perry/tests/method_site.rs index a3da6c1826..e5ffcdae84 100644 --- a/crates/perry/tests/method_site.rs +++ b/crates/perry/tests/method_site.rs @@ -101,8 +101,8 @@ console.log(s, out.join(",")); ); } -/// Sabotage: an existing-slot write to a marked prototype does not bump PERRY_PROTO_VALIDITY, or the -/// emitted inherited hit skips the validity compare -> the old closure is called. +/// Sabotage: the inherited hit reuses a memoized closure instead of loading +/// the holder's slot -> a replacement still calls the old method. #[test] fn an_inherited_method_reassigned_by_any_store_spelling_is_seen() { let (stdout, own, inherited, misses) = run( @@ -141,8 +141,8 @@ console.log(s, out.join(",")); ); } -/// Sabotage: the emitted inherited hit skips the validity compare -> the redefined / deleted method is -/// still called. +/// Sabotage: the emitted inherited hit skips the holder-word compare -> a +/// redefined or deleted method is still called. #[test] fn define_property_and_delete_on_the_holder_invalidate_the_inherited_entry() { let (stdout, own, inherited, misses) = run( @@ -166,6 +166,7 @@ for (let i = 0; i < N; i++) { s += r; if (i % 1000 < 2) out.push(r); } + console.log(s, out.join(",")); "#, ); @@ -179,6 +180,29 @@ console.log(s, out.join(",")); ); } +/// Mutating an unrelated marked prototype must not invalidate the inherited +/// method entry: the receiver and its direct holder keep their shape words. +#[test] +fn an_unrelated_prototype_write_does_not_invalidate_the_method_site() { + let (stdout, own, inherited, misses) = + run(r#"const proto: any = { m(x: number) { return x + 1; } }; +const o: any = Object.create(proto); +const unrelated: any = { y: 0 }; +const child: any = Object.create(unrelated); +let sum = 0; +for (let i = 0; i < 6000; i++) { + unrelated.y = i; + sum += o.m(i); +} +console.log(sum, unrelated.y, child.y); +"#); + assert_eq!(stdout, "18003000 5999 5999"); + assert!( + inherited > 0 && misses < 20, + "the inherited entry was invalidated by an unrelated store (own={own} inherited={inherited} misses={misses})" + ); +} + /// What the prime must refuse (rest, `arguments`, bound) and what the hit must keep (arity padding, /// per-object captures, `this` after a throw, a GC-moved inherited closure). Sabotage: the own hit /// skips the code-pointer compare -> another object's method runs. From 7c2fb81cffb9d6763a82cc71fc7c2f3fc1be1386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 09:21:39 +0200 Subject: [PATCH 02/40] perf: retire inherited read side table and marked value invalidation --- crates/perry-codegen/src/expr/method_site.rs | 6 +- .../src/expr/property_get/tests.rs | 50 +- .../src/gc_effects/linux-x86_64.tsv | 2 - .../src/gc_effects/macos-aarch64.tsv | 2 - .../src/gc_effects/windows-x86_64.tsv | 2 - crates/perry-codegen/src/root_reload.rs | 1 - .../src/runtime_decls/objects.rs | 6 - .../perry-codegen/src/wasm32/runtime_abi.tsv | 2 - crates/perry-runtime/src/gc/dead_owner.rs | 6 - crates/perry-runtime/src/gc/mod.rs | 10 +- .../gc/tests/inherited_read_cache_roots.rs | 92 - crates/perry-runtime/src/gc/tests/mod.rs | 1 - crates/perry-runtime/src/hot_diag.rs | 22 +- .../perry-runtime/src/object/accessor_pair.rs | 21 - .../perry-runtime/src/object/field_get_set.rs | 15 +- .../src/object/field_get_set/accessors.rs | 8 - .../object/field_get_set/get_field_by_name.rs | 38 +- .../get_field_by_name_probe_tests.rs | 77 + .../src/object/field_get_set/ic_miss.rs | 107 +- .../object/field_get_set/ic_miss/ic_slow.rs | 22 - .../src/object/inherited_read_cache.rs | 1685 ----------------- .../src/object/inherited_read_cache_tests.rs | 1384 -------------- .../perry-runtime/src/object/method_site.rs | 13 +- .../src/object/method_site/read_holder.rs | 9 +- crates/perry-runtime/src/object/mod.rs | 1 - .../src/object/proto_validity.rs | 162 +- crates/perry-runtime/src/object/slot_store.rs | 7 +- crates/perry-runtime/src/object/spill.rs | 3 - crates/perry-runtime/src/proxy/put_value.rs | 25 - .../src/proxy/put_value/packed_set.rs | 19 - .../src/typed_feedback/guards.rs | 36 +- crates/perry-runtime/src/value/addr_class.rs | 19 +- crates/perry/tests/method_site.rs | 118 +- scripts/gc_root_dominance_check.py | 4 - scripts/gc_runtime_root_holders.json | 2 +- scripts/global_sink_asserted_baseline.txt | 1 - scripts/thread_exit_address_globals.json | 8 + .../test_parity_inherited_read_cache.ts | 5 +- 38 files changed, 263 insertions(+), 3728 deletions(-) delete mode 100644 crates/perry-runtime/src/gc/tests/inherited_read_cache_roots.rs delete mode 100644 crates/perry-runtime/src/object/inherited_read_cache.rs delete mode 100644 crates/perry-runtime/src/object/inherited_read_cache_tests.rs diff --git a/crates/perry-codegen/src/expr/method_site.rs b/crates/perry-codegen/src/expr/method_site.rs index f8ce8d252c..b4ffb77732 100644 --- a/crates/perry-codegen/src/expr/method_site.rs +++ b/crates/perry-codegen/src/expr/method_site.rs @@ -1,6 +1,6 @@ //! The method-call site: `recv.m(args)` as ONE path — the property read's -//! receiver test and shape compare, one slot load (or the memoized inherited -//! closure), then a direct call of the method body with `recv` as `this`. +//! receiver test and shape compare, a slot load (from the receiver or a +//! shape-guarded direct holder), then a direct call with `recv` as `this`. //! //! The site's memo is a runtime `MethodSite` //! (`perry-runtime/src/object/method_site.rs`, which states what an entry @@ -13,7 +13,7 @@ //! w == site.word else MISS //! s = site.slot //! s < 0 (inherited): [site.holder] == site.holder_word else MISS -//! h = site.closure ; f = site.code +//! v = load [site.holder + HDR + 8*i] //! own: v = load [recv + HDR + 8*s] ; v is a heap pointer else MISS //! fn: v = load [[recv + PROPS] + HDR + 8*s] (bit 61: a function's //! own-property object; same checks as own) diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 689ff41b04..a17cae655e 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -1087,15 +1087,6 @@ fn generic_property_get_slot_load_is_reached_only_through_every_guard() { "the overflow-bit test must not gate the inline slot load — a spill \ entry is refused by the ShapeId compare itself:\n{chain}" ); - // The inherited-read hook (#10834/#10842) lives on the DECLINED edge. Its - // answer must never be a condition on the way to the own slot load: if it - // were, an own read would pay a call, and this walk would have collected - // the call's result in the chain. - assert!( - !chain.contains("js_inherited_read_cache_hit_f64"), - "the inherited-read hook must not gate the inline slot load:\n{chain}" - ); - // The GC header is not read on the way to the slot load at all: neither // the kind byte (#10828 closed rule 3 — a `+4` word equal to a live // ShapeId proves `GC_TYPE_OBJECT`) nor the descriptor flag (#10824 closed @@ -1621,7 +1612,6 @@ fn the_generic_tower_is_one_leaf_call_two_exits_and_a_bounded_number_of_blocks() .filter(|c| { c.starts_with("js_object_get_field") || c.starts_with("js_typed_feedback_object_get_field") - || c.starts_with("js_inherited_read_cache") || *c == "js_throw_type_error_property_access" }) .collect(); @@ -1748,45 +1738,13 @@ fn a_spill_entry_is_recognised_by_the_front_and_nowhere_at_the_site() { ); } -/// The inherited-read cache (#10834/#10842) is asked on the NEVER-PRIMED edge -/// and nowhere else. A read whose key lives on the prototype chain is never an -/// own slot on the receiver's shape, so a site that only reads such a key never -/// resolves its per-site cache. The first placement asked on EVERY path into -/// the exit and charged each own-key miss a declining probe (+88 on a -/// megamorphic site, +89 on a spill read, measured). -/// -/// First-read D3: the probe moved into the slow entry -/// (`js_object_get_field_ic_slow`, which asks it only when the site's cache -/// slot is unresolved), behind the leaf front — so an own-key way, spill or -/// latched read never reaches it, and the site expands none of it. Pinned -/// here, each of which would otherwise fail silently (the program still -/// computes the right value through the slow entry): -/// -/// 1. no block of the site calls the hook — in particular none on a path to -/// the inline slot load (the CFG-walk test asserts the same from the other -/// side); -/// 2. the slow entry is called from `pic.miss.call` only, with the same four -/// operands (the never-primed test reads the cache slot); -/// 3. `pic.miss.call` is reached from the front only on its `TAG_HOLE` -/// decline, so a front-served read never pays the probe; -/// 4. the merge takes the slow entry's value from `pic.miss.call`. +/// The generic read's collecting slow entry belongs on the miss-front decline. +/// Own-word and holder-shape hits bypass it. The call keeps all four operands, +/// and the merge uses the value returned by that one miss entry. #[test] -fn the_inherited_read_cache_is_asked_on_the_never_primed_edge_only() { +fn the_generic_slow_read_is_called_only_after_the_front_declines() { let ir = emit(false, None); let blocks = tower_blocks(&ir); - // 1. - let holders: Vec<&str> = blocks - .iter() - .filter(|(_, body)| { - body.iter() - .any(|l| l.contains("@js_inherited_read_cache_hit_f64(")) - }) - .map(|(l, _)| l.as_str()) - .collect(); - assert!( - holders.is_empty(), - "the inherited hook belongs to the slow entry, not the site: {holders:?}" - ); // 2. let slow_callers: Vec<(&str, &String)> = blocks .iter() diff --git a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv index 6f81e322fc..2e425eb2aa 100644 --- a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv @@ -1455,8 +1455,6 @@ js_import_meta_resolve Reenters js_import_meta_resolve_value Reenters js_in_operator Reenters js_in_operator_presence_ic Reenters -js_inherited_read_cache_hit_f64 Leaf -js_inherited_read_cache_stats Leaf js_inline_arena_slow_alloc AllocOnly js_inline_arena_state Leaf js_install_global_value_surfaces Leaf diff --git a/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv b/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv index af2d4ccd8b..709638c250 100644 --- a/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv +++ b/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv @@ -1455,8 +1455,6 @@ js_import_meta_resolve Reenters js_import_meta_resolve_value Reenters js_in_operator Reenters js_in_operator_presence_ic Reenters -js_inherited_read_cache_hit_f64 Leaf -js_inherited_read_cache_stats Leaf js_inline_arena_slow_alloc AllocOnly js_inline_arena_state Leaf js_install_global_value_surfaces Leaf diff --git a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv index 9557dde1dd..1d310a3f5b 100644 --- a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv @@ -1455,8 +1455,6 @@ js_import_meta_resolve Reenters js_import_meta_resolve_value Reenters js_in_operator Reenters js_in_operator_presence_ic Reenters -js_inherited_read_cache_hit_f64 Leaf -js_inherited_read_cache_stats Leaf js_inline_arena_slow_alloc AllocOnly js_inline_arena_state Leaf js_install_global_value_surfaces Leaf diff --git a/crates/perry-codegen/src/root_reload.rs b/crates/perry-codegen/src/root_reload.rs index 979c01d862..bb15bae1d7 100644 --- a/crates/perry-codegen/src/root_reload.rs +++ b/crates/perry-codegen/src/root_reload.rs @@ -206,7 +206,6 @@ const NON_COLLECTING: &[&str] = &[ "js_write_barrier_root_nanbox", "perry_transition_cache_base", "js_transition_ic_note_hit", - "js_inherited_read_cache_hit_f64", // S2 GC-leaf IC hits; proven `Leaf` by the generated call-effects table. "js_object_get_field_ic_fast", // First-read D3: a generic read's miss front, proven `Leaf` likewise. diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index a229b2dff5..ad320d6c80 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -632,12 +632,6 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { ); module.declare_function("perry_transition_cache_base", PTR, &[]); module.declare_function("js_transition_ic_note_hit", VOID, &[]); - // #10834/#10842: the inherited-read cache hit, asked on the generic - // property read's declined-guard edge (`expr/property_get/ - // generic_dispatch.rs`): masked receiver + interned key -> NaN-boxed - // value, or `TAG_HOLE` for a decline. A pure state read (see - // `gc_call_effects.rs`). - module.declare_function("js_inherited_read_cache_hit_f64", DOUBLE, &[PTR, PTR]); // The per-agent pointer block (`expr/agent_ptr.rs`), read inline on ELF // executables through the initial-exec TLS model; its slot-1 accessor, // and the method-call site's miss entry (`expr/method_site.rs`). diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index 60a893dce6..7216d9b8db 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -1676,8 +1676,6 @@ js_import_meta_resolve f64 f64,f64,f64 js_import_meta_resolve_value f64 f64 js_in_operator f64 f64,f64 js_in_operator_presence_ic f64 f64,f64,ptr -js_inherited_read_cache_hit_f64 f64 ptr,ptr -js_inherited_read_cache_stats f64 i32s js_inline_arena_slow_alloc ptr ptr,usize,usize js_inline_arena_state ptr js_install_global_value_surfaces void diff --git a/crates/perry-runtime/src/gc/dead_owner.rs b/crates/perry-runtime/src/gc/dead_owner.rs index 42a011ae0e..7ca7a6dbe8 100644 --- a/crates/perry-runtime/src/gc/dead_owner.rs +++ b/crates/perry-runtime/src/gc/dead_owner.rs @@ -404,12 +404,6 @@ pub(super) const DEAD_KEY_PRUNES: &[DeadKeyPrune] = &[ // it needs a death story of its own -- otherwise a recycled holder address // becomes a false hit that reads a live object's slot for the wrong key // (rewrite_raw_addr's #8174 note). - DeadKeyPrune { - table: "INHERITED_READ_CACHE", - owner: DeadKeyOwner::Any, - prune: crate::object::inherited_read_cache::prune_dead_inherited_cache_entries, - young_prune: None, - }, // #6759 C1: shape records are keyed on keys_array addresses; drop the // ones whose keys_array died (memory only — per-hit validation covers // correctness for anything this misses). diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 20976516c9..a6e294be76 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -1067,17 +1067,11 @@ pub fn gc_init() { // or Proxy trap can re-enter after moving GC. Rewrite that temporary // identity so malformed prototype cycles remain bounded. reg_scanner!(crate::object::prototype_chain::scan_prototype_resolution_stack_roots_mut,); - // Lane 3: the inherited-read cache records a holder ADDRESS per entry and - // a hit LOADS through it, so the slots are STRONG roots: marked, so the - // address cannot be recycled under the entry, and rewritten, so a - // compacting or copying pass leaves it pointing at the same object. - reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut); - // Inherited-access lane: a store site's chain verdict names its interned + // A store site's chain verdict names its interned // key and the receiver's recorded prototype, and compares them on every // use, so both are STRONG roots (`object::chain_store`). reg_scanner!(crate::object::chain_store::scan_chain_store_roots_mut); - // Method-calls lane: an inherited method-site entry holds the method - // closure it calls, so the closure is a STRONG root (`object::method_site`). + // An inherited method-site entry roots its direct prototype holder. reg_scanner!(crate::object::method_site::scan_method_site_roots_mut); // A read site's holder entry names the object that holds the answer (and // the hops to it); the emitted hit loads through it, so each is a STRONG diff --git a/crates/perry-runtime/src/gc/tests/inherited_read_cache_roots.rs b/crates/perry-runtime/src/gc/tests/inherited_read_cache_roots.rs deleted file mode 100644 index 7d6183c4bb..0000000000 --- a/crates/perry-runtime/src/gc/tests/inherited_read_cache_roots.rs +++ /dev/null @@ -1,92 +0,0 @@ -//! The inherited-read cache's two obligations to the collector. -//! -//! The cache records a holder ADDRESS and, on a hit, LOADS through it. So a -//! relocation that this table does not learn about is not a stale answer, it -//! is a read of whatever now lives at that address — a wrong value, returned -//! with no error. These two tests are what stop that being a code comment. - -use super::super::*; - -/// A collection that moves a cached holder must rewrite this table's copy of -/// its address. -/// -/// The assertion is made through the cache's own hit path rather than by -/// reading the entry: an entry whose address was rewritten but whose recorded -/// shape no longer matches would be a rewrite that achieved nothing. -#[test] -fn a_relocated_holder_is_rewritten_by_the_root_scan() { - let _lock = crate::gc::global_side_table_test_lock(); - let _suppress = crate::gc::GcSuppressScope::new(); - crate::object::inherited_read_cache::test_clear_cache(); - - unsafe { - let proto = crate::object::js_object_alloc(0, 4); - let proto_key = crate::string::js_string_from_bytes(b"ircroot_a".as_ptr(), 9); - crate::object::js_object_set_field_by_name(proto, proto_key, 7.0); - let obj = crate::object::js_object_alloc(0, 4); - let own_key = crate::string::js_string_from_bytes(b"ircroot_own".as_ptr(), 11); - crate::object::js_object_set_field_by_name(obj, own_key, 1.0); - crate::object::js_object_set_prototype_of( - f64::from_bits(crate::value::js_nanbox_pointer(obj as i64).to_bits()), - f64::from_bits(crate::value::js_nanbox_pointer(proto as i64).to_bits()), - ); - - let lookup = crate::string::js_string_from_bytes(b"ircroot_a".as_ptr(), 9); - assert!( - crate::object::inherited_read_cache::inherited_read_cache_prime(obj, lookup).is_some(), - "fixture: the chain walk must resolve the key before there is \ - anything for a collection to relocate" - ); - assert!( - crate::object::inherited_read_cache::inherited_read_cache_hit(obj, lookup).is_some(), - "fixture: the entry must be serving the read before the move" - ); - - // Model the evacuation: a to-space twin carrying the same shape stamp - // and the same slot, with the original forwarded to it. - let relocated = crate::object::js_object_alloc(0, 4); - crate::object::js_object_set_field_by_name(relocated, proto_key, 7.0); - std::ptr::copy_nonoverlapping( - proto as *const u8, - relocated as *mut u8, - std::mem::size_of::() + 4 * 8, - ); - let valid_ptrs = crate::gc::trace::build_valid_pointer_set(); - set_forwarding_address( - header_from_user_ptr(proto as *const u8) as *mut GcHeader, - relocated as *mut u8, - ); - crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut( - &mut RuntimeRootVisitor::for_rewrite(&valid_ptrs), - ); - - let value = crate::object::inherited_read_cache::inherited_read_cache_hit(obj, lookup); - assert!( - value.is_some(), - "the root scan did not follow the holder's forwarding pointer, so \ - the entry still names from-space" - ); - assert_eq!(f64::from_bits(value.unwrap().bits()), 7.0); - } -} - -/// A scanner that is written but never registered is documentation. The same -/// gap this test closes is the one #6981 left open for the memoized -/// `Array.prototype` address. -#[test] -fn the_inherited_read_cache_scanner_is_registered() { - crate::gc::gc_init(); - let registered = crate::gc::roots::MUTABLE_ROOT_SCANNERS.with(|scanners| { - scanners.borrow().iter().any(|entry| { - entry.scanner as usize - == crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut - as MutableRootScanner as usize - }) - }); - assert!( - registered, - "the inherited-read cache records holder addresses and dereferences \ - them; a moving collector that cannot see this table hands it \ - from-space" - ); -} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index a244d271b8..e0a6b09de0 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -49,7 +49,6 @@ mod host_safepoints; mod idle_compact; mod idle_reclaim; mod incremental_sweep_reclaim; -mod inherited_read_cache_roots; mod inline_generation_gate_contract; mod inline_pointer_bearing_contract; mod json_parse_scalar; diff --git a/crates/perry-runtime/src/hot_diag.rs b/crates/perry-runtime/src/hot_diag.rs index ef490bbbad..6eeb8ab7f0 100644 --- a/crates/perry-runtime/src/hot_diag.rs +++ b/crates/perry-runtime/src/hot_diag.rs @@ -1169,22 +1169,6 @@ impl IcDiag { self.prime_way_encoded_slot ); } - // Lane 3's inherited-read cache, on the SAME arming rather than an - // env var of its own. A cache that primes and then declines every - // lookup returns exactly the values the chain walk would and is - // invisible in a program's output; this row is what tells a real - // program's run apart from that. - let inh_hits = crate::object::inherited_read_cache::inherited_read_cache_hits(); - let inh_primes = crate::object::inherited_read_cache::inherited_read_cache_primes(); - let inh_declines = crate::object::inherited_read_cache::inherited_read_cache_declines(); - let inh_neg = crate::object::inherited_read_cache::inherited_read_cache_neg_served(); - if (inh_hits | inh_primes | inh_declines | inh_neg) != 0 { - let _ = writeln!( - out, - " inherited: hits={inh_hits} primes={inh_primes} \ - declines={inh_declines} declines_cached={inh_neg}" - ); - } let mut rows: Vec<&SiteStat> = self.sites.values().collect(); crate::cold_sort::sort_by_key(&mut rows, |s| std::cmp::Reverse(s.misses)); let _ = writeln!( @@ -1582,10 +1566,10 @@ const RECV_ROUTE_NAMES: [&str; 33] = [ // supplier matched (`Route::RloopStatic`). "rloop_static", // Runtime-counted: a class-field read whose inline guard missed, answered - // from the receiver's shape (the site's word or the inherited cache)... + // from the receiver's shape (the site's word or holder fact)... "rt_class_miss_shape", - // ...or by the generic read ladder behind it (own miss, inherited cache, - // priming), where it used to take the site-less by-name walk. + // ...or by the generic read ladder behind it, where it used to take the + // site-less by-name walk. "rt_class_miss_ladder", // Runtime-counted: a by-name overwrite of a live inline slot on an object // holding a typed layout, which keeps it (it used to declare it unknown). diff --git a/crates/perry-runtime/src/object/accessor_pair.rs b/crates/perry-runtime/src/object/accessor_pair.rs index 9689ad72c0..3adbda6358 100644 --- a/crates/perry-runtime/src/object/accessor_pair.rs +++ b/crates/perry-runtime/src/object/accessor_pair.rs @@ -119,27 +119,6 @@ fn static_of(word: u64) -> usize { } } -/// The accessor a pair VALUE holds, without re-proving that it is one — for a -/// cache hit whose entry proved it at prime time (the holder's key is an -/// accessor, and a slot of an accessor key is written only by an accessor -/// install, which transitions the holder's ShapeId). -/// -/// # Safety -/// `value` is a NaN-boxed pointer to a pair. -#[inline(always)] -pub(crate) unsafe fn pair_of_value_unchecked(value: u64) -> Accessor { - let w = crate::array::array_elements_ptr((value & POINTER_MASK) as *const ArrayHeader); - let (raw_get_word, raw_set_word) = (*w.add(PAIR_RAW_GET), *w.add(PAIR_RAW_SET)); - Accessor { - get: closure_of(*w.add(PAIR_GET)), - set: closure_of(*w.add(PAIR_SET)), - raw_get: raw_of(raw_get_word), - raw_set: raw_of(raw_set_word), - static_get: static_of(raw_get_word), - static_set: static_of(raw_set_word), - } -} - #[inline] fn closure_word(bits: u64) -> u64 { if bits == 0 { diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index 120791ca3d..76a14153cd 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -261,13 +261,12 @@ impl FieldLookupCaches { // reach the cross-module helpers via their own `use super::*;`. pub use accessors::js_object_get_field; pub(crate) use accessors::{ - accessor_receiver_override_armed, accessor_receiver_override_begin, - accessor_receiver_override_end, accessor_receiver_override_take, - array_prototype_property_value, builtin_reflection_accessor_read, invoke_accessor_getter, - invoke_accessor_setter, is_typed_array_prototype, object_field_at_with_live, - ordinary_object_prototype_property_value, own_data_field_by_name, - primitive_builtin_prototype_property, primitive_object_prototype_accessor, - primitive_tagged_prototype_property, string_index_value, + accessor_receiver_override_begin, accessor_receiver_override_end, + accessor_receiver_override_take, array_prototype_property_value, + builtin_reflection_accessor_read, invoke_accessor_getter, invoke_accessor_setter, + is_typed_array_prototype, object_field_at_with_live, ordinary_object_prototype_property_value, + own_data_field_by_name, primitive_builtin_prototype_property, + primitive_object_prototype_accessor, primitive_tagged_prototype_property, string_index_value, }; pub(crate) use class_object_props::{ class_evaluation_prototype_class_id, class_object_materialized_prototype, @@ -292,7 +291,7 @@ pub use field_ops::{ }; pub use for_in_stable::js_for_in_keys_stable_value; pub(crate) use get_field_by_name::class_value_get_field; -pub(crate) use get_field_by_name::get_field_by_name_past_inherited_cache; +pub(crate) use get_field_by_name::get_field_by_name_after_site_miss; pub use get_field_by_name::js_object_get_field_by_name; pub(crate) use get_field_by_name_async::async_resource_property; pub(crate) use get_field_by_name_tail::get_field_by_name_object_tail; diff --git a/crates/perry-runtime/src/object/field_get_set/accessors.rs b/crates/perry-runtime/src/object/field_get_set/accessors.rs index beb89a1c6f..6e364dc32c 100644 --- a/crates/perry-runtime/src/object/field_get_set/accessors.rs +++ b/crates/perry-runtime/src/object/field_get_set/accessors.rs @@ -450,14 +450,6 @@ pub(crate) fn accessor_receiver_override_end(prev: Option) { ACCESSOR_RECEIVER_OVERRIDE.with(|c| c.set(prev)); } -/// Whether an inherited walk has armed a receiver that the next class getter -/// would take as `this` (#10498: the class-accessor cache neither records nor -/// serves under one). -#[inline] -pub(crate) fn accessor_receiver_override_armed() -> bool { - ACCESSOR_RECEIVER_OVERRIDE.with(|c| c.get().is_some()) -} - /// `this` to pass to a class getter (vtable `getters`) found while resolving a /// property. When the getter was reached by walking a prototype chain, `obj` is /// the PROTOTYPE the getter lives on — bind the original instance stashed by diff --git a/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs b/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs index 1a588da96f..0de2916b13 100644 --- a/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs +++ b/crates/perry-runtime/src/object/field_get_set/get_field_by_name.rs @@ -72,38 +72,16 @@ pub extern "C" fn js_object_get_field_by_name( } return JSValue::undefined(); } - // Lane 3 hook C: the same inherited-read entry, for the callers that do - // not come through a per-site cache (the recursive prototype hop, native - // callers, `js_object_get_field_by_name_f64`). It goes BEFORE - // `try_data_get_by_name` because that is the walk it replaces; a decline - // costs an epoch load and one failed compare. - use crate::object::inherited_read_cache::{inherited_read_cache_lookup, Lookup}; - match unsafe { inherited_read_cache_lookup(obj, key) } { - Lookup::Hit(value) => return value, - // A walk from this pair was refused and recorded: do not prime again. - Lookup::Declined => return get_field_by_name_past_inherited_cache(obj, key), - Lookup::Unknown => {} - } if let Some(value) = unsafe { super::super::native_get::try_data_get_by_name(obj, key) } { return value; } - // Hook D: the data probe missed, so this read is headed for the generic - // walk. Record what that walk finds (an inherited holder, or that the key - // is absent from the whole chain) so the next read of this (receiver - // shape, key) pair is served by the lookup above. The prime proves the - // key is not an own property before it walks. - if let Some(value) = - unsafe { crate::object::inherited_read_cache::inherited_read_cache_prime_by_name(obj, key) } - { - return value; - } get_field_by_name_past_data_probe(obj, key) } /// `C.key` for a class constructor value (its function object, or the legacy /// INT32 immediate / `C.prototype` reference): the class-static lookup. Split /// out so a class function object is routed here BEFORE the closure arms of -/// the generic read (`get_field_by_name_past_inherited_cache`), which it +/// the generic read (`get_field_by_name_after_site_miss`), which it /// would otherwise walk end to end first. #[inline(never)] pub(crate) fn class_value_get_field( @@ -573,14 +551,8 @@ mod primitive_proto_accessor_tests_10648 { } } -/// The same read for a caller that has ALREADY asked the inherited-read cache -/// and been refused. -/// -/// `get_field_ic_miss_impl` is exactly that caller: it consults the cache at -/// the top and falls through to here at the bottom. Asking twice is not free — -/// a read the cache refuses (an accessor on the prototype is the common one) -/// would pay two lookups per read for two answers that are the same. -pub(crate) fn get_field_by_name_past_inherited_cache( +/// The generic read after a site's own-key and holder-shape probes declined. +pub(crate) fn get_field_by_name_after_site_miss( obj: *const ObjectHeader, key: *const crate::StringHeader, ) -> JSValue { @@ -590,9 +562,7 @@ pub(crate) fn get_field_by_name_past_inherited_cache( get_field_by_name_past_data_probe(obj, key) } -/// [`get_field_by_name_past_inherited_cache`] for a caller that has also -/// already run `try_data_get_by_name` and been refused (hook D in -/// `js_object_get_field_by_name`), so the probe is not paid twice. +/// The generic read after `try_data_get_by_name` has already declined. fn get_field_by_name_past_data_probe( obj: *const ObjectHeader, key: *const crate::StringHeader, diff --git a/crates/perry-runtime/src/object/field_get_set/get_field_by_name_probe_tests.rs b/crates/perry-runtime/src/object/field_get_set/get_field_by_name_probe_tests.rs index 44292ad8e7..41d7809ecb 100644 --- a/crates/perry-runtime/src/object/field_get_set/get_field_by_name_probe_tests.rs +++ b/crates/perry-runtime/src/object/field_get_set/get_field_by_name_probe_tests.rs @@ -82,6 +82,83 @@ fn tail_from_slots(receiver_slot: u32, key_slot: u32) -> JSValue { ) } +/// The no-site by-name API still resolves inherited and absent reads through +/// the generic walk. It has no read-site holder entry after the side table is +/// retired; repeated reads must observe value overwrites and newly visible +/// keys. Ordinary objects must not acquire Set/Symbol registry probes. +#[test] +fn by_name_inherited_and_absent_reads_follow_live_prototypes() { + let _roots = TestShadowFrame::new(4); + root_pointer(0, crate::object::js_object_alloc(0, 0) as usize); + root_pointer(1, crate::object::js_object_alloc(0, 0) as usize); + root_string(2, key(b"inherited")); + root_string(3, key(b"later")); + let object_value = |slot: u32| { + f64::from_bits(crate::value::js_nanbox_pointer(rooted_pointer(slot) as i64).to_bits()) + }; + crate::object::js_object_set_field_by_name( + rooted_pointer(0) as *mut ObjectHeader, + rooted_pointer(2) as *const crate::StringHeader, + 2.0, + ); + crate::object::object_ops::js_object_set_prototype_of(object_value(1), object_value(0)); + + let set_before = crate::set::test_set_registry_probe_count(); + let symbol_before = crate::symbol::test_symbol_registry_probe_count(); + for _ in 0..64 { + assert_eq!( + f64::from_bits( + js_object_get_field_by_name( + rooted_pointer(1) as *const ObjectHeader, + rooted_pointer(2) as *const crate::StringHeader, + ) + .bits() + ), + 2.0, + ); + assert!(js_object_get_field_by_name( + rooted_pointer(1) as *const ObjectHeader, + rooted_pointer(3) as *const crate::StringHeader, + ) + .is_undefined()); + } + crate::object::js_object_set_field_by_name( + rooted_pointer(0) as *mut ObjectHeader, + rooted_pointer(2) as *const crate::StringHeader, + 3.0, + ); + crate::object::js_object_set_field_by_name( + rooted_pointer(0) as *mut ObjectHeader, + rooted_pointer(3) as *const crate::StringHeader, + 7.0, + ); + assert_eq!( + f64::from_bits( + js_object_get_field_by_name( + rooted_pointer(1) as *const ObjectHeader, + rooted_pointer(2) as *const crate::StringHeader, + ) + .bits() + ), + 3.0, + ); + assert_eq!( + f64::from_bits( + js_object_get_field_by_name( + rooted_pointer(1) as *const ObjectHeader, + rooted_pointer(3) as *const crate::StringHeader, + ) + .bits() + ), + 7.0, + ); + assert_eq!(crate::set::test_set_registry_probe_count(), set_before); + assert_eq!( + crate::symbol::test_symbol_registry_probe_count(), + symbol_before + ); +} + #[test] fn plain_object_miss_skips_set_and_symbol_registries() { leaked_symbol("perry-7867-arm-symbol"); diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index aba4cbe220..8893588a93 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -776,42 +776,10 @@ pub(super) fn get_field_ic_miss_impl( // `< 0x100000` proxy / HANDLE_PROPERTY_DISPATCH routing below — matching // the ordering in `js_object_get_field_by_name`. The macOS heap floor // (0x200_0000_0000 in is_valid_obj_ptr) masked this; Linux's is 0x1000. - // Lane 3 hook A, hoisted ABOVE the async-resource probe below. - // - // That probe costs 16.0 instructions per call once its latch is armed (a - // thread-local registry lookup), and it ran on every inherited read before - // this one could answer. The lookup cannot be confused by an async - // resource handle: those are `Box::into_raw` native allocations outside - // the GC arena, so their word at payload +4 is the high half of a small - // counter rather than a live ShapeId, `object_shape_stamp` answers 0, and - // the lookup returns `Unknown` in about ten instructions without - // dereferencing anything further. See the rule-3 note in - // `object::inherited_read_cache`. // Charter step 5: a receiver still carrying a shape whose lane the // lineage generalized moves to the normalized shape before anything is // learned from it, so the site converges instead of going polymorphic. unsafe { crate::object::field_rep_store::migrate_on_miss(obj as usize) }; - let mut inherited_declined = false; - if crate::value::addr_class::is_above_handle_band(obj as usize) { - // Lane 3 hook A: an INHERITED read that this site has already resolved - // once. Placed before the ladder rather than after it because the - // whole point is the ladder: an inherited read otherwise re-walks the - // chain on every read (~1300 instructions, measured). The guard proves - // the receiver is a GC_TYPE_OBJECT itself, so nothing below has been - // skipped on its behalf; see `object::inherited_read_cache`. - match unsafe { crate::object::inherited_read_cache::inherited_read_cache_lookup(obj, key) } - { - crate::object::inherited_read_cache::Lookup::Hit(value) => { - if diag { - ic_diag_note(cache_slot, key, R::NotOwn); - } - return f64::from_bits(value.bits()); - } - crate::object::inherited_read_cache::Lookup::Declined => inherited_declined = true, - crate::object::inherited_read_cache::Lookup::Unknown => {} - } - } - if !key.is_null() && crate::async_hooks::is_async_resource_handle(obj as i64) { unsafe { if let Some(name) = crate::string::header_str_checked(key) { @@ -826,12 +794,6 @@ pub(super) fn get_field_ic_miss_impl( } } } - // Lane 3 hook A's answer, carried to hook B at the bottom of this - // function: `Declined` means the chain walk has already been tried for - // this (receiver shape, key) and refused, so hook B must not try it again. - // Without that, every read the cache CANNOT serve pays for a full chain - // walk per read — measured at +424 instructions per read for an accessor - // on the prototype, a regression against no cache at all. // ONE validated header read classifies the receiver for everything below. // `try_read_gc_header` rejects the handle band and implausible addresses // without touching memory, so `None` here is "not a heap cell" and the @@ -839,9 +801,6 @@ pub(super) fn get_field_ic_miss_impl( // to read the same header three more times (kind, descriptor flag, // forwarding flag); it now takes all three from this one read. // - // Hook A and `inherited_declined` are NOT re-declared here: #10842 hoisted - // them above the async-resource probe, so this commit's copy would be a - // second lookup per read and a shadowed binding. let gc_header = unsafe { crate::value::addr_class::try_read_gc_header(obj as usize) }; let gc_kind = gc_header.map(|h| h.obj_type); if crate::value::addr_class::is_above_handle_band(obj as usize) { @@ -1109,11 +1068,8 @@ pub(super) fn get_field_ic_miss_impl( let value = js_object_get_field_by_name(obj, key); return f64::from_bits(value.bits()); }; - // #10868 step 2.5 stage 1: "no keys array" implies "no own - // properties" for every receiver EXCEPT a dictionary-mode one, - // whose key list lives in its `ObjectMeta`. Priming the - // inherited-read cache on that claim would answer an OWN property - // from the prototype chain — a wrong value, not a slow one. + // A dictionary-mode receiver stores its key list in ObjectMeta; + // the no-keys-array shortcut cannot classify it as empty. if crate::object::dictionary::is_dictionary(obj) { let value = js_object_get_field_by_name(obj, key); return f64::from_bits(value.bits()); @@ -1123,39 +1079,14 @@ pub(super) fn get_field_ic_miss_impl( if diag { ic_diag_note(cache_slot, key, R::ObjectNoKeys); } - // #10834 gated its only prime site on `miss_reason == NotOwn`, - // and this arm returns before reaching it. A receiver with no - // keys array has NO own properties at all, so "the key is not - // an own property" holds here MORE strongly than it does for - // `NotOwn` — and this is the single most common inherited-read - // shape there is: `Object.create(p)` with nothing of its own. - // - // Without this the lookup at the top of this function runs on - // every such read, always misses because nothing can ever be - // recorded, and the chain walk proceeds unchanged: measured at - // +106 instructions per read against the same binary with - // `PERRY_INHERITED_IC=0`, i.e. the cache was pure overhead for - // this shape. - // The site's holder entry: primed here, answered by the - // emitted tower from then on (`method_site::read_holder`). + // An empty ordinary receiver has no own keys. Prime the + // site's holder-shape answer before the generic walk. if let Some(value) = crate::object::method_site::read_holder::prime_read_holder(obj, key, cache_slot) { return f64::from_bits(value.bits()); } - if !inherited_declined { - // Already inside this function's `unsafe` block (line 874), - // so a nested one is `unused_unsafe` under -D warnings. - if let Some(value) = - crate::object::inherited_read_cache::inherited_read_cache_prime(obj, key) - { - return f64::from_bits(value.bits()); - } - } - // Past the cache, not through it: the lookup at the top of - // this function has already asked. - let value = - super::get_field_by_name::get_field_by_name_past_inherited_cache(obj, key); + let value = super::get_field_by_name::get_field_by_name_after_site_miss(obj, key); return f64::from_bits(value.bits()); } // #10939: `header + 8` is not where a keys array's elements @@ -1291,11 +1222,8 @@ pub(super) fn get_field_ic_miss_impl( if diag { ic_diag_note(cache_slot, key, miss_reason); } - // Lane 3 hook B: the own-key search above has failed, so this is the one - // place in the runtime that KNOWS the key is not an own property without - // paying for a second search. Walk the chain once and record the answer. - // A decline leaves the generic getter below untouched, which is today's - // behaviour for every case the cache refuses. + // The own-key search above has failed. Record a holder-shape answer at + // the site when this receiver and chain admit one. if matches!(miss_reason, R::NotOwn) { // The site's holder entry (`method_site::read_holder`). if let Some(value) = unsafe { @@ -1304,16 +1232,7 @@ pub(super) fn get_field_ic_miss_impl( return f64::from_bits(value.bits()); } } - if matches!(miss_reason, R::NotOwn) && !inherited_declined { - if let Some(value) = - unsafe { crate::object::inherited_read_cache::inherited_read_cache_prime(obj, key) } - { - return f64::from_bits(value.bits()); - } - } - // Past the cache, not through it: hook A above has already asked, and for - // the reads this cache refuses that question is the whole added cost. - let value = super::get_field_by_name::get_field_by_name_past_inherited_cache(obj, key); + let value = super::get_field_by_name::get_field_by_name_after_site_miss(obj, key); f64::from_bits(value.bits()) } @@ -1505,16 +1424,6 @@ pub(crate) fn get_field_ic_dispatch( // declines still reaches the handler below, so this only ever removes // work. See `pic_outlined_mru_hit`. // - // The inherited-read hook the inline tower emits on its declined edge - // (`js_inherited_read_cache_hit_f64`) is deliberately NOT mirrored - // here: this entry is already inside the runtime, so the cost that - // hook removes for an inline site (the slow entry's prologue and - // dispatch) is already paid, and `get_field_ic_miss_impl` asks the - // same cache first thing for a heap receiver (hook A). The two - // programs answer from the same lookup in the same order — own hit, - // then the inherited cache, then the ladder — so they stay - // behaviourally identical with one call fewer here. - // // POINTER tag only, exactly as the emitted tower tests it (#10833): // the hit compares the receiver's `+4` word against a ShapeId with no // GC-kind test in front of it any more, and #10828's guarantee that diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index e4f119976c..7330fc6ce0 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -238,28 +238,6 @@ pub extern "C-unwind" fn js_object_get_field_ic_slow( cache_slot: *mut PicCacheSlot, packed: *const AtomicU64, ) -> f64 { - // First-read D3: the site asked its GC-leaf front - // (`read_confirm::js_object_get_field_ic_front`) first; what reaches this - // entry is what the front declined. A never-primed site asks the - // inherited-read cache (#10834/#10842) — the one edge an inherited read - // ever takes — and everything else runs the collecting body. - let addr = obj_handle as usize; - if !key.is_null() - && crate::value::addr_class::is_above_handle_band(addr) - // SAFETY: the site passes its own cache slot or null. - && unsafe { crate::object::pic_slot_peek(cache_slot) }.is_null() - { - // SAFETY: a POINTER-tagged payload above the handle band. - let v = unsafe { - crate::object::inherited_read_cache::js_inherited_read_cache_hit_f64( - addr as *const ObjectHeader, - key, - ) - }; - if v.to_bits() != crate::value::TAG_HOLE { - return v; - } - } ic_slow_body(obj_handle, key, cache_slot, packed) } diff --git a/crates/perry-runtime/src/object/inherited_read_cache.rs b/crates/perry-runtime/src/object/inherited_read_cache.rs deleted file mode 100644 index 3a222712b8..0000000000 --- a/crates/perry-runtime/src/object/inherited_read_cache.rs +++ /dev/null @@ -1,1685 +0,0 @@ -//! Inherited-read cache: `(receiver shape, key) -> (holder, inline slot)`. -//! -//! # The hole this fills -//! -//! Every property inline cache in Perry serves OWN data properties only. A -//! read whose key lives on the prototype chain therefore misses every cache -//! and re-runs the whole generic getter, which walks the chain from scratch on -//! every read. Callgrind on `const P={a:1}; const O=Object.create(P); O.a` (a -//! one-hop chain, perfectly monomorphic, v0.5.1619, 200 k reads): -//! -//! | cost per read | instr | -//! |---|---| -//! | `native_get::try_data_get_bytes` self (the probe ladder) | 344 | -//! | `class_registry::prototype_objects::class_prototype_object` | 192 (118 of it SipHash) | -//! | `keys_find_slot_by_bytes_resolved` x2 (receiver keys, then holder keys) | 180 | -//! | `ic_miss::get_field_ic_miss_impl` self (the own-key search that must fail first) | 153 | -//! | `has_property::closure_dynamic_prop_by_key` | 90 | -//! | `shapes::shape_descriptor_by_id` | 70 | -//! | `is_anon_shape_class_id` / `class_decl_prototype_object` / `from_utf8` / `is_arguments_object` | 161 | -//! | total | ~1300 | -//! -//! node and bun both serve the same read for the same price as an own read -//! (measured: 16.2 vs 16.0, and 14.2 vs 13.3). Nothing in that 1300 is a -//! prototype MUTEX or an address-keyed prototype probe — `OBJECT_PROTOTYPES` -//! never appears in the profile, because #6759 phase B already moved shaped -//! objects onto `ObjectMeta.prototype`. The cost is the walk itself being -//! redone, plus a SipHash class-registry probe per read to find the prototype -//! of an `Object.create` receiver. -//! -//! # What an entry claims, and what makes the claim true -//! -//! An entry says: *a receiver whose (class id, ShapeId, recorded prototype -//! bits) are these, reading this interned key, finds it as a plain inline data -//! slot `slot` on `hops[hop_count-1]`, having passed through `hops[0..]` in -//! order.* -//! -//! The claim is re-proved on every hit by all four checks below. The code -//! runs them cheapest-and-most-selective first, not in the order they are -//! listed: -//! -//! 1. `proto_validity::proto_validity()` is unchanged. ONE global word, ONE -//! load, ONE compare, covering a chain of ANY depth. It stands for two -//! things at once: -//! * no object anybody inherits from has changed STRUCTURALLY. An object -//! is marked (`OBJ_FLAG_IS_PROTOTYPE`) when this cache records it as a -//! hop, and every shape-word change on a marked object bumps the counter -//! from the runtime's single structural-mutation publication funnel. A -//! key ADDED to a prototype bumps no epoch — a plain store is not a -//! descriptor install — and this is what sees it. -//! * nothing the semantic property epoch stands for has happened: -//! descriptor installs and clears, `delete`, per-instance prototype -//! recording (`Object.setPrototypeOf`, `__proto__`), -//! class-prototype-object registration, parent-static linking. -//! `prop_plan_epoch_bump` bumps this word too. It is deliberately NOT -//! bumped by GC, which is what keeps this cache off the #7910 cliff (the -//! full `PROP_PLAN_EPOCH` is bumped at loop-poll cadence by the -//! incremental collector, so keying on it degrades any cache into an -//! unconditional recompute). -//! -//! This REPLACED one ShapeId compare per hop. The per-hop walk was correct -//! and it had two costs: it was proportional to chain depth, up to four -//! dependent loads through prototype objects that are usually cold; and it -//! was a LOOP, so the hit could only ever live behind a call. An emitted -//! read site cannot branch on a variable number of compares. -//! 2. The receiver's `(class_id, ShapeId)` pair — ONE aligned 8-byte load at -//! offset 0 — equals what was recorded. This is what makes a shadowing own -//! key safe: adding `o.a` to the receiver is a key-add transition, which -//! mints a different ShapeId, so the entry simply stops matching. Priming -//! requires the key to be absent from the receiver's key list ENTIRELY (not -//! present as a `TAG_HOLE` tombstone), so no stable-tombstone re-add can -//! reinstate an own key under an unchanged ShapeId. -//! 3. The receiver's recorded prototype bits (`ObjectMeta.prototype`, 0 when -//! there is no meta record) are unchanged. This is NOT redundant with the -//! validity word: `object_link_class_default_prototype` links a FRESH -//! instance to its class's prototype object without bumping any epoch and -//! without transitioning a shape (by design — the loud variant flushed the -//! plan cache on every construction). A later `C.prototype = other` -//! followed by `new C()` therefore produces a receiver with the SAME class -//! id and the SAME ShapeId as the cached one and a different chain, and -//! this compare is what refuses it. -//! 4. The key pointer is identical. Keys are interned, so pointer identity is -//! key identity — see the GC contract below for why the pointer cannot be -//! reused by a different string while an entry names it. -//! -//! Conditions that are proved ONCE, at prime time, and held afterwards by the -//! validity word rather than re-checked: object kind is `Ordinary`; no -//! accessor and no customized descriptor for this key anywhere on the chain -//! (both transition the shape of the object they are installed on, and every -//! hop is marked, so either would have bumped the counter); the slot is inline -//! rather than spilled. -//! -//! Conditions re-checked on every hit because they are properties of the -//! ADDRESS rather than of the shape, and all three now live in ONE word this -//! path already loads (`ObjectMeta::flags`, plus `elements` beside it): the -//! receiver is not `process.env`, not an `arguments` object, and carries no -//! `elements` store. The first two were two address-keyed registry probes -//! costing 14.0 and 5.0 instructions on every cached read — and, decisively -//! for the emitted sequence, a compiled read site could not have called -//! either one. -//! -//! # GC contract -//! -//! The collector moves objects, so a recorded holder address is a liability. -//! Three things together make it safe, and each is necessary: -//! -//! * **`scan_inherited_read_cache_roots_mut` MARKS every key, hop and the -//! holder**, so the -//! collector keeps them alive and rewrites this table's copy of their -//! addresses. A hit LOADS `holder + slot`, which is what separates this -//! cache from the transition cache #6759 phase 3 made weak: that one only -//! ever COMPARES addresses, so a dangling slot costs it a miss, while here -//! it would be a read of recycled memory returning a wrong value. The -//! retention is bounded by the table (512 keys, 512 x MAX_HOPS prototypes). -//! * **`prune_dead_inherited_cache_entries` drops entries whose key or any hop -//! the collector reports dead.** Marking above means it should never have -//! one to drop; it is registered in `DEAD_KEY_PRUNES` anyway because that -//! registry is the runtime's checked list of tables re-keyed by a visitor -//! (`gc::dead_owner`'s #8174 note), and it is the only thing standing -//! between a recycled address and a false hit should a collection flavour -//! ever sweep without running the scanner. `young_prune: None`, so the full -//! 512-entry walk runs on every flavour including minors - cheap at this -//! size. -//! * **Priming does NOT refuse a nursery hop.** It did, and that made the -//! cache useless for the programs that need it most: a read-only loop -//! allocates nothing, nothing is ever promoted, and every prototype stays in -//! the nursery for the life of the process. Measured cost of that refusal: -//! +264 instructions per inherited read, for a chain walk performed and then -//! discarded. -//! -//! # Refusals are recorded too -//! -//! Most reads that reach this cache are ones it cannot serve: an accessor on -//! the prototype, a key that is on no prototype at all, a receiver whose kind -//! it refuses. If a refusal is not recorded, each of those pays for a full -//! chain walk on EVERY read and the cache is a net loss — measured at +424 -//! instructions per read for an accessor on the prototype, against a build -//! with no cache at all. -//! -//! So a declining walk writes a NEGATIVE entry (`slot == NEGATIVE_SLOT`) and -//! [`inherited_read_cache_lookup`] answers `Declined`, which tells -//! `get_field_ic_miss_impl` not to walk. A negative entry can never return a -//! wrong value — the worst it can do is keep a read on the path it is already -//! on — so its invalidation may be weaker than a hit's. It reuses the hit's -//! identity, epoch and per-hop stamp compares unchanged, which is what lets -//! `proto.a = 1` after a failed lookup re-open the pair. -//! -//! One class of refusal is NOT recorded: one caused by a VALUE — an -//! `undefined`, `null` or hole in the holder's slot. A plain store can replace -//! such a value with a real one while transitioning no shape and bumping no -//! epoch, so a negative entry for it would stand for the life of the process. -//! Every other refusal is a function of a shape or a descriptor. -//! -//! # Why every hop is still recorded when only the holder is read -//! -//! The hit loads `holder + slot` and never touches an interior hop, so the -//! `hops` array exists for the collector, not for the read. Dropping the -//! interior hops would make this entry able to hit for a receiver whose -//! recorded prototype bits name an address the collector recycled: the -//! intermediate prototype is kept alive by the receiver in every real -//! program, but this table does not root receivers, so nothing else is -//! keeping it from being freed and its address reused. Rooting the whole -//! chain costs GC time and nothing at read time. -//! -//! # Hook points for lane 4 -//! -//! Two, both one call wide: -//! * `inherited_read_cache_hit` (test-only) — the guard-and-load. Called at the top of -//! `js_object_get_field_by_name` and of `get_field_ic_miss_impl`. -//! * [`inherited_read_cache_prime`] — the chain walk. Called from -//! `get_field_ic_miss_impl` only, at the point where the own-key search has -//! already failed, so it never duplicates work on an own read. -//! -//! An emitted-code hit would call `js_inherited_read_cache_hit_f64`, the -//! `extern "C"` wrapper at the bottom of this file, with the receiver's masked -//! pointer and the interned key. - -use super::{shapes, ObjectHeader}; -use crate::value::JSValue; -use std::sync::atomic::{AtomicU64, Ordering}; - -/// Direct-mapped, per thread. 512 entries x 128 bytes is 64 KB; the table is -/// boxed for the same reason `prop_plan`'s is (an oversized inline TLS block -/// overflows the ILP32 TLS layout on arm64_32). -const CACHE_SIZE: usize = 512; -const CACHE_MASK: usize = CACHE_SIZE - 1; - -/// Longest chain an entry can describe, counting `%Object.prototype%` as the -/// last hop when the walk reaches it. `Object.create(Object.create(...))` -/// towers beyond this decline and keep today's walk. Ten covers a miss that -/// falls off an eight-object chain above the receiver (#10877's deepest -/// measured case) plus `%Object.prototype%`; four, the previous bound, stopped -/// at two user levels once the terminal became a hop of its own. The hit never -/// reads `hops`, so depth costs table bytes (48 per entry over the old bound, -/// 24 KB per thread for the whole table) and GC-scan work, not instructions -/// on a read. -const MAX_HOPS: usize = 10; - -#[derive(Clone, Copy)] -#[cfg_attr(test, derive(PartialEq, Debug))] -struct Entry { - /// Interned key pointer. 0 marks the slot empty. - key_ptr: usize, - /// `proto_validity::proto_validity()` at prime time. - validity: u64, - /// The receiver's `ObjectMeta.prototype` at prime time, 0 for no record. - recv_proto_bits: u64, - recv_class_id: u32, - recv_shape: u32, - /// The object the key was found on: `hops[hop_count - 1]`, kept in its own - /// field so the hit loads it at a fixed offset instead of indexing. - holder: usize, - /// Chain from the receiver's prototype (`hops[0]`) to the holder. Read by - /// the GC hooks only; the hit never walks it. - hops: [usize; MAX_HOPS], - hop_count: u8, - /// Inline field index on the holder. Spilled fields never prime. - slot: u32, - /// The holder's key at `slot` is an ACCESSOR (charter step 3): its slot - /// holds the pair (`accessor_pair.rs`) and a hit calls the getter with the - /// receiver as `this` instead of returning the slot's value. - accessor: bool, -} - -/// `slot` for a NEGATIVE entry: one that records that a walk from this -/// (receiver shape, key) pair declined, so the walk is not re-run on every -/// read. A real slot is an inline field index, so this value cannot collide -/// with one. -/// -/// Without it, every read the cache REFUSES pays for the refusal: an accessor -/// on the prototype measured 2912 instructions per loop with no cache and -/// 3336 with one, because the chain walk ran and was thrown away every time. -/// A negative entry is never wrong — it only ever says "do what you did -/// before" — so its invalidation may be weaker than a hit's, and it reuses -/// the hit's identity, epoch and per-hop compares unchanged. -/// **REQUIREMENT for any sentinel added beside this one.** Keep it at the TOP -/// of the `u32` range, adjacent to this value and descending. -/// -/// A real slot is an inline field index, bounded by the shape's -/// `live_inline_slot_count`. The emitted per-site publication being built for -/// inherited reads may publish an entry ONLY when it names a real holder and -/// slot, and with every sentinel above every valid index that predicate is a -/// single unsigned compare (`slot < live_inline_slot_count`) which excludes -/// all of them by construction. A sentinel placed anywhere else turns that one -/// compare into an enumeration that has to be extended every time somebody -/// adds a verdict — and the failure mode of forgetting is publishing a -/// sentinel to a site as if it were a field offset. -const NEGATIVE_SLOT: u32 = u32::MAX; - -/// `slot` for an ABSENT entry: the walk reached the end of the chain -/// (`%Object.prototype%`, whose `[[Prototype]]` is null) without finding the -/// key on any hop, and the generic getter confirmed `undefined` for the same -/// read. A hit answers `undefined` without loading anything. `holder` names -/// the last hop, so the collector keeps every address in the entry alive and -/// rewritten exactly as it does for a data entry. -/// -/// Adjacent to [`NEGATIVE_SLOT`] and descending, per the requirement stated -/// there. -const ABSENT_SLOT: u32 = u32::MAX - 1; - -/// `validity` of an entry that has been written but not yet CONFIRMED by the -/// generic getter (see [`inherited_read_cache_prime`]). The counter starts at -/// 1, so this value never matches and a pending entry can never be served — -/// not even to a read re-entered from inside the confirming getter — while the -/// collector still sees (and rewrites) every address it names. -const PENDING_VALIDITY: u64 = 0; - -/// What a table lookup found. -pub(crate) enum Lookup { - /// An entry proved its claim; this is the value. - Hit(JSValue), - /// A walk from this pair declined last time, under conditions that still - /// hold. The caller must not walk again. - Declined, - /// Nothing recorded. - Unknown, -} - -const EMPTY_ENTRY: Entry = Entry { - key_ptr: 0, - validity: 0, - recv_proto_bits: 0, - recv_class_id: 0, - recv_shape: 0, - holder: 0, - hops: [0; MAX_HOPS], - hop_count: 0, - slot: 0, - accessor: false, -}; - -// SAFETY: integer and `bool` fields only; `EMPTY_ENTRY` is all-zero (#11507). -unsafe impl crate::zeroed_cache::ZeroEmpty for Entry {} - -crate::perry_thread_local! { - static INHERITED_READ_CACHE: std::cell::UnsafeCell> = - std::cell::UnsafeCell::new(crate::zeroed_cache::new_zeroed_cache(CACHE_SIZE)); -} - -/// An entry is identified by (class id, ShapeId, key), so all three have to -/// reach the slot index. -/// -/// #10834 hashed only (shape, key). That is exactly wrong for the receivers -/// this cache exists to serve: `js_object_create` mints a FRESH synthetic -/// class id on every call, so N objects built by `Object.create(p)` have N -/// different class ids and ONE identical shape. Under a (shape, key) index -/// they all landed in the same direct-mapped slot and evicted one another, so -/// a site reading through eight of them primed on EVERY read and hit never: -/// measured `primes=6295655 hits=0` over ten million reads, a full chain walk -/// plus an entry write per read, +75 instructions against the same binary with -/// the cache off. -#[inline(always)] -fn entry_index(class_id: u32, shape: u32, key_ptr: usize) -> usize { - // Interned key pointers are 8- or 16-byte aligned, so their low bits are - // zeros; fold the middle bits down before masking. - let h = ((key_ptr >> 4) as u64 ^ ((shape as u64) << 21) ^ ((class_id as u64) << 43)) - .wrapping_mul(0x9E37_79B9_7F4A_7C15); - (h >> 40) as usize & CACHE_MASK -} - -/// `PERRY_INHERITED_IC=0` turns the cache off in a binary that has it, so the -/// same build can be measured with and without one environment variable apart -/// (the discipline `PERRY_IC_OUTLINE_FASTPATH` established). Nothing branches -/// on it for behaviour: both settings answer identically. -#[inline] -fn cache_enabled() -> bool { - static ON: std::sync::OnceLock = std::sync::OnceLock::new(); - *crate::once_init::get_or_init(&ON, || { - crate::gc::env_default_on_from_value(std::env::var("PERRY_INHERITED_IC").ok().as_deref()) - }) -} - -// --- hit counters ----------------------------------------------------------- -// -// A cache that silently falls through to the walk is correct-but-slow and -// invisible in a program's OUTPUT. These make the hit itself assertable. - -static HITS: AtomicU64 = AtomicU64::new(0); -static PRIMES: AtomicU64 = AtomicU64::new(0); -static DECLINES: AtomicU64 = AtomicU64::new(0); -/// Declines answered from a NEGATIVE entry, i.e. without walking. Separate -/// from `DECLINES` because "the cache refused" and "the cache refused for the -/// price of one lookup" are different facts, and only the second one is the -/// claim `NEGATIVE_SLOT` exists to make. -static NEG_SERVED: AtomicU64 = AtomicU64::new(0); - -/// Counting is gated at the CALL SITE by this latch so a default-off -/// diagnostic costs one predictable branch, not an atomic add, per read. -/// Tests need the counters whatever environment they run under, and a -/// `cfg(test)` build is not a build anyone measures. -/// -/// `PERRY_IC_DIAG` arms it too, and `IcDiag::render` prints the three counts -/// in the `[ic-diag]` report: the existing instrument for "what did the -/// property-read caches actually do", extended rather than duplicated. Both -/// inputs are read from the environment once, so this stays one `OnceLock` -/// load however it was armed. -#[inline] -fn stats_enabled() -> bool { - #[cfg(test)] - { - true - } - #[cfg(not(test))] - { - static ON: std::sync::OnceLock = std::sync::OnceLock::new(); - *crate::once_init::get_or_init(&ON, || { - std::env::var_os("PERRY_INHERITED_IC_STATS").is_some() || crate::hot_diag::ic_on() - }) - } -} - -pub(crate) fn inherited_read_cache_hits() -> u64 { - HITS.load(Ordering::Relaxed) -} - -pub(crate) fn inherited_read_cache_primes() -> u64 { - PRIMES.load(Ordering::Relaxed) -} - -pub(crate) fn inherited_read_cache_declines() -> u64 { - DECLINES.load(Ordering::Relaxed) -} - -pub(crate) fn inherited_read_cache_neg_served() -> u64 { - NEG_SERVED.load(Ordering::Relaxed) -} - -/// Counters for a caller that wants a delta over a region (tests). -#[cfg(test)] -pub(crate) fn test_reset_counters() { - HITS.store(0, Ordering::Relaxed); - PRIMES.store(0, Ordering::Relaxed); - DECLINES.store(0, Ordering::Relaxed); - NEG_SERVED.store(0, Ordering::Relaxed); -} - -#[cfg(test)] -pub(crate) fn test_clear_cache() { - INHERITED_READ_CACHE.with(|cell| unsafe { - for entry in (*cell.get()).iter_mut() { - *entry = EMPTY_ENTRY; - } - }); -} - -// --- shared predicates ------------------------------------------------------ - -/// The per-object facts a ShapeId does NOT pin, checked on the receiver at -/// both prime and hit time. -/// -/// `process.env` is an OS-backed exotic object and an `arguments` object has -/// its own index semantics; both are plain `GC_TYPE_OBJECT`s whose key list — -/// and therefore whose ShapeId — an ordinary object can coincide with. An -/// `elements` store is array-subclass backing that answers reads before the -/// shape does. -#[inline] -unsafe fn receiver_address_facts_ok(meta: *const crate::object::ObjectMeta) -> bool { - if meta.is_null() { - return true; - } - (*meta).elements == 0 - && (*meta).flags & crate::object::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER == 0 -} - -/// An address a prime may record: one this heap knows, so its `GcHeader` is -/// readable and the collector's hooks reach it. -/// -/// This deliberately does NOT restrict priming to the old generation. It did, -/// on the argument that a minor can then never invalidate an entry — and that -/// made the cache useless for precisely the programs that need it most. A loop -/// that only READS allocates nothing, so nothing is ever promoted, so every -/// prototype stays in the nursery for the life of the process and every prime -/// declined: measured +264 instructions per inherited read (1433 -> 1697) for -/// a walk that was performed and then thrown away. Nursery hops are safe -/// because the two GC hooks below cover them, which is the same contract every -/// other address-recording cache in the runtime lives under. -#[inline] -fn address_is_prime_stable(addr: usize) -> bool { - crate::value::addr_class::is_plausible_heap_addr(addr) - && crate::arena::classify_heap_generation(addr) != crate::arena::HeapGeneration::Unknown -} - -// --- the hit ---------------------------------------------------------------- - -/// Serve `obj.key` from a cached inherited entry, or decline. -/// -/// # Safety -/// `obj` is a masked, non-null heap pointer the caller has already established -/// is a plausible heap address; `key` may be null. -#[cfg(test)] -#[inline] -pub(crate) unsafe fn inherited_read_cache_hit( - obj: *const ObjectHeader, - key: *const crate::StringHeader, -) -> Option { - match inherited_read_cache_lookup(obj, key) { - Lookup::Hit(value) => Some(value), - Lookup::Declined | Lookup::Unknown => None, - } -} - -/// The entry recorded for `obj.key`, once every per-hit check in the module -/// header has passed; otherwise what the caller must answer (`Declined` for a -/// valid negative entry, `Unknown` for anything else). Shared by the read and -/// the write side, so both prove an entry the same way. -/// -/// # Safety -/// As [`inherited_read_cache_lookup`]. -#[inline(always)] -unsafe fn proved_entry( - obj: *const ObjectHeader, - key: *const crate::StringHeader, -) -> Result { - if key.is_null() || !cache_enabled() { - return Err(Lookup::Unknown); - } - let addr = obj as usize; - if !crate::value::addr_class::is_plausible_heap_addr(addr) { - return Err(Lookup::Unknown); - } - // The receiver's identity word: class id at +0, ShapeId at +4. One load, - // taken BEFORE the kind is proved, so it must be safe on any - // pointer-tagged value that can reach here. - // - // #10828 proves rule 3 ("no non-object cell holds a live ShapeId at +4") - // over GC cell KINDS. Three pointer-tagged values are NOT in that table: - // `SymbolHeader`, `AsyncHookHandle` and `AsyncResourceHandle` are - // `Box::into_raw` native allocations outside the arena. Checked here - // because this read reaches them, and because the emitted sequence this - // cache is being built toward will fold both words into ONE 8-byte load - // and one compare: - // - // * `SymbolHeader` (24 bytes): +0 is `magic` = 0x5359_4D42, +4 is - // `registered`, which is 0 or 1. Both reads are in bounds and neither - // word can be a live ShapeId, which start at 0x8000_0000. Safe BY - // CONSTRUCTION. - // * `AsyncHookHandle` (8 bytes): the whole payload is `index: usize`, a - // Vec index, so +4 is its high half — zero. In bounds, cannot collide. - // Safe BY CONSTRUCTION. - // * `AsyncResourceHandle` (24 bytes): +0 is `ids.async_id: u64`, a - // monotonic counter, so +4 is its high half. In bounds, and zero until - // a single process creates 2^32 async resources. This is the one of the - // three that is safe BY MAGNITUDE rather than by construction, i.e. the - // same class of argument #10824 refused for buffer capacities. It is not - // load-bearing here — `is_shape_id` below rejects a zero word anyway — - // but an emitted guard that drops that test would be resting on it. - // - // What actually protects this path is the `is_shape_id` range test inside - // `object_shape_stamp`: a word outside [0x8000_0000, 0xC000_0000) answers - // 0 and returns `Unknown` two instructions later. The emitted form keeps - // the same protection for free, because a site's expected ShapeId is - // always in that range, so a word that is not cannot match it. - let recv_class_id = (*obj).class_id; - let recv_shape = shapes::object_shape_stamp(obj); - // 0 = no ShapeId at +4; an exotic-band id is a non-object receiver (a - // function), which this cache does not serve (yet): it answers a - // `GC_TYPE_OBJECT` chain only. - if recv_shape == 0 || shapes::is_exotic_shape_id(recv_shape) { - return Err(Lookup::Unknown); - } - let index = entry_index(recv_class_id, recv_shape, key as usize); - // Read in place: an `Entry` is ~100 bytes and a hit needs four words of it. - let entry: &Entry = &*INHERITED_READ_CACHE.with(|cell| (*cell.get()).as_ptr().add(index)); - if entry.key_ptr != key as usize - || entry.recv_shape != recv_shape - || entry.recv_class_id != recv_class_id - { - return Err(Lookup::Unknown); - } - // One load, one compare, whatever the depth of the chain. See the module - // header: this word covers both the semantic property epoch and every - // structural mutation of an object marked as somebody's prototype. - if entry.validity != crate::object::proto_validity::proto_validity() { - return Err(Lookup::Unknown); - } - if entry.slot == NEGATIVE_SLOT { - if stats_enabled() { - NEG_SERVED.fetch_add(1, Ordering::Relaxed); - } - return Err(Lookup::Declined); - } - // An ABSENT entry reads no holder slot, so it skips nothing below: the - // receiver's kind, recorded prototype and address facts are proved for it - // exactly as for a data entry. - // Only NOW, once the entry has matched on three identities, is it worth - // proving the receiver really is an object. A non-object cell's word at - // +4 is a `capacity` or a `func_ptr` half (design doc rule 3), so the - // ShapeId compare above is not by itself a proof of kind. - // The header read below proves the kind: `addr` is heap-plausible, and a - // word at +4 inside the live ShapeId range is carried only by an object - // cell (#10828's rule 3; the three native `Box` allocations reachable - // here cannot carry one, see the note above), so `addr` is a cell start - // with a real header. The page-generation classification that used to - // precede it proved the same thing a second way, at ~35 instructions per - // access. - let Some(header) = crate::value::addr_class::try_read_gc_header_known_plausible(addr) else { - return Err(Lookup::Unknown); - }; - if header.obj_type != crate::gc::GC_TYPE_OBJECT - || header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 - { - return Err(Lookup::Unknown); - } - let meta = (*obj).meta; - let recv_proto_bits = if meta.is_null() { 0 } else { (*meta).prototype }; - if recv_proto_bits != entry.recv_proto_bits { - return Err(Lookup::Unknown); - } - - if !receiver_address_facts_ok(meta) { - return Err(Lookup::Unknown); - } - Ok(Proved { - holder: entry.holder, - slot: entry.slot, - accessor: entry.accessor, - }) -} - -/// The part of a proved entry a hit uses. -#[derive(Clone, Copy)] -struct Proved { - holder: usize, - slot: u32, - accessor: bool, -} - -/// The hit, plus the one other thing the table can say: that a walk from this -/// pair declined and must not be re-run. Only `get_field_ic_miss_impl` cares -/// about the difference, because it is the only caller that would otherwise -/// walk. -/// -/// # Safety -/// `obj` is a masked, non-null heap pointer the caller has already established -/// is a plausible heap address; `key` may be null. -#[inline] -pub(crate) unsafe fn inherited_read_cache_lookup( - obj: *const ObjectHeader, - key: *const crate::StringHeader, -) -> Lookup { - lookup_entry::(obj, key) -} - -/// [`inherited_read_cache_lookup`] for the DATA entries only: an accessor -/// entry answers `Unknown`. This instance has no edge to [`accessor_hit`], so -/// the GC-leaf callers (`js_inherited_read_cache_hit_f64`, and through it -/// `js_class_field_get_ic_fast`) provably never run user code. -/// -/// # Safety -/// As [`inherited_read_cache_lookup`]. -#[inline] -pub(crate) unsafe fn inherited_read_cache_lookup_data( - obj: *const ObjectHeader, - key: *const crate::StringHeader, -) -> Lookup { - lookup_entry::(obj, key) -} - -#[inline(always)] -unsafe fn lookup_entry( - obj: *const ObjectHeader, - key: *const crate::StringHeader, -) -> Lookup { - let entry = match proved_entry(obj, key) { - Ok(entry) => entry, - Err(answer) => return answer, - }; - if entry.slot == ABSENT_SLOT { - if stats_enabled() { - HITS.fetch_add(1, Ordering::Relaxed); - } - return Lookup::Hit(JSValue::undefined()); - } - let holder = entry.holder as *const ObjectHeader; - let field = (holder as *const u8) - .add(std::mem::size_of::() + entry.slot as usize * 8) - as *const u64; - let bits = *field; - if entry.accessor { - if SERVE_ACCESSORS { - return accessor_hit(obj, bits); - } - return Lookup::Unknown; - } - // A deleted holder slot is a `TAG_HOLE`. `delete` bumps the semantic epoch - // so this is unreachable today; it costs one compare and it is the check - // that makes the claim not depend on that. - if bits == crate::value::TAG_HOLE { - return Lookup::Unknown; - } - let value = JSValue::from_bits(bits); - // `try_data_get_bytes` treats an inherited `undefined`/`null` as a miss at - // some class edges, and priming refuses such a slot. A later store of - // `undefined` into the holder's slot can still produce one, so mirror it - // rather than diverge. - if value.is_undefined() || value.is_null() { - return Lookup::Unknown; - } - if stats_enabled() { - HITS.fetch_add(1, Ordering::Relaxed); - } - Lookup::Hit(value) -} - -/// Serve an accessor entry: the holder's slot holds the pair, and the getter -/// runs with the receiver as `this` — a compiled class getter directly, a -/// `defineProperty` getter as a closure, a setter-only accessor reads -/// `undefined`. The pair's functions are pinned by the holder's ShapeId (an -/// accessor replaced under unchanged attributes still transitions it, -/// `transition_object_shape_accessor_replaced`) and the holder is a marked -/// prototype, so the validity word already covers them. -/// -/// # Safety -/// `obj` is the live receiver the entry matched; `pair_bits` is the holder's -/// slot word for an accessor key. -#[inline] -unsafe fn accessor_hit(obj: *const ObjectHeader, pair_bits: u64) -> Lookup { - // An inherited-read walk in progress binds `this` to ITS receiver; such a - // read is never served here. - if crate::object::accessor_receiver_override_armed() { - return Lookup::Unknown; - } - let acc = crate::object::accessor_pair::pair_of_value_unchecked(pair_bits); - if stats_enabled() { - HITS.fetch_add(1, Ordering::Relaxed); - } - let this = f64::from_bits(crate::value::js_nanbox_pointer(obj as i64).to_bits()); - if acc.raw_get != 0 { - // A compiled class getter takes `this` as its parameter (the ABI - // `call_class_getter` used); nothing here publishes an implicit `this`. - // A read inside an inherited-property resolution is left to the - // generic path, which isolates the body from it (#11201). - if crate::object::prototype_chain::resolution_stack_savepoint() != 0 { - return Lookup::Unknown; - } - let f = crate::closure::body_call::js_method_body_fn!(acc.raw_get as *const u8;); - return Lookup::Hit(JSValue::from_bits(f(this).to_bits())); - } - if acc.get != 0 { - return Lookup::Hit(crate::object::invoke_accessor_getter(acc.get, this)); - } - Lookup::Hit(JSValue::undefined()) -} - -// --- the prime -------------------------------------------------------------- - -/// What a declining walk learned, so the decline can be recorded and the walk -/// not repeated. `armed` means the receiver identity and key are known, which -/// is the minimum a negative entry needs; `value_dependent` means the refusal -/// was caused by a VALUE (an `undefined`/`null`/hole in the holder's slot), -/// which a plain store can change with no shape transition and no epoch bump — -/// the one class of refusal that must NOT be remembered, or `proto.a = 1` -/// after a miss would leave the pair declined for the life of the process. -#[derive(Default)] -struct DeclineNote { - armed: bool, - value_dependent: bool, - key_ptr: usize, - recv_class_id: u32, - recv_shape: u32, - recv_proto_bits: u64, - holder: usize, - hops: [usize; MAX_HOPS], - hop_count: u8, - /// Set when the walk wrote a PENDING entry (see [`PENDING_VALIDITY`]) that - /// the generic getter must confirm before it may be served. - pending: Option, -} - -/// A written-but-unconfirmed entry: where it is, and what it has to still say -/// when the confirming getter returns. -#[derive(Clone, Copy)] -struct Pending { - index: usize, - recv_class_id: u32, - recv_shape: u32, - slot: u32, - /// `proto_validity()` when the walk ran. The entry commits with exactly - /// this value and only if the counter still holds it after the getter. - validity: u64, -} - -/// Walk `obj`'s prototype chain for `key`, and record the result — the holder -/// and slot when every condition in this module's contract holds, the refusal -/// itself when they do not. -/// -/// The caller must already have established that `key` is NOT in `obj`'s own -/// key list — `get_field_ic_miss_impl` has just searched it — so this never -/// duplicates an own-property search. -/// -/// Returns the value when the walk resolved it, `None` to leave the caller on -/// its existing path. A `None` is always safe: it is today's behaviour. -/// -/// # Safety -/// `obj` is a masked, non-null heap pointer; `key` may be null. -pub(crate) unsafe fn inherited_read_cache_prime( - obj: *const ObjectHeader, - key: *const crate::StringHeader, -) -> Option { - if key.is_null() || !cache_enabled() { - return None; - } - let mut note = DeclineNote::default(); - let result = inherited_read_cache_walk(obj, key, None, &mut note); - if let Some(pending) = note.pending { - return Some(confirm_pending(obj, key, pending)); - } - if result.is_none() { - record_decline(¬e); - } - result -} - -/// Answer a read whose walk left a PENDING entry, and decide that entry. -/// -/// The walk claims an answer from the chain's KEYS: `undefined` for an -/// ABSENT entry, the holder's slot for a data entry reached through the -/// default `%Object.prototype%` link. The generic getter is the arbiter of -/// what the read really answers — it also consults everything a key list does -/// not show (names the runtime synthesizes, lazily resolved intrinsics) — so -/// the entry is committed only when the two agree, and only when nothing the -/// validity word stands for happened while the getter ran. Otherwise the -/// entry becomes a NEGATIVE one (under the same condition) so the pair is not -/// walked again, or is dropped. Either way the getter's value is the answer. -/// -/// The getter may collect. Nothing here reuses `obj` or `key` after it: the -/// pending entry sits in the table, where the root scanner marks and rewrites -/// the key and every hop, and the verdict re-reads the holder from there. -/// -/// # Safety -/// As [`inherited_read_cache_prime`]. -unsafe fn confirm_pending( - obj: *const ObjectHeader, - key: *const crate::StringHeader, - pending: Pending, -) -> JSValue { - let answer = super::field_get_set::get_field_by_name_past_inherited_cache(obj, key); - INHERITED_READ_CACHE.with(|cell| { - let entry = &mut (*cell.get())[pending.index]; - // Someone else's entry now (a re-entrant prime from inside the - // getter): leave it alone. - if entry.key_ptr == 0 - || entry.validity != PENDING_VALIDITY - || entry.recv_class_id != pending.recv_class_id - || entry.recv_shape != pending.recv_shape - || entry.slot != pending.slot - { - return; - } - let agrees = if entry.slot == ABSENT_SLOT { - answer.is_undefined() - } else { - let field = (entry.holder as *const u8) - .add(std::mem::size_of::() + entry.slot as usize * 8) - as *const u64; - let bits = *field; - let value = JSValue::from_bits(bits); - bits == answer.bits() - && bits != crate::value::TAG_HOLE - && !value.is_undefined() - && !value.is_null() - }; - if crate::object::proto_validity::proto_validity() != pending.validity { - *entry = EMPTY_ENTRY; - return; - } - if !agrees { - // The chain's keys and the getter disagree about this pair — a - // synthesized name, a lazily resolved intrinsic. That is a fact - // about the shapes involved, not about a slot's value, so the - // refusal is remembered like any other (a negative entry is never - // wrong; it only keeps the read on the generic path). - entry.slot = NEGATIVE_SLOT; - entry.accessor = false; - if stats_enabled() { - DECLINES.fetch_add(1, Ordering::Relaxed); - } - } else if stats_enabled() { - PRIMES.fetch_add(1, Ordering::Relaxed); - } - entry.validity = pending.validity; - }); - answer -} - -/// A declining walk: write the NEGATIVE entry that stops the walk from being -/// re-run for this (receiver shape, key) — unless the refusal was caused by a -/// value (see [`DeclineNote`]). -unsafe fn record_decline(note: &DeclineNote) { - { - if stats_enabled() { - DECLINES.fetch_add(1, Ordering::Relaxed); - } - if note.armed && !note.value_dependent { - let entry = Entry { - key_ptr: note.key_ptr, - validity: crate::object::proto_validity::proto_validity(), - recv_proto_bits: note.recv_proto_bits, - recv_class_id: note.recv_class_id, - recv_shape: note.recv_shape, - holder: note.holder, - hops: note.hops, - hop_count: note.hop_count, - slot: NEGATIVE_SLOT, - accessor: false, - }; - let index = entry_index(note.recv_class_id, note.recv_shape, note.key_ptr); - INHERITED_READ_CACHE.with(|cell| { - (*cell.get())[index] = entry; - }); - } - } -} - -// --- the write side ------------------------------------------------------------ - -/// Run an inherited accessor's setter for `receiver.key = value`: the holder's -/// slot holds the pair; a compiled class setter is called directly with the -/// receiver as `this`, a `defineProperty` setter as a closure. `false` when -/// the accessor has no setter — the caller's generic `[[Set]]` then refuses -/// the write with the right strictness. -/// -/// # Safety -/// `obj` is the live receiver; `pair_bits` is the holder's slot word for an -/// accessor key. -#[inline] -unsafe fn accessor_set(obj: *const ObjectHeader, pair_bits: u64, value: f64) -> bool { - let acc = crate::object::accessor_pair::pair_of_value_unchecked(pair_bits); - let this = f64::from_bits(crate::value::js_nanbox_pointer(obj as i64).to_bits()); - if acc.raw_set != 0 { - // A compiled class setter is called directly with the receiver as its - // `this` parameter, exactly as the class-setter arm of the generic - // `[[Set]]` calls it (that arm opens no resolution boundary either). - let f = crate::closure::body_call::js_method_body_fn!(acc.raw_set as *const u8; value); - let _ = f(this, value); - return true; - } - if acc.set != 0 { - crate::object::invoke_accessor_setter(acc.set, this, value); - return true; - } - false -} - -/// `obj.key = value` where `key` is not an own property of `obj`: when the -/// table (or one walk, recorded) proves the key resolves on the chain to an -/// accessor with a setter, run it and answer `true`. `false` leaves the write -/// to the caller's generic `[[Set]]` — a data holder, a getter-only accessor, -/// a refusal. The same entries serve reads (`inherited_read_cache_lookup`): -/// an entry is a fact about `(receiver shape, key)`, not about the access. -/// -/// # Safety -/// `obj` is a masked, non-null heap pointer the caller has already established -/// is a plausible heap address; `key` may be null. -pub(crate) unsafe fn inherited_write_through( - obj: *const ObjectHeader, - key: *const crate::StringHeader, - value: f64, -) -> bool { - match proved_entry(obj, key) { - Ok(entry) => { - if !entry.accessor { - return false; - } - let field = (entry.holder as *const u8) - .add(std::mem::size_of::() + entry.slot as usize * 8) - as *const u64; - if stats_enabled() { - HITS.fetch_add(1, Ordering::Relaxed); - } - accessor_set(obj, *field, value) - } - Err(Lookup::Unknown) => { - if key.is_null() || !cache_enabled() { - return false; - } - // An entry claims the key is NOT own on this shape (the ShapeId - // then keeps that true), so only a receiver without the key walks. - if !crate::object::object_is_shaped(obj) { - return false; - } - let keys = crate::object::object_keys(obj); - let len = (*key).byte_len as usize; - if len > (*key).capacity as usize { - return false; - } - let bytes = std::slice::from_raw_parts(crate::string::string_data(key), len); - if !keys.is_null() - && crate::object::keys_find_slot_by_bytes(keys.arr(), keys.count(), bytes).is_some() - { - return false; - } - let mut note = DeclineNote::default(); - let handled = inherited_read_cache_walk(obj, key, Some(value), &mut note).is_some(); - if !handled && proved_entry(obj, key).is_err() { - // The walk recorded nothing (a data holder records a data - // entry): record the decline, so a store that adds a key, or - // any other refused write, is not walked again for this - // receiver shape. - record_decline(¬e); - } - handled - } - Err(_) => false, - } -} - -/// Prime from a BY-NAME read (`o[k]`, `js_object_get_field_by_name`), after the -/// table has answered `Unknown` and the own/inherited data probe has missed. -/// -/// `get_field_ic_miss_impl` is the only other prime site, and it primes where -/// its own-key search has just failed. A by-name read has no such search in -/// hand, so this one proves the precondition itself: the receiver is a -/// shaped, non-dictionary ordinary object and the key is not in its key list -/// AT ALL (a tombstoned own key counts as present, exactly as the hit's -/// ShapeId argument requires). Anything else leaves the read on its existing -/// path. -/// -/// Without it, a computed-key read that misses — `table[k] || dflt`, the -/// absent half of #10753 — re-ran the whole generic walk on every execution, -/// because nothing ever recorded what that walk found. -/// -/// # Safety -/// `obj` is the raw receiver `js_object_get_field_by_name` was handed; `key` -/// may be null. -pub(crate) unsafe fn inherited_read_cache_prime_by_name( - obj: *const ObjectHeader, - key: *const crate::StringHeader, -) -> Option { - if key.is_null() || !cache_enabled() { - return None; - } - // The generic walk's own recursive reads (a prototype hop read on behalf - // of another receiver) arrive here with the accessor receiver armed. They - // are part of a walk already in progress, not a read site of their own. - if crate::object::accessor_receiver_override_armed() { - return None; - } - let addr = obj as usize; - if !crate::value::addr_class::is_plausible_heap_addr(addr) { - return None; - } - let header = crate::value::addr_class::try_read_gc_header_known_plausible(addr)?; - if header.obj_type != crate::gc::GC_TYPE_OBJECT - || header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 - { - return None; - } - // The filter first: after a pair has been given up on, this compare is - // all hook D costs. - let slot = by_name_attempt(obj, key)?; - if !crate::object::object_is_shaped(obj) || crate::object::dictionary::is_dictionary(obj) { - by_name_give_up(slot); - return None; - } - let len = (*key).byte_len as usize; - if len > (*key).capacity as usize { - by_name_give_up(slot); - return None; - } - let bytes = std::slice::from_raw_parts(crate::string::string_data(key), len); - let keys = crate::object::object_keys(obj); - if !keys.is_null() - && crate::object::keys_find_slot_by_bytes(keys.arr(), keys.count(), bytes).is_some() - { - // An OWN key the data probe refused (an accessor, a hole): no entry - // can ever describe it, and without this the linear key scan above - // would run on every read of the pair. - by_name_give_up(slot); - return None; - } - inherited_read_cache_prime(obj, key) -} - -/// Direct-mapped filter of (receiver ShapeId, key address) pairs seen by hook -/// D. Each word is a FINGERPRINT (upper 30 bits) plus a sighting count (low 2 -/// bits); nothing here is ever dereferenced or compared as a pointer, so it is -/// not a GC root, and a collector move, a recycled address or a collision can -/// only cost one extra, or one skipped, prime. -const SEEN_SIZE: usize = 256; -const SEEN_COUNT_MASK: u32 = 3; -/// Count at which hook D stops trying for a pair. -const SEEN_GIVEN_UP: u32 = 3; - -crate::perry_thread_local! { - static BY_NAME_SEEN: std::cell::UnsafeCell<[u32; SEEN_SIZE]> = - const { std::cell::UnsafeCell::new([0; SEEN_SIZE]) }; -} - -/// Should hook D try to prime this (receiver shape, key) pair now? `Some` -/// (the filter slot, for [`by_name_give_up`]) on the 2nd and 3rd sightings. -/// -/// * First sighting: `None`. The prime walks the chain AND runs the generic -/// getter to confirm, so it costs more than the read it replaces and pays -/// off only for a pair read again. A key minted fresh for every read — -/// `o["k" + i]`, or a runtime path that builds a transient key string per -/// call — never repeats its address, so it never pays. -/// * Second and third sightings: try. A successful prime is served by the -/// table from then on and never comes back here; a refusal the table -/// records is answered `Declined` there. Two tries, because a walk that -/// first MARKS a prototype abandons and records nothing, by design. -/// * After that, or once the pair is known unprimeable: `None`, for the price -/// of this one compare. Without it, a pair that repeats but can never prime -/// (an own key the data probe refuses) paid the own-key scan on every read: -/// +11.5% on cron/next_dates before this cap existed. -/// -/// # Safety -/// `obj` is a proved ordinary object. -#[inline] -unsafe fn by_name_attempt( - obj: *const ObjectHeader, - key: *const crate::StringHeader, -) -> Option { - let shape = shapes::object_shape_stamp(obj) as u64; - let h = ((key as usize as u64) >> 4 ^ shape << 20).wrapping_mul(0x9E37_79B9_7F4A_7C15); - let index = (h >> 56) as usize & (SEEN_SIZE - 1); - let id = ((h >> 24) as u32 & !SEEN_COUNT_MASK) | (SEEN_COUNT_MASK + 1); - BY_NAME_SEEN.with(|cell| { - let seen = &mut *cell.get(); - let word = seen[index]; - if word & !SEEN_COUNT_MASK != id { - seen[index] = id | 1; - return None; - } - let count = word & SEEN_COUNT_MASK; - if count >= SEEN_GIVEN_UP { - return None; - } - seen[index] = id | (count + 1); - Some(index) - }) -} - -/// Stop trying for the pair whose filter slot [`by_name_attempt`] returned. -#[inline] -fn by_name_give_up(index: usize) { - BY_NAME_SEEN.with(|cell| unsafe { - let seen = &mut *cell.get(); - seen[index] |= SEEN_GIVEN_UP; - }); -} - -/// The walk itself. Every `None` here is a refusal; `note` is what makes the -/// refusal recordable. -/// -/// # Safety -/// As [`inherited_read_cache_prime`], whose guards this runs under. -/// -/// `write`: `Some(value)` walks for a `[[Set]]` of `value` instead of a read. -/// An accessor holder is recorded either way; a write then runs the setter -/// and answers `Some(undefined)` ("handled"), and a data holder is recorded -/// but answers `None` (the generic `[[Set]]` creates the own property). -unsafe fn inherited_read_cache_walk( - obj: *const ObjectHeader, - key: *const crate::StringHeader, - write: Option, - note: &mut DeclineNote, -) -> Option { - let key_addr = key as usize; - if !crate::value::addr_class::is_plausible_heap_addr(key_addr) - || !address_is_prime_stable(key_addr) - { - return None; - } - if (*key).byte_len > (*key).capacity || (*key).byte_len >= 1 << 28 { - return None; - } - let key_bytes = - std::slice::from_raw_parts(crate::string::string_data(key), (*key).byte_len as usize); - // Mirror `native_get::try_data_get_bytes`'s refusals exactly: private - // members and non-UTF-8 keys, whose descriptor summaries use a different - // hash. `constructor` is synthesized per receiver, so no key list alone - // can answer it: it is walked, but only ever into a PENDING entry the - // generic getter must confirm (`must_confirm` below). - if key_bytes.first() == Some(&b'#') || std::str::from_utf8(key_bytes).is_err() { - return None; - } - let accessor_bit = 1u64 << (super::key_bytes_hash(key_bytes.as_ptr(), key_bytes.len()) & 63); - - // The caller is `get_field_ic_miss_impl`'s fall-through, which is reached - // by NON-OBJECT receivers too (`miss_reason` only distinguishes them when - // IC diagnostics are on). Prove the kind before dereferencing anything: - // an Array's word at +0 is its `length` and at +4 its `capacity`, so - // reading them as `class_id` / ShapeId is not a type error the compiler - // can see. - let obj_addr = obj as usize; - if !crate::value::addr_class::is_plausible_heap_addr(obj_addr) - || crate::arena::classify_heap_generation(obj_addr) == crate::arena::HeapGeneration::Unknown - { - return None; - } - let recv_header = match crate::value::addr_class::try_read_gc_header_known_plausible(obj_addr) { - Some(header) => header, - None => return None, - }; - if recv_header.obj_type != crate::gc::GC_TYPE_OBJECT - || recv_header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 - || recv_header._reserved & crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO != 0 - { - return None; - } - match shapes::object_shape_descriptor(obj) { - Some(shape) if shape.object_kind.is_ordinary_layout() => {} - _ => return None, - } - let recv_class_id = (*obj).class_id; - let recv_shape = shapes::object_shape_stamp(obj); - if recv_shape == 0 { - return None; - } - let recv_meta = (*obj).meta; - if !receiver_address_facts_ok(recv_meta) { - return None; - } - if key_bytes == b"toJSON" && crate::perf_hooks::is_perf_entry_object(obj) { - return None; - } - let recv_proto_bits = if recv_meta.is_null() { - 0 - } else { - (*recv_meta).prototype - }; - // From here on the pair (receiver identity, key) is known, so a refusal - // can be recorded against it. Everything refused ABOVE this line is - // refused on grounds that are not a function of that pair. - note.armed = true; - note.key_ptr = key_addr; - note.recv_class_id = recv_class_id; - note.recv_shape = recv_shape; - note.recv_proto_bits = recv_proto_bits; - // An accessor or a customized descriptor for this key ON THE RECEIVER - // means the generic path owns this read. - if !recv_meta.is_null() - && ((*recv_meta).accessor_key_bits & accessor_bit != 0 - || (*recv_meta).attr_key_bits & accessor_bit != 0) - { - return None; - } - - let mut hops = [0usize; MAX_HOPS]; - let mut hop_count = 0usize; - let mut current = obj; - let mut current_class_id = recv_class_id; - let mut current_meta = recv_meta; - // Set once the walk has taken the default `%Object.prototype%` link (and - // from the start for `constructor`). What it then claims is a claim about - // the generic getter, which consults more than key lists — the ordinary- - // object fallback, a synthesized `constructor` — so every entry written - // under it is PENDING until that getter confirms it. - let mut must_confirm = key_bytes == b"constructor"; - // `%Object.prototype%` for this realm, resolved once per walk (0 when the - // realm has none yet, in which case the default link is never taken). - let object_prototype = if write.is_none() { - crate::array::object_prototype_addr_if_resolved() - } else { - 0 - }; - - loop { - // Resolve the next prototype the way `try_data_get_bytes` does. - let next: *const ObjectHeader = if !current_meta.is_null() && (*current_meta).prototype != 0 - { - let prototype = JSValue::from_bits((*current_meta).prototype); - if !prototype.is_pointer() { - return None; - } - prototype.as_pointer() - } else { - let synthetic = current_class_id >= 0x8000_0000 - && current_class_id - < super::NEXT_SYNTHETIC_CLASS_ID.load(std::sync::atomic::Ordering::Relaxed); - if synthetic { - if !super::class_decl_prototype_object(current_class_id).is_null() { - // Declared prototype metadata has its own precedence. - return None; - } - super::class_prototype_object(current_class_id) - } else if current_class_id == 0 || hop_count != 0 { - // No class of its own (id 0), or a hop that is not a class - // instance: the only link left is the ordinary default. - if !takes_default_link(current, current_class_id, object_prototype) { - return None; - } - must_confirm = true; - object_prototype as *const ObjectHeader - } else { - // Charter step 3: a declared-class instance with no recorded - // `[[Prototype]]` inherits from its class's declared prototype - // (the class id IS the link). Only from the receiver itself: - // a declared prototype carries its class's id too, and its own - // parent link is its recorded prototype. Not materialized here - // (that allocates and could move the receiver) — the generic - // path builds it, and the next read primes. Any other - // non-synthetic id is a default builtin prototype, which may - // still need lazy construction and can be replaced through - // `globalThis`. - let decl = super::class_decl_prototype_object(current_class_id); - if !decl.is_null() && decl as usize != obj_addr { - decl as *const ObjectHeader - } else if takes_default_link(current, current_class_id, object_prototype) { - // An object literal: its id is an anonymous shape's, which - // has no declared prototype. See `takes_default_link`. - must_confirm = true; - object_prototype as *const ObjectHeader - } else { - note.armed = false; - return None; - } - } - }; - if next.is_null() || next == current || next == obj { - return None; - } - if hop_count == MAX_HOPS { - return None; - } - let next_addr = next as usize; - if !crate::value::addr_class::is_plausible_heap_addr(next_addr) - || !address_is_prime_stable(next_addr) - { - return None; - } - let header = match crate::value::addr_class::try_read_gc_header_known_plausible(next_addr) { - Some(header) => header, - None => return None, - }; - if header.obj_type != crate::gc::GC_TYPE_OBJECT - || header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 - || header._reserved & crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO != 0 - { - return None; - } - let shape = match shapes::object_shape_descriptor(next) { - Some(shape) => shape, - None => return None, - }; - if !shape.object_kind.is_ordinary_layout() { - return None; - } - if shapes::object_shape_stamp(next) == 0 { - return None; - } - let meta = (*next).meta; - if !receiver_address_facts_ok(meta) { - return None; - } - // The hop must ALREADY be marked as somebody's prototype. An entry - // through an unmarked prototype is one that a key added to that - // prototype would not invalidate, so it must not be created. - // - // The `[[Prototype]]` install funnel marks, which covers most routes; - // this is the safety net for the ones it does not, and it is what - // makes the coverage obligation self-healing rather than a list to - // keep complete. Mark the hop and ABANDON the walk: marking allocates - // a meta record, which can move `obj`, `next` and every address in - // `hops`, so not one of them may be touched afterwards. The next read - // of this pair finds the hop marked and primes normally. - // - // The refusal is deliberately NOT remembered (`note.armed = false`): - // marking bumps no validity, so a negative entry recorded here would - // decline the pair for the life of the process — the same trap the - // value-dependent refusals avoid. - if meta.is_null() || (*meta).flags & crate::object::OBJECT_META_FLAG_IS_PROTOTYPE == 0 { - note.armed = false; - crate::object::proto_validity::mark_object_as_prototype(next_addr); - return None; - } - if key_bytes == b"toJSON" && crate::perf_hooks::is_perf_entry_object(next) { - return None; - } - // A clear Bloom bit PROVES no accessor and no customized descriptor - // for this key on this hop; a collision declines conservatively. - // Charter step 3: an ordinary hop's attributes live with its keys, so - // the key's own entry (read below) decides; its meta Bloom bits are - // not consulted. - - hops[hop_count] = next_addr; - hop_count += 1; - note.holder = next_addr; - note.hops = hops; - note.hop_count = hop_count as u8; - - // #10868 step 2.5 stage 1: a dictionary-mode hop keeps its own keys - // in its `ObjectMeta`, so reading them off its shape would walk PAST - // an own property and cache a farther-up value. - if crate::object::dictionary::is_dictionary(next) { - return None; - } - let keys = shape.keys as usize as *const crate::array::ArrayHeader; - if !keys.is_null() { - if let Some(slot) = - super::keys_find_slot_by_bytes_resolved(keys, shape.logical_key_count, key_bytes) - { - // Spilled fields are not reachable by the one-load hit path. - if slot >= shape.live_inline_slot_count { - return None; - } - let field = (next as *const u8) - .add(std::mem::size_of::() + slot as usize * 8) - as *const u64; - let bits = *field; - let entry_byte = super::key_attrs::keys_entry(keys, slot); - if entry_byte & super::key_attrs::ENTRY_ACCESSOR != 0 { - // Only a CLASS accessor (a compiled entry in its pair) is - // served from here. A builtin prototype's accessor (e.g. - // `Map.prototype.size`) is a native closure whose receiver - // handling — a subclass instance resolved to its backing — - // lives on the generic path, so it keeps that path. - match super::accessor_pair::pair_of_value(bits) { - Some(acc) if acc.raw_get != 0 || acc.raw_set != 0 => {} - _ => return None, - } - // An accessor entry runs its getter on the prime itself, - // so it cannot wait for the generic getter's confirmation - // the default link requires. Keep such a read generic. - if must_confirm { - return None; - } - let entry = Entry { - key_ptr: key_addr, - validity: crate::object::proto_validity::proto_validity(), - recv_proto_bits, - recv_class_id, - recv_shape, - holder: next_addr, - hops, - hop_count: hop_count as u8, - slot, - accessor: true, - }; - let index = entry_index(recv_class_id, recv_shape, key_addr); - INHERITED_READ_CACHE.with(|cell| { - (*cell.get())[index] = entry; - }); - if stats_enabled() { - PRIMES.fetch_add(1, Ordering::Relaxed); - } - // The prime answers the access too: run the accessor now. - if let Some(value) = write { - return accessor_set(obj, bits, value).then(JSValue::undefined); - } - return match accessor_hit(obj, bits) { - Lookup::Hit(value) => Some(value), - _ => None, - }; - } - if bits == crate::value::TAG_HOLE { - note.value_dependent = true; - return None; - } - let value = JSValue::from_bits(bits); - if value.is_undefined() || value.is_null() { - // `try_data_get_bytes` treats these as a miss on an - // inherited read; do not record a claim it would refuse. - // A later store can make this slot a real value with no - // shape transition, so this refusal is not remembered. - note.value_dependent = true; - return None; - } - let entry = Entry { - key_ptr: key_addr, - validity: crate::object::proto_validity::proto_validity(), - recv_proto_bits, - recv_class_id, - recv_shape, - holder: next_addr, - hops, - hop_count: hop_count as u8, - slot, - accessor: false, - }; - if must_confirm { - write_pending(entry, note); - return None; - } - let index = entry_index(recv_class_id, recv_shape, key_addr); - INHERITED_READ_CACHE.with(|cell| { - (*cell.get())[index] = entry; - }); - if stats_enabled() { - PRIMES.fetch_add(1, Ordering::Relaxed); - } - if write.is_some() { - return None; - } - return Some(value); - } - } - - // The key is on no hop up to and including `%Object.prototype%`, - // whose `[[Prototype]]` is null: the chain has ended and the read - // answers `undefined`. Recorded as a PENDING absent entry for the - // generic getter to confirm (see `confirm_pending`). Read-only: a - // write never resolves `object_prototype`, so it never gets here. - if object_prototype != 0 && next_addr == object_prototype { - if !meta.is_null() && (*meta).prototype != 0 { - return None; - } - let entry = Entry { - key_ptr: key_addr, - validity: crate::object::proto_validity::proto_validity(), - recv_proto_bits, - recv_class_id, - recv_shape, - holder: next_addr, - hops, - hop_count: hop_count as u8, - slot: ABSENT_SLOT, - accessor: false, - }; - write_pending(entry, note); - return None; - } - - current = next; - current_class_id = (*next).class_id; - current_meta = meta; - } -} - -/// May the walk take the default `%Object.prototype%` link from `obj`, a proved -/// ordinary `GC_TYPE_OBJECT` with no recorded `[[Prototype]]`? -/// -/// An object with no class of its own — an object literal, or a plain object -/// used as a prototype — inherits from `%Object.prototype%`, which is where -/// `ordinary_object_prototype_property_value` sends the generic getter. Before -/// this link existed, every read on such a receiver that was not an own -/// property declined, unrecorded, and re-ran the whole generic walk on every -/// execution. -/// -/// "No class of its own" is: class id 0 or an anonymous literal shape's id -/// that no module also uses for a declared class, not born with a null -/// prototype, and no prototype object registered for that id in either class -/// registry — the two tables -/// `prototype_chain::class_link_prototype` consults, so a receiver the generic -/// path would route elsewhere is refused here. A declared class's instance or -/// prototype object carries its class's id and is never taken. Read-only -/// walks only: a write walk passes `object_prototype == 0`. -/// -/// # Safety -/// `obj` is a live object whose `GcHeader` precedes it. -unsafe fn takes_default_link( - obj: *const ObjectHeader, - class_id: u32, - object_prototype: usize, -) -> bool { - if object_prototype == 0 || obj as usize == object_prototype { - return false; - } - if class_id != 0 && !is_anon_shape_class_id_memo(class_id) { - return false; - } - match crate::value::addr_class::try_read_gc_header_known_plausible(obj as usize) { - Some(header) if header._reserved & crate::gc::OBJ_FLAG_NULL_PROTO == 0 => {} - _ => return false, - } - // Class ids are handed out per module, so an anonymous shape's id can be - // numerically equal to another module's DECLARED class - // (`declared_class_outranks_anon_shape`). A registered class name is the - // declared class's positive evidence; such an id never takes the default - // link, whatever its prototype objects look like right now. - class_id == 0 - || (super::class_name_for_id(class_id).is_none() - && super::class_decl_prototype_object(class_id).is_null() - && super::class_prototype_object(class_id).is_null()) -} - -/// Class ids already proved NOT to be an anonymous literal shape's, direct -/// mapped. A class id's anon-ness is fixed for the life of the process (an id -/// is registered as an anonymous shape's when that shape is created, before -/// any object carries it, and never unregistered), so a remembered "no" stays -/// true. Only the "no" is remembered: it is the answer every read on a -/// declared-class instance with no materialized prototype object asks for -/// again, and `is_anon_shape_class_id` answers it through a lock and a SipHash -/// probe once its fast mirror has overflowed. Class ids are plain integers, -/// not heap references, so this table is not a GC root. -const NOT_ANON_MEMO_SIZE: usize = 64; - -crate::perry_thread_local! { - static NOT_ANON_MEMO: std::cell::UnsafeCell<[u32; NOT_ANON_MEMO_SIZE]> = - const { std::cell::UnsafeCell::new([0; NOT_ANON_MEMO_SIZE]) }; -} - -/// [`super::is_anon_shape_class_id`] with the "no" answers memoized. -/// `class_id` is non-zero (0 is never anon and never stored, so an empty slot -/// cannot match). -#[inline] -fn is_anon_shape_class_id_memo(class_id: u32) -> bool { - let index = (class_id as usize) & (NOT_ANON_MEMO_SIZE - 1); - let known_not_anon = NOT_ANON_MEMO.with(|cell| unsafe { (*cell.get())[index] == class_id }); - if known_not_anon { - return false; - } - let anon = super::is_anon_shape_class_id(class_id); - if !anon { - NOT_ANON_MEMO.with(|cell| unsafe { (*cell.get())[index] = class_id }); - } - anon -} - -/// Write `entry` PENDING and tell the prime where it is (see -/// [`confirm_pending`]). The walk then returns `None`; the prime, not the walk, -/// produces the read's value. -unsafe fn write_pending(mut entry: Entry, note: &mut DeclineNote) { - let index = entry_index(entry.recv_class_id, entry.recv_shape, entry.key_ptr); - let validity = entry.validity; - entry.validity = PENDING_VALIDITY; - INHERITED_READ_CACHE.with(|cell| { - (*cell.get())[index] = entry; - }); - note.pending = Some(Pending { - index, - recv_class_id: entry.recv_class_id, - recv_shape: entry.recv_shape, - slot: entry.slot, - validity, - }); -} - -// --- GC --------------------------------------------------------------------- - -/// Root scan. Both the key and every hop are MARKED, not merely rewritten. -/// -/// #6759 phase 3 made the transition cache's equivalent slots weak, and was -/// right to: that cache holds 16384 keys arrays and never dereferences one, so -/// rewrite-only plus a death prune loses nothing. This cache is the other -/// case. A hit LOADS `holder + slot`, so a weak slot that the prune has not -/// yet reached — or that a deadness predicate answers conservatively — is a -/// read of recycled memory, and the value it returns is wrong rather than -/// merely stale. Marking makes that unrepresentable. -/// -/// The retention it buys is bounded by the table: at most 512 keys and -/// 512 x MAX_HOPS prototype objects, and a prototype that a program still reads -/// through is reachable from its constructor anyway. The prune below still -/// runs, so an entry whose owner the collector calls dead is dropped rather -/// than kept alive indefinitely by eviction pressure alone. -pub(crate) fn scan_inherited_read_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - INHERITED_READ_CACHE.with(|cell| unsafe { - for entry in (*cell.get()).iter_mut() { - if entry.key_ptr == 0 { - continue; - } - visitor.visit_tagged_usize_slot(&mut entry.key_ptr, crate::value::STRING_TAG); - for i in 0..entry.hop_count as usize { - visitor.visit_usize_slot(&mut entry.hops[i]); - } - // The same object as the last hop, in its own slot so the hit - // loads it at a fixed offset. Visiting one object through two - // slots is what every other multi-slot root does; the second visit - // finds the forwarding record the first one installed. - visitor.visit_usize_slot(&mut entry.holder); - } - }); -} - -/// Drop entries naming an owner that did not survive. -/// -/// Rewrite-only re-keying has no death story on its own: the address of a -/// collected holder stays in the table and a later allocation at that address -/// turns the entry into a false hit that reads a live object's slot for the -/// wrong key. This is the obligation `gc::dead_owner`'s registry exists to -/// make un-forgettable. -pub(crate) fn prune_dead_inherited_cache_entries(is_dead_owner: &dyn Fn(usize) -> bool) { - INHERITED_READ_CACHE.with(|cell| unsafe { - for entry in (*cell.get()).iter_mut() { - if entry.key_ptr == 0 { - continue; - } - let mut dead = is_dead_owner(entry.key_ptr); - for i in 0..entry.hop_count as usize { - dead |= is_dead_owner(entry.hops[i]); - } - dead |= entry.holder != 0 && is_dead_owner(entry.holder); - if dead { - *entry = EMPTY_ENTRY; - } - } - }); -} - -// --- emitted-code entry (for lane 4) ---------------------------------------- - -/// Hit / prime / decline counts, for a program or a harness that wants to -/// assert the cache is actually being HIT. A cache that primes and then -/// declines every lookup returns the same values the chain walk would and is -/// invisible in a program's output; this is the only thing that tells them -/// apart from outside the runtime. `which`: 0 hits, 1 primes, 2 declines, -/// 3 declines served from a negative entry (i.e. without a chain walk). -/// Arm the counting with `PERRY_INHERITED_IC_STATS`. -#[no_mangle] -pub extern "C" fn js_inherited_read_cache_stats(which: i32) -> f64 { - match which { - 0 => inherited_read_cache_hits() as f64, - 1 => inherited_read_cache_primes() as f64, - 2 => inherited_read_cache_declines() as f64, - 3 => inherited_read_cache_neg_served() as f64, - _ => -1.0, - } -} - -/// The hit, as the emitted read sequence would call it: masked receiver -/// pointer plus interned key, NaN-boxed value back, `TAG_HOLE` for a decline -/// (which no ordinary value can be, so the caller branches on one compare). -/// -/// DATA entries only. Codegen lists this call as a GC leaf -/// (`gc_call_effects.rs`, `root_reload.rs`): nothing is spilled or reloaded -/// around it, so it must never run user code or collect. An ACCESSOR entry -/// runs a getter, so here it declines, and the miss handler — a collection -/// point — serves it through [`inherited_read_cache_lookup`]. -/// -/// # Safety -/// `obj` is a masked heap pointer whose pointer tag the caller established. -#[no_mangle] -pub unsafe extern "C" fn js_inherited_read_cache_hit_f64( - obj: *const ObjectHeader, - key: *const crate::StringHeader, -) -> f64 { - match inherited_read_cache_lookup_data(obj, key) { - Lookup::Hit(value) => f64::from_bits(value.bits()), - Lookup::Declined | Lookup::Unknown => f64::from_bits(crate::value::TAG_HOLE), - } -} - -#[cfg(test)] -#[path = "inherited_read_cache_tests.rs"] -mod tests; diff --git a/crates/perry-runtime/src/object/inherited_read_cache_tests.rs b/crates/perry-runtime/src/object/inherited_read_cache_tests.rs deleted file mode 100644 index ee9c9ac5de..0000000000 --- a/crates/perry-runtime/src/object/inherited_read_cache_tests.rs +++ /dev/null @@ -1,1384 +0,0 @@ -//! Invalidation suite for the inherited-read cache. -//! -//! Every test here asserts the COUNTERS as well as the value. A cache that -//! quietly declines and lets the chain walk answer returns the right value and -//! is invisible in a program's output; the only way to tell a working hit from -//! a broken one is to count it. Conversely, every invalidation test asserts -//! that the second read is NOT a hit — an entry that keeps matching after a -//! mutation returns a stale value, which is a wrong answer, not a slow one. - -use super::*; - -fn key(name: &str) -> *const crate::StringHeader { - crate::string::js_string_from_bytes(name.as_ptr(), name.len() as u32) -} - -fn set(obj: *mut ObjectHeader, name: &str, value: f64) { - crate::object::js_object_set_field_by_name(obj, key(name), value); -} - -/// A getter whose closure bits are a placeholder: the cache must refuse the -/// key on the STRENGTH OF THE DESCRIPTOR, never on whether the getter is -/// callable, so a test that installed a real closure would pass with a cache -/// that looked at the wrong thing. -fn install_getter(obj: *mut ObjectHeader, name: &str) { - crate::object::descriptor_state::set_accessor_descriptor( - obj as usize, - name.to_string(), - crate::object::descriptor_state::AccessorDescriptor { get: 0, set: 0 }, - ); -} - -fn boxed(obj: *mut ObjectHeader) -> f64 { - f64::from_bits(crate::value::js_nanbox_pointer(obj as i64).to_bits()) -} - -/// These tests assert entry IDENTITY, so a collection moving an object -/// mid-test would make an invalidation assertion pass for the wrong reason. -/// Suppress it and start from an empty table. -struct PrimeScope { - _suppress: crate::gc::GcSuppressScope, - _lock: std::sync::MutexGuard<'static, ()>, -} - -impl PrimeScope { - fn new() -> Self { - let lock = crate::gc::global_side_table_test_lock(); - let scope = Self { - _suppress: crate::gc::GcSuppressScope::new(), - _lock: lock, - }; - // Materialize the realm FIRST. Its bootstrap reads builtins by name - // through `js_object_get_field_by_name`, whose hook D primes this - // cache; left lazy, that happens inside whichever test first touches - // the realm and lands in its counters. It also makes - // `%Object.prototype%` resolvable, which the default-link tests need. - crate::object::js_get_global_this(); - test_clear_cache(); - test_reset_counters(); - scope - } -} - -/// `O -> P`, `a` on `P` only. -unsafe fn one_level() -> (*mut ObjectHeader, *mut ObjectHeader) { - let proto = crate::object::js_object_alloc(0, 4); - set(proto, "irc_a", 7.0); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(proto)); - (obj, proto) -} - -#[test] -fn a_repeated_inherited_read_is_served_by_the_cache() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, _proto) = one_level(); - let k = key("irc_a"); - // Setup reads (a `constructor` read, a builtin lookup) prime entries of - // their own through hook D; count only the operation under test. - test_reset_counters(); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "nothing is primed yet" - ); - let primed = inherited_read_cache_prime(obj, k).expect("the walk must resolve irc_a"); - assert_eq!(f64::from_bits(primed.bits()), 7.0); - assert_eq!(inherited_read_cache_primes(), 1); - - for _ in 0..5 { - let value = inherited_read_cache_hit(obj, k).expect("the entry must serve the read"); - assert_eq!(f64::from_bits(value.bits()), 7.0); - } - assert_eq!( - inherited_read_cache_hits(), - 5, - "the cache returned the right value but never actually hit — a \ - fallback to the chain walk is correct and invisible" - ); - } -} - -/// An entry is keyed on a SHAPE, not on a receiver, so two receivers that -/// genuinely have one shape must be served by one entry. -/// -/// The construction order below is the test, not incidental setup. Two -/// receivers built "the same way" do NOT automatically have one ShapeId, and -/// the two things that decide it are both ordering-sensitive: -/// -/// 1. **The key-add must reuse the first receiver's edge.** A ShapeId's -/// identity includes the keys ARRAY ADDRESS, and two objects share one only -/// when the second's key-add hits `object::transition_cache_lookup` — a -/// 16384-entry DIRECT-MAPPED table hashed on `(predecessor ShapeId, the -/// interned key's address)`. A collision from an unrelated entry evicts the -/// edge, the second receiver mints its own keys array and its own ShapeId, -/// and nothing is wrong: a transition-cache miss costs a duplicate shape, -/// never a wrong answer. But it is address-keyed, so whether it collides -/// varies with heap placement RUN TO RUN. With an unrelated call between -/// the two `set`s this test failed 2 runs in 6 of the same binary. Both -/// key-adds therefore happen back to back, with only an allocation between -/// them, so the edge the first one inserts is certain to still be there. -/// 2. **Both prototype links must precede the prime.** `Object.setPrototypeOf` -/// is a semantic property event: it bumps `prop_plan_epoch`, which bumps the -/// one validity word every entry is re-proved against -/// (`object::proto_validity`, check 1). Linking `second` AFTER priming -/// `first` retires the entry that was just made. Measured, every other field -/// of the entry matched the second receiver exactly — same class id `0x0`, -/// same ShapeId, same recorded prototype bits, same slot index — and only -/// `validity` differed, by one. The miss that followed said nothing -/// whatever about entry sharing. -/// -/// The ShapeId merge is then asserted rather than assumed. #10931 mints a -/// prototype divergence's generation as a pure function of `(predecessor -/// ShapeId, the prototype's stable serial, link kind)`, so with (1) holding, -/// these two land on ONE ShapeId. Before it, each drew a fresh value from the -/// monotonic counter and they never could, which is why this test was written -/// with an `if (*first).parent_class_id == (*second).parent_class_id` guard -/// around its assertion. That word IS the ShapeId -/// ([`shapes::object_shape_stamp`] reads it), the two were never equal, and the -/// body never ran: the test was dormant from the day it was written. It is an -/// assertion now, so it can never go quiet again. -#[test] -fn a_second_receiver_of_the_same_shape_shares_the_entry() { - let _scope = PrimeScope::new(); - unsafe { - let proto = crate::object::js_object_alloc(0, 4); - set(proto, "irc_a", 7.0); - - // A second receiver reaching the SAME prototype through the same - // operation: same class id, same recorded prototype bits, same shape. - let first = crate::object::js_object_alloc(0, 4); - let second = crate::object::js_object_alloc(0, 4); - set(first, "irc_own", 1.0); - set(second, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(first), boxed(proto)); - crate::object::js_object_set_prototype_of(boxed(second), boxed(proto)); - - assert_eq!( - (*first).class_id, - (*second).class_id, - "the two receivers must share a class id, or the entry index \ - separates them for a reason that has nothing to do with shape" - ); - assert_eq!( - shapes::object_shape_stamp(first), - shapes::object_shape_stamp(second), - "#10931: the same divergence from the same predecessor to the same \ - prototype must mint ONE ShapeId. Two here and this test has no \ - subject — the assertions below would be asking whether two \ - DIFFERENT shapes share an entry, which they must not" - ); - - let k = key("irc_a"); - inherited_read_cache_prime(first, k).expect("prime"); - let hits_before = inherited_read_cache_hits(); - let value = inherited_read_cache_hit(second, k) - .expect("two receivers with one shape must share one entry"); - assert_eq!(f64::from_bits(value.bits()), 7.0); - assert_eq!( - inherited_read_cache_hits(), - hits_before + 1, - "the second receiver was answered without the cache hitting — a \ - fall-through to the chain walk returns the same 7.0 and is \ - invisible in a program's output" - ); - } -} - -#[test] -fn a_shadowing_own_key_on_the_receiver_stops_the_entry_matching() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, _proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - set(obj, "irc_a", 99.0); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "an own key shadowing the cached inherited one is a key-ADD \ - transition; the receiver's ShapeId must no longer match" - ); - } -} - -#[test] -fn deleting_the_shadowing_own_key_exposes_the_inherited_value_again() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, _proto) = one_level(); - let k = key("irc_a"); - set(obj, "irc_a", 99.0); - // With the own key present the walk must not prime at all: the key is - // an own property, so an inherited entry would be a lie. - assert!( - inherited_read_cache_prime(obj, k).is_none() || { - // Priming is only reached after the caller's own-key search fails, - // so a prime here would be a caller contract violation, not a - // cache bug. Assert the value is at least the own one. - true - } - ); - crate::object::js_object_delete_field(obj, k); - let value = inherited_read_cache_prime(obj, k).expect("the inherited value is visible now"); - assert_eq!(f64::from_bits(value.bits()), 7.0); - } -} - -#[test] -fn adding_a_key_to_the_prototype_invalidates_through_proto_validity() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - // A plain store is not a descriptor install, so it does NOT bump the - // semantic epoch. It is a key-add transition on an object the prime - // MARKED, so the shape-stamp funnel bumps the validity word — the - // whole reason that hook exists. - assert!( - crate::object::proto_validity::object_is_marked_prototype(proto as usize), - "priming must mark the hop, or nothing will ever see a mutation of it" - ); - set(proto, "irc_b", 3.0); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "a key added to the prototype changed its ShapeId and the entry \ - still matched: the prototype-validity bump is not load-bearing" - ); - } -} - -#[test] -fn deleting_the_key_from_the_prototype_invalidates() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - crate::object::js_object_delete_field(proto, k); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "a stable-tombstone delete can keep the holder's ShapeId, so this \ - invalidation rests on the semantic epoch" - ); - } -} - -#[test] -fn redefining_the_prototype_key_as_an_accessor_invalidates() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - install_getter(proto, "irc_a"); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "an accessor installed over the cached data slot must retire the \ - entry: serving the slot would skip the getter" - ); - } -} - -#[test] -fn set_prototype_of_on_the_receiver_invalidates() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, _proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - let other = crate::object::js_object_alloc(0, 4); - set(other, "irc_a", 42.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(other)); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "the receiver now inherits from a different object" - ); - } -} - -#[test] -fn set_prototype_of_on_an_interior_prototype_invalidates() { - let _scope = PrimeScope::new(); - unsafe { - // O -> P1 -> P2, `a` on P2. The holder is P2 and the receiver is O; - // re-pointing P1 is visible to NEITHER of their guards. - let p2 = crate::object::js_object_alloc(0, 4); - set(p2, "irc_a", 7.0); - let p1 = crate::object::js_object_alloc(0, 4); - set(p1, "irc_mid", 1.0); - crate::object::js_object_set_prototype_of(boxed(p1), boxed(p2)); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(p1)); - - let k = key("irc_a"); - let primed = inherited_read_cache_prime(obj, k).expect("a two-hop chain must prime"); - assert_eq!(f64::from_bits(primed.bits()), 7.0); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - let replacement = crate::object::js_object_alloc(0, 4); - set(replacement, "irc_other", 5.0); - crate::object::js_object_set_prototype_of(boxed(p1), boxed(replacement)); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "an interior prototype swap left the entry matching — the chain \ - now ends somewhere else and the cached holder is unreachable" - ); - } -} - -/// Drive the read through the REAL inline-cache entry the compiled code calls, -/// not through `inherited_read_cache_prime` directly. -/// -/// That distinction is the whole point of the two tests below: both defects -/// they pin live in `get_field_ic_miss_impl`'s routing, so a test that calls -/// the cache's own functions cannot see either one. Measured against a build -/// without the fixes, these reads prime zero times (first test) or once per -/// read forever (second), and in both cases the cache is pure overhead — the -/// probe runs on every read, never serves, and the chain walk proceeds -/// unchanged. -unsafe fn read_through_the_inline_cache( - obj: *mut ObjectHeader, - k: *const crate::StringHeader, - slot: &mut crate::object::field_get_set::PicCacheSlot, - site: u64, -) -> f64 { - let bits = crate::value::js_nanbox_pointer(obj as i64).to_bits() as i64; - crate::object::field_get_set::js_object_get_field_ic(bits, k, site, slot) -} - -#[test] -fn a_receiver_with_no_own_keys_is_cached() { - let _scope = PrimeScope::new(); - unsafe { - let proto = crate::object::js_object_alloc(0, 4); - set(proto, "irc_nokeys", 11.0); - // `Object.create(p)` with nothing of its own: the receiver has no keys - // array at all, so the miss handler reports `ObjectNoKeys` rather than - // `NotOwn`. This is the single most common inherited-read shape there - // is, and the prime site was gated on `NotOwn` alone. - let created = crate::object::js_object_create(boxed(proto)); - let obj = crate::value::js_nanbox_get_pointer(created) as *mut ObjectHeader; - let k = key("irc_nokeys"); - let mut slot: crate::object::field_get_set::PicCacheSlot = std::ptr::null_mut(); - for _ in 0..4 { - let v = read_through_the_inline_cache(obj, k, &mut slot, 9001); - assert_eq!(v, 11.0, "the read must still answer correctly"); - } - assert!( - inherited_read_cache_primes() >= 1, - "a keyless receiver never reached the prime, so the cache can never \ - serve this shape and its probe is pure overhead on every read" - ); - assert!(inherited_read_cache_hits() >= 1, "primed but never served"); - } -} - -#[test] -fn several_object_create_receivers_do_not_evict_each_other() { - let _scope = PrimeScope::new(); - unsafe { - let proto = crate::object::js_object_alloc(0, 4); - set(proto, "irc_shared", 13.0); - // Fresh objects sharing a prototype also share a class id and shape. - // Their inherited lookup should reuse a single cache entry. - let mut objs = Vec::new(); - for i in 0..8 { - let created = crate::object::js_object_create(boxed(proto)); - let o = crate::value::js_nanbox_get_pointer(created) as *mut ObjectHeader; - set(o, "irc_own", i as f64); - objs.push(o); - } - let k = key("irc_shared"); - // Setup reads (a `constructor` read, a builtin lookup) prime entries of - // their own through hook D; count only the operation under test. - test_reset_counters(); - let mut slot: crate::object::field_get_set::PicCacheSlot = std::ptr::null_mut(); - let rounds = 8; - for _ in 0..rounds { - for o in &objs { - let v = read_through_the_inline_cache(*o, k, &mut slot, 9002); - assert_eq!(v, 13.0, "the read must still answer correctly"); - } - } - // Account for EVERY read rather than bounding the hits, because a - // loose lower bound on hits is what an off-by-one hides in. - let primes = inherited_read_cache_primes(); - let hits = inherited_read_cache_hits(); - let declines = inherited_read_cache_declines(); - let neg = inherited_read_cache_neg_served(); - let reads = (objs.len() * rounds) as u64; - - assert_eq!( - primes, 1, - "equivalent receivers should share one cache entry" - ); - - // At most ONE decline, and it is expected rather than tolerated. - // - // The inherited-read cache refuses to record a hop that the - // `[[Prototype]]` install funnel has not marked, and when its walk - // meets an unmarked hop it marks that hop and ABANDONS the walk - // without recording anything (`object::proto_validity`). Marking - // allocates a meta record, which can move the receiver, the hop and - // every address the walk is holding, so nothing it was holding may be - // touched afterwards — abandoning is not a shortcut, it is the only - // safe thing to do once the allocation has happened. - // - // These eight receivers share ONE prototype, so at most one read pays - // that: the first to reach an unmarked hop. Every later read finds it - // marked and primes normally. Without the marking stack in the tree - // this is 0; with it, 1. Both are correct, and the accounting below - // pins the difference to exactly that one read instead of loosening - // the hit count to absorb it. - assert!( - declines <= 1, - "{declines} declines: at most one mark-and-abandon is expected for \ - a single shared prototype" - ); - - assert_eq!( - hits + primes + declines + neg, - reads, - "every read must be exactly one of: served from an entry ({hits}), \ - the walk that recorded one ({primes}), a mark-and-abandon \ - ({declines}), or served from a negative entry ({neg}) — and they \ - sum to {}, not the {reads} reads performed", - hits + primes + declines + neg - ); - } -} - -#[test] -fn a_key_added_at_the_far_end_of_a_three_hop_chain_invalidates() { - let _scope = PrimeScope::new(); - unsafe { - // O -> P1 -> P2 -> P3, `a` on P3. Under the per-hop walk this cost - // three dependent loads on every hit; under the validity word it costs - // the same one compare a one-hop chain costs. - let p3 = crate::object::js_object_alloc(0, 4); - set(p3, "irc_a", 7.0); - let p2 = crate::object::js_object_alloc(0, 4); - set(p2, "irc_m2", 1.0); - crate::object::js_object_set_prototype_of(boxed(p2), boxed(p3)); - let p1 = crate::object::js_object_alloc(0, 4); - set(p1, "irc_m1", 1.0); - crate::object::js_object_set_prototype_of(boxed(p1), boxed(p2)); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(p1)); - - let k = key("irc_a"); - let primed = inherited_read_cache_prime(obj, k).expect("a three-hop chain must prime"); - assert_eq!(f64::from_bits(primed.bits()), 7.0); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - // Not the holder, not the receiver's prototype: the middle of the - // chain, whose mutation neither end's guard can see. - set(p2, "irc_new", 3.0); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "a key added to an INTERIOR prototype left the entry matching" - ); - } -} - -#[test] -fn an_attribute_change_on_the_prototype_invalidates() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - // Not a value change and not a key change: only the attributes move. - crate::object::descriptor_state::set_property_attrs( - proto as usize, - "irc_a".to_string(), - crate::object::descriptor_state::PropertyAttrs::new(false, false, false), - ); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "a non-enumerable/non-writable redefinition of the cached key must \ - retire the entry: the slot is no longer the whole answer" - ); - } -} - -#[test] -fn a_mutation_of_an_object_nobody_inherits_from_does_not_invalidate() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, _proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - // The common case of all mutation. A global validity counter that was - // not gated on the mark would invalidate here, and this cache would be - // a recompute in every program that builds objects in a loop. - for i in 0..8 { - let bystander = crate::object::js_object_alloc(0, 4); - set(bystander, "irc_bystander", i as f64); - set(bystander, "irc_bystander2", i as f64); - } - assert!( - inherited_read_cache_hit(obj, k).is_some(), - "building unrelated objects invalidated the chain: the validity \ - bump is not gated on OBJ_FLAG_IS_PROTOTYPE" - ); - } -} - -#[test] -fn replacing_a_registered_class_prototype_invalidates() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, _proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - // The chain of an `Object.create` / `new C()` receiver with no meta - // record is resolved out of `CLASS_PROTOTYPE_OBJECTS`. Re-registering - // mutates NOTHING this entry records: not the receiver, not the - // holder, not any shape. Only the surface generation moves. - let replacement = crate::object::js_object_alloc(0, 4); - set(replacement, "irc_a", 42.0); - crate::object::class_prototype_object_root_store(0x4242_0001, replacement); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "a re-registered class prototype left the entry matching: it would \ - now answer with a different object than the chain walk beside it" - ); - } -} - -/// Coverage for the `[[Prototype]]` INSTALL sites, as a TEST rather than a -/// counter nobody reads. -/// -/// The cache no longer marks its own hops: it refuses one that the install -/// funnel did not mark. That is the fail-safe polarity — a missed install site -/// costs a cache HIT, never a stale value — but "fail-safe" and "works" are -/// different claims, and only the hit counter can tell them apart, because a -/// cache that declines everything returns exactly the values the chain walk -/// would and is invisible in a program's output. -/// -/// So each construction style below builds a receiver the way real code does, -/// through the same runtime entry points the compiled code calls, and asserts -/// the READ WAS SERVED BY THE CACHE. A future change that adds a way to build -/// a prototype without marking it fails here instead of quietly costing every -/// inherited read through it. -fn assert_style_is_cached(style: &str, obj: *mut ObjectHeader, key_name: &str, want: f64) { - unsafe { - let k = key(key_name); - // One warm-up attempt is allowed: a route the install funnel does not - // cover marks its hop on the first walk and abandons it, so the SECOND - // read is the one that primes. What is not allowed is never priming, - // which is what a route with no marking at all would do. - if inherited_read_cache_prime(obj, key(key_name)).is_none() { - let _ = inherited_read_cache_prime(obj, key(key_name)); - } - test_clear_cache(); - test_reset_counters(); - let primed = inherited_read_cache_prime(obj, k) - .unwrap_or_else(|| panic!("{style}: the walk did not resolve {key_name}")); - assert_eq!(f64::from_bits(primed.bits()), want, "{style}: wrong value"); - assert_eq!( - inherited_read_cache_primes(), - 1, - "{style}: the walk resolved the key but REFUSED to record it — the \ - prototype it went through was never marked by an install site, so \ - every read of it re-walks the chain" - ); - let served = inherited_read_cache_hit(obj, k) - .unwrap_or_else(|| panic!("{style}: the entry did not serve the next read")); - assert_eq!( - f64::from_bits(served.bits()), - want, - "{style}: wrong value on hit" - ); - assert_eq!( - inherited_read_cache_hits(), - 1, - "{style}: not counted as a hit" - ); - } -} - -#[test] -fn every_common_way_of_building_a_receiver_is_cached() { - let _scope = PrimeScope::new(); - // EVERY style gets its OWN prototype object. Sharing one would let a - // style pass because an EARLIER style's install site marked it, which - // is the exact shape of a test that cannot fail for the reason it - // names. - let fresh_proto = |value: f64| { - let p = crate::object::js_object_alloc(0, 4); - set(p, "cov_a", value); - p - }; - - // 1. `Object.setPrototypeOf` on an object literal. - let p1 = fresh_proto(1.0); - let literal = crate::object::js_object_alloc(0, 4); - set(literal, "cov_own", 0.0); - crate::object::js_object_set_prototype_of(boxed(literal), boxed(p1)); - assert_style_is_cached("setPrototypeOf on a literal", literal, "cov_a", 1.0); - - // 2. `Object.create(p)` — a different install route than 1. - let p2 = fresh_proto(2.0); - let created_bits = crate::object::js_object_create(boxed(p2)); - let created = crate::value::js_nanbox_get_pointer(created_bits) as *mut ObjectHeader; - set(created, "cov_own", 0.0); - assert_style_is_cached("Object.create", created, "cov_a", 2.0); - - // 3. A class-DEFAULT prototype link, as `new C()` performs it. This - // link deliberately bumps no epoch and transitions no shape, so if - // it did not mark, nothing else would. - let p3 = fresh_proto(3.0); - let instance = crate::object::js_object_alloc(0, 4); - set(instance, "cov_own", 0.0); - crate::object::prototype_chain::object_link_class_default_prototype( - instance as usize, - crate::value::js_nanbox_pointer(p3 as i64).to_bits(), - ); - assert_style_is_cached("class-default link (new C())", instance, "cov_a", 3.0); - - // 4. An evaluated class prototype (#9502) — its own link kind. - let p4 = fresh_proto(4.0); - let evaluated = crate::object::js_object_alloc(0, 4); - set(evaluated, "cov_own", 0.0); - crate::object::prototype_chain::object_link_class_evaluation_prototype( - evaluated as usize, - crate::value::js_nanbox_pointer(p4 as i64).to_bits(), - ); - assert_style_is_cached("class-evaluation link", evaluated, "cov_a", 4.0); - - // 5. Two hops: a base class's prototype reached through a derived - // one. The MIDDLE object must be marked as well as the holder, or - // the walk refuses at hop 1 and never reaches the answer. - let base_proto = crate::object::js_object_alloc(0, 4); - set(base_proto, "cov_method", 9.0); - let derived_proto = crate::object::js_object_alloc(0, 4); - set(derived_proto, "cov_mid", 0.0); - crate::object::js_object_set_prototype_of(boxed(derived_proto), boxed(base_proto)); - let derived = crate::object::js_object_alloc(0, 4); - set(derived, "cov_own", 0.0); - crate::object::js_object_set_prototype_of(boxed(derived), boxed(derived_proto)); - assert_style_is_cached("two hops (extends)", derived, "cov_method", 9.0); - - // 6. A prototype whose key is ASSIGNED AFTER the receiver exists — - // `C.prototype.m = ...` after construction. The link is already - // marked; this checks the later key add does not disturb it. - let late_proto = crate::object::js_object_alloc(0, 4); - set(late_proto, "cov_placeholder", 0.0); - let late = crate::object::js_object_alloc(0, 4); - set(late, "cov_own", 0.0); - crate::object::js_object_set_prototype_of(boxed(late), boxed(late_proto)); - set(late_proto, "cov_late", 5.0); - assert_style_is_cached("key added to the prototype later", late, "cov_late", 5.0); -} - -#[test] -fn an_unmarked_prototype_is_refused_rather_than_cached_unsafely() { - let _scope = PrimeScope::new(); - unsafe { - // The fail-safe polarity, asserted directly. A chain reachable by a - // route that never marked its hop must DECLINE, not cache: an entry - // through an unmarked prototype is one that a key added to that - // prototype would not invalidate. - let proto = crate::object::js_object_alloc(0, 4); - set(proto, "cov_u", 3.0); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "cov_own", 0.0); - // Install the prototype WITHOUT the funnel, the way a future install - // site that forgot to mark would. - let meta = crate::object::object_meta_ensure(obj); - (*meta).prototype = crate::value::js_nanbox_pointer(proto as i64).to_bits(); - assert!( - !crate::object::proto_validity::object_is_marked_prototype(proto as usize), - "this test is vacuous unless the prototype really is unmarked" - ); - assert!( - inherited_read_cache_prime(obj, key("cov_u")).is_none(), - "the cache recorded an entry through an UNMARKED prototype: a key \ - added to that prototype would bump no validity and the entry \ - would keep serving a stale value" - ); - } -} - -/// **The walk never passes a hop that was unmarked when it began.** -/// -/// This is not a property of this cache. It is what somebody ELSE's mechanism -/// rests on: an ABSENT verdict — "this key is on nothing in the whole chain" — -/// is invalidated by `proto_validity`, which bumps only for MARKED prototypes. -/// So an absent verdict is sound only if every hop on an exhausted chain was -/// marked, and the only reason that is true is the mark-and-abandon below: -/// the walk marks one unmarked hop, abandons without recording, and the NEXT -/// read gets one hop further. Proving exhaustion therefore implies every hop -/// it passed was already marked. -/// -/// Nothing states that, and removing the abandon is an obvious-looking -/// optimisation — it costs one declined read per hop and appears to buy -/// nothing. This test is the guard. It is red on a build where the walk -/// proceeds past an unmarked hop, whether that hop gets marked in passing or -/// not at all, which are the two shapes such a "cleanup" takes. -/// -/// A comment would lose this argument to a plausible cleanup in six months; a -/// red test does not. -#[test] -fn the_walk_never_passes_a_hop_that_was_unmarked_when_it_began() { - let _scope = PrimeScope::new(); - unsafe { - // O -> P1 -> P2, with `irc_deep` only on P2, and BOTH hops installed - // without the funnel so neither is marked. That is the state a future - // `[[Prototype]]` install site that forgot to mark would leave, and it - // is also the state every chain is in before its first walk. - let p2 = crate::object::js_object_alloc(0, 4); - set(p2, "irc_deep", 21.0); - let p1 = crate::object::js_object_alloc(0, 4); - set(p1, "irc_mid", 1.0); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 0.0); - let link = |from: *mut ObjectHeader, to: *mut ObjectHeader| { - let meta = crate::object::object_meta_ensure(from); - (*meta).prototype = crate::value::js_nanbox_pointer(to as i64).to_bits(); - }; - link(p1, p2); - link(obj, p1); - - let marked = |o: *mut ObjectHeader| { - crate::object::proto_validity::object_is_marked_prototype(o as usize) - }; - assert!( - !marked(p1) && !marked(p2), - "vacuous unless both hops really start unmarked" - ); - - let k = key("irc_deep"); - - // Walk 1 reaches P1, finds it unmarked, marks it and stops. If it had - // continued, P2 would be marked too — or, on a build that dropped the - // marking entirely, neither would be. - assert!( - inherited_read_cache_prime(obj, k).is_none(), - "the walk resolved through hops that were unmarked when it began" - ); - assert!(marked(p1), "the walk must mark the hop it stopped at"); - assert!( - !marked(p2), - "the walk passed P1 in the same pass that marked it. An absent \ - verdict recorded through a chain walked this way rests on a hop \ - that a key add would not invalidate, and goes stale silently" - ); - - // Walk 2 gets exactly one hop further, for the same reason. - assert!(inherited_read_cache_prime(obj, k).is_none()); - assert!(marked(p2), "the second walk must reach and mark P2"); - - // Only now, with every hop marked BEFORE the walk begins, may the walk - // resolve — and this is the state in which an exhausted chain may be - // recorded as absent. - let resolved = inherited_read_cache_prime(obj, k) - .expect("with every hop marked, the walk must resolve"); - assert_eq!(f64::from_bits(resolved.bits()), 21.0); - } -} - -#[test] -fn a_null_prototype_receiver_never_primes() { - let _scope = PrimeScope::new(); - unsafe { - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of( - boxed(obj), - f64::from_bits(crate::value::TAG_NULL), - ); - assert!(inherited_read_cache_prime(obj, key("irc_a")).is_none()); - assert_eq!(inherited_read_cache_primes(), 0); - } -} - -extern "C" fn forty_two_getter( - _c: *const crate::closure::ClosureHeader, - _this: crate::closure::JsThis, -) -> f64 { - 42.0 -} - -extern "C" fn forty_two_raw_getter(_this: f64) -> f64 { - 42.0 -} - -/// A CLASS accessor on `proto`: its pair carries a compiled getter entry, as -/// a ClassBody `get` does (`decl_accessors.rs`). -unsafe fn install_class_getter(proto: *mut ObjectHeader, name: &str) { - let getter = crate::closure::js_closure_alloc(crate::fn_info!(forty_two_getter, 0), 0); - crate::object::set_builtin_accessor_pair( - proto as usize, - name.to_string(), - crate::object::accessor_pair::Accessor { - get: crate::value::js_nanbox_pointer(getter as i64).to_bits(), - set: 0, - raw_get: forty_two_raw_getter as *const () as usize, - ..Default::default() - }, - crate::object::PropertyAttrs::new(true, false, true), - ); -} - -/// A closure-only accessor (a builtin prototype's, or one `defineProperty` -/// installed) keeps the generic path, whose receiver handling it may need -/// (`Map.prototype.size` on a subclass instance). Sabotage: dropping the -/// compiled-entry requirement in the walk primes it. -#[test] -fn a_closure_only_accessor_is_not_primed() { - let _scope = PrimeScope::new(); - unsafe { - let proto = crate::object::js_object_alloc(0, 4); - set(proto, "irc_closure", 7.0); - let getter = crate::closure::js_closure_alloc(crate::fn_info!(forty_two_getter, 0), 0); - crate::object::descriptor_state::set_accessor_descriptor( - proto as usize, - "irc_closure".to_string(), - crate::object::descriptor_state::AccessorDescriptor { - get: crate::value::js_nanbox_pointer(getter as i64).to_bits(), - set: 0, - }, - ); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(proto)); - // Setup reads (a `constructor` read, a builtin lookup) prime entries of - // their own through hook D; count only the operation under test. - test_reset_counters(); - assert!(inherited_read_cache_prime(obj, key("irc_closure")).is_none()); - assert_eq!(inherited_read_cache_primes(), 0); - } -} - -/// Charter step 3: a CLASS accessor on the prototype primes an ACCESSOR entry -/// (the holder's slot holds the pair), and a hit runs the compiled getter. -/// Sabotage: serving the entry as a data slot returns the pair word instead -/// of 42. -#[test] -fn an_accessor_on_the_prototype_primes_an_accessor_entry() { - let _scope = PrimeScope::new(); - unsafe { - let proto = crate::object::js_object_alloc(0, 4); - set(proto, "irc_acc", 7.0); - install_class_getter(proto, "irc_acc"); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(proto)); - let k = key("irc_acc"); - // Setup reads (a `constructor` read, a builtin lookup) prime entries of - // their own through hook D; count only the operation under test. - test_reset_counters(); - - let primed = inherited_read_cache_prime(obj, k).expect("the accessor primes"); - assert_eq!( - f64::from_bits(primed.bits()), - 42.0, - "the prime runs the getter" - ); - assert_eq!(inherited_read_cache_primes(), 1); - match inherited_read_cache_lookup(obj, k) { - Lookup::Hit(v) => assert_eq!(f64::from_bits(v.bits()), 42.0, "a hit runs the getter"), - _ => panic!("the accessor entry must serve the next read"), - } - } -} - -#[test] -fn an_undefined_holder_slot_never_primes() { - let _scope = PrimeScope::new(); - unsafe { - let proto = crate::object::js_object_alloc(0, 4); - crate::object::js_object_set_field_by_name( - proto, - key("irc_u"), - f64::from_bits(crate::value::TAG_UNDEFINED), - ); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(proto)); - assert!( - inherited_read_cache_prime(obj, key("irc_u")).is_none(), - "the generic getter treats an inherited undefined as a miss; a \ - cache that answers `undefined` here diverges from it" - ); - } -} - -#[test] -fn a_refusal_is_remembered_so_the_chain_is_walked_once() { - let _scope = PrimeScope::new(); - unsafe { - // A key on no prototype at all: the walk runs off the chain. - let proto = crate::object::js_object_alloc(0, 4); - set(proto, "irc_other", 7.0); - // End the chain at an explicit null: a key on no hop of a chain that - // reaches `%Object.prototype%` is now a confirmed ABSENT entry (see - // `an_absent_key_primes_a_confirmed_absent_entry`), so a refusal needs - // a chain the walk genuinely cannot describe. - crate::object::js_object_set_prototype_of( - boxed(proto), - f64::from_bits(crate::value::TAG_NULL), - ); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(proto)); - let k = key("irc_absent"); - // Setup reads (a `constructor` read, a builtin lookup) prime entries of - // their own through hook D; count only the operation under test. - test_reset_counters(); - - assert!(inherited_read_cache_prime(obj, k).is_none()); - assert_eq!( - inherited_read_cache_neg_served(), - 0, - "the first walk had nothing to be served from" - ); - assert!(matches!( - inherited_read_cache_lookup(obj, k), - Lookup::Declined - )); - assert_eq!( - inherited_read_cache_neg_served(), - 1, - "the refusal was not remembered — every read of a key this cache \ - cannot serve then re-walks the whole chain, which is slower than \ - having no cache at all" - ); - assert_eq!(inherited_read_cache_primes(), 0); - } -} - -#[test] -fn a_refusal_caused_by_a_value_is_not_remembered() { - let _scope = PrimeScope::new(); - unsafe { - // The holder slot holds `undefined`, which the generic getter treats - // as a miss. A plain store can replace it with a real value, and a - // plain store transitions nothing and bumps no epoch — so remembering - // THIS refusal would leave the pair declined for the life of the - // process. Every other refusal is a function of a shape or a - // descriptor, which is why only this class is excluded. - let proto = crate::object::js_object_alloc(0, 4); - crate::object::js_object_set_field_by_name( - proto, - key("irc_later"), - f64::from_bits(crate::value::TAG_UNDEFINED), - ); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(proto)); - let k = key("irc_later"); - assert!(inherited_read_cache_prime(obj, k).is_none()); - assert!( - matches!(inherited_read_cache_lookup(obj, k), Lookup::Unknown), - "a value-caused refusal was recorded as a standing decline" - ); - - set(proto, "irc_later", 5.0); - let value = inherited_read_cache_prime(obj, k).expect( - "the slot now holds a real value and the pair must become \ - cacheable again", - ); - assert_eq!(f64::from_bits(value.bits()), 5.0); - } -} - -#[test] -fn adding_the_key_to_the_prototype_re_opens_a_remembered_refusal() { - let _scope = PrimeScope::new(); - unsafe { - // The key is on NO hop, so the walk refuses at the end of the chain. - // That refusal is remembered — but against the hop shapes it saw, so - // the `proto.a = 1` that makes the pair resolvable invalidates it. - let proto = crate::object::js_object_alloc(0, 4); - set(proto, "irc_other", 1.0); - // End the chain at an explicit null: a key on no hop of a chain that - // reaches `%Object.prototype%` is now a confirmed ABSENT entry (see - // `an_absent_key_primes_a_confirmed_absent_entry`), so a refusal needs - // a chain the walk genuinely cannot describe. - crate::object::js_object_set_prototype_of( - boxed(proto), - f64::from_bits(crate::value::TAG_NULL), - ); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(proto)); - let k = key("irc_late"); - // Setup reads (a `constructor` read, a builtin lookup) prime entries of - // their own through hook D; count only the operation under test. - test_reset_counters(); - assert!(inherited_read_cache_prime(obj, k).is_none()); - assert!(matches!( - inherited_read_cache_lookup(obj, k), - Lookup::Declined - )); - - set(proto, "irc_late", 9.0); - assert!( - matches!(inherited_read_cache_lookup(obj, k), Lookup::Unknown), - "the standing decline outlived the key add that resolves it" - ); - let value = inherited_read_cache_prime(obj, k).expect("prime"); - assert_eq!(f64::from_bits(value.bits()), 9.0); - } -} - -#[test] -fn a_nursery_prototype_primes() { - let _scope = PrimeScope::new(); - unsafe { - // The refusal this replaces cost +264 instructions per inherited read - // and returned nothing: a read-only loop never promotes anything, so - // under it NO ordinary program's prototype was ever cacheable. - let (obj, proto) = one_level(); - assert_eq!( - crate::arena::classify_heap_generation(proto as usize), - crate::arena::HeapGeneration::Nursery, - "fixture is vacuous — the prototype was not in the nursery, so \ - this test would pass with the old-generation refusal in place" - ); - // Setup reads (a `constructor` read, a builtin lookup) prime entries of - // their own through hook D; count only the operation under test. - test_reset_counters(); - assert!(inherited_read_cache_prime(obj, key("irc_a")).is_some()); - assert_eq!(inherited_read_cache_primes(), 1); - } -} - -#[test] -fn the_prune_drops_an_entry_whose_holder_died() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - assert!(inherited_read_cache_hit(obj, k).is_some()); - - let holder = proto as usize; - prune_dead_inherited_cache_entries(&|addr| addr == holder); - assert!( - inherited_read_cache_hit(obj, k).is_none(), - "an entry survived its holder's death; the next allocation at that \ - address turns it into a false hit" - ); - } -} - -#[test] -fn the_prune_drops_an_entry_whose_key_died() { - let _scope = PrimeScope::new(); - unsafe { - let (obj, _proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - let key_addr = k as usize; - prune_dead_inherited_cache_entries(&|addr| addr == key_addr); - assert!(inherited_read_cache_hit(obj, k).is_none()); - } -} - -#[test] -fn a_proxy_in_the_chain_never_primes() { - let _scope = PrimeScope::new(); - unsafe { - // The twin first: the SAME target object, reached directly, does - // prime. Without it a decline proves nothing — every other refusal in - // this module would produce the same `None`. - let target = crate::object::js_object_alloc(0, 4); - set(target, "irc_a", 7.0); - let direct = crate::object::js_object_alloc(0, 4); - set(direct, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(direct), boxed(target)); - assert!( - inherited_read_cache_prime(direct, key("irc_a")).is_some(), - "fixture is vacuous — the target is not cacheable even unwrapped" - ); - - let handler = crate::object::js_object_alloc(0, 4); - let proxy = crate::proxy::js_proxy_new(boxed(target), boxed(handler)); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), proxy); - let before = inherited_read_cache_primes(); - assert!( - inherited_read_cache_prime(obj, key("irc_a")).is_none(), - "a proxy hop primed; the entry would then read the TARGET's slot \ - and the `get` trap would never run" - ); - assert_eq!(inherited_read_cache_primes(), before); - } -} - -#[test] -fn the_validity_guard_is_load_bearing() { - // Sabotage: freeze the epoch the entry recorded, then delete the key from - // the prototype. With the guard working the hit must still fail (via the - // shape stamps, if they happen to change) OR the entry must be gone; the - // assertion that matters is that the value never comes back stale. - let _scope = PrimeScope::new(); - unsafe { - let (obj, proto) = one_level(); - let k = key("irc_a"); - inherited_read_cache_prime(obj, k).expect("prime"); - let epoch_before = crate::object::prop_plan::prop_plan_semantic_epoch(); - crate::object::js_object_delete_field(proto, k); - let epoch_after = crate::object::prop_plan::prop_plan_semantic_epoch(); - assert_ne!( - epoch_before, epoch_after, - "`delete` no longer bumps the semantic epoch, so the invalidation \ - this cache rests on has silently stopped happening" - ); - } -} - -#[test] -fn the_cache_can_be_turned_off_for_an_a_b_measurement() { - // The knob exists so one binary can be measured with and without the - // cache. If it stopped being read, the two arms would be the same arm. - assert!( - cache_enabled() || !cache_enabled(), - "cache_enabled must be reachable" - ); -} - -/// The emitted hit is a GC leaf in codegen's call-effect tables, so it must -/// never run a getter: an accessor entry declines there (`TAG_HOLE`) and is -/// served by the miss handler. Sabotage: dropping the accessor decline in -/// `js_inherited_read_cache_hit_f64` makes it return 42. -#[test] -fn the_emitted_leaf_hit_never_runs_a_getter() { - let _scope = PrimeScope::new(); - unsafe { - let proto = crate::object::js_object_alloc(0, 4); - set(proto, "irc_leaf", 7.0); - install_class_getter(proto, "irc_leaf"); - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - crate::object::js_object_set_prototype_of(boxed(obj), boxed(proto)); - let k = key("irc_leaf"); - inherited_read_cache_prime(obj, k).expect("the accessor primes"); - assert_eq!( - super::js_inherited_read_cache_hit_f64(obj, k).to_bits(), - crate::value::TAG_HOLE, - "a GC-leaf call must not run user code" - ); - assert!(matches!( - inherited_read_cache_lookup(obj, k), - Lookup::Hit(_) - )); - } -} - -/// #11507: the table is zero-allocated rather than filled with `EMPTY_ENTRY`, -/// so a thread's first view of it must be `EMPTY_ENTRY` in every slot. -#[test] -fn fresh_thread_cache_reads_empty_everywhere() { - std::thread::spawn(|| { - INHERITED_READ_CACHE.with(|cell| { - let cache = unsafe { &*cell.get() }; - assert_eq!(cache.len(), CACHE_SIZE); - for entry in cache.iter() { - assert_eq!(*entry, EMPTY_ENTRY); - } - }); - }) - .join() - .unwrap(); -} - -// --- the default `%Object.prototype%` link and ABSENT entries (#10495) ----- - -/// `%Object.prototype%`, marked as a prototype so the walk records through it -/// on the first attempt instead of marking it and abandoning (which is its -/// own, separately tested behaviour). -unsafe fn marked_object_prototype() -> *mut ObjectHeader { - let op = crate::array::object_prototype_addr(); - assert_ne!(op, 0, "the realm must be materialized (PrimeScope does it)"); - crate::object::proto_validity::mark_object_as_prototype(op); - op as *mut ObjectHeader -} - -/// A receiver with no class and no recorded `[[Prototype]]`: what an object -/// literal is. -unsafe fn plain_receiver() -> *mut ObjectHeader { - let obj = crate::object::js_object_alloc(0, 4); - set(obj, "irc_own", 1.0); - obj -} - -/// Before the default link existed, this pair declined unrecorded and every -/// read re-ran the whole generic walk to `Object.prototype` (#10495's -/// `literal_absent3`, lru-cache's options bag). Sabotage: an ABSENT hit that -/// fell through to the holder load would read `Object.prototype`'s slot -/// `u32::MAX - 1`. -#[test] -fn an_absent_key_primes_a_confirmed_absent_entry() { - let _scope = PrimeScope::new(); - unsafe { - marked_object_prototype(); - let obj = plain_receiver(); - let k = key("irc_nowhere_at_all"); - test_reset_counters(); - let primed = inherited_read_cache_prime(obj, k).expect("the walk reaches the end"); - assert!(primed.is_undefined()); - assert_eq!( - inherited_read_cache_primes(), - 1, - "the ABSENT entry was not confirmed" - ); - for _ in 0..3 { - match inherited_read_cache_lookup(obj, k) { - Lookup::Hit(v) => assert!(v.is_undefined()), - _ => panic!("the confirmed ABSENT entry must serve the read"), - } - } - assert_eq!(inherited_read_cache_hits(), 3); - let leaf = js_inherited_read_cache_hit_f64(obj, k); - assert_eq!( - leaf.to_bits(), - crate::value::TAG_UNDEFINED, - "the emitted leaf must serve an ABSENT entry as undefined, not decline" - ); - } -} - -#[test] -fn adding_the_key_to_object_prototype_retires_an_absent_entry() { - let _scope = PrimeScope::new(); - unsafe { - let op = marked_object_prototype(); - let obj = plain_receiver(); - let k = key("irc_later_on_object_prototype"); - assert!(inherited_read_cache_prime(obj, k) - .expect("absent") - .is_undefined()); - assert!(matches!( - inherited_read_cache_lookup(obj, k), - Lookup::Hit(_) - )); - - set(op, "irc_later_on_object_prototype", 3.0); - assert!( - matches!(inherited_read_cache_lookup(obj, k), Lookup::Unknown), - "an ABSENT entry outlived a key added to Object.prototype — a stale \ - `undefined` is a wrong answer, not a slow one" - ); - // Object.prototype carries many keys, so a late one lands in a - // spilled slot, which the walk declines by design. Whatever the - // cache does now, no read may see the old `undefined`. - if let Some(value) = inherited_read_cache_prime(obj, k) { - assert_eq!(f64::from_bits(value.bits()), 3.0); - } - if let Lookup::Hit(v) = inherited_read_cache_lookup(obj, k) { - assert_eq!(f64::from_bits(v.bits()), 3.0); - } - let read = crate::object::js_object_get_field_by_name(obj, k); - assert_eq!(f64::from_bits(read.bits()), 3.0); - } -} - -/// The confirmation is what lets the walk answer from key lists alone: for -/// every key, what the prime returns and what a later hit returns must be -/// exactly what the generic getter answers — including names the generic -/// getter synthesizes (`constructor`, `__proto__`) rather than reads from a -/// key list. -#[test] -fn the_prime_and_the_hit_answer_what_the_generic_getter_answers() { - let _scope = PrimeScope::new(); - unsafe { - marked_object_prototype(); - let obj = plain_receiver(); - for name in [ - "toString", - "hasOwnProperty", - "valueOf", - "isPrototypeOf", - "constructor", - "__proto__", - "irc_not_anywhere", - ] { - let k = key(name); - let truth = - crate::object::field_get_set::get_field_by_name_past_inherited_cache(obj, k); - if let Some(primed) = inherited_read_cache_prime(obj, k) { - assert_eq!(primed.bits(), truth.bits(), "prime of {name} diverged"); - } - if let Lookup::Hit(hit) = inherited_read_cache_lookup(obj, k) { - assert_eq!(hit.bits(), truth.bits(), "hit of {name} diverged"); - } - } - } -} - -/// Hook D: a by-name read primes on the SECOND miss of a (shape, key) pair -/// and is served from the entry after that. -#[test] -fn a_by_name_read_primes_on_its_second_miss() { - let _scope = PrimeScope::new(); - unsafe { - marked_object_prototype(); - let obj = plain_receiver(); - let k = key("irc_by_name_absent"); - test_reset_counters(); - assert!(crate::object::js_object_get_field_by_name(obj, k).is_undefined()); - assert_eq!( - inherited_read_cache_primes(), - 0, - "a first sighting must not prime" - ); - assert!(crate::object::js_object_get_field_by_name(obj, k).is_undefined()); - assert_eq!( - inherited_read_cache_primes(), - 1, - "the second sighting primes" - ); - assert!(crate::object::js_object_get_field_by_name(obj, k).is_undefined()); - assert_eq!(inherited_read_cache_hits(), 1, "the third read is served"); - } -} - -/// A key minted fresh for every read never repeats its address, so hook D -/// never pays a prime (a walk AND a confirming getter) for it. -#[test] -fn a_fresh_key_per_read_never_primes_by_name() { - let _scope = PrimeScope::new(); - unsafe { - marked_object_prototype(); - let obj = plain_receiver(); - test_reset_counters(); - for _ in 0..16 { - let k = key("irc_fresh_every_time"); - assert!(crate::object::js_object_get_field_by_name(obj, k).is_undefined()); - } - assert_eq!(inherited_read_cache_primes(), 0); - } -} - -/// The size figures in the `CACHE_SIZE` and `MAX_HOPS` docs are this number. -#[test] -fn an_entry_is_the_size_the_docs_state() { - assert_eq!(std::mem::size_of::(), 128); -} diff --git a/crates/perry-runtime/src/object/method_site.rs b/crates/perry-runtime/src/object/method_site.rs index a0caaeaa5a..db26e46e47 100644 --- a/crates/perry-runtime/src/object/method_site.rs +++ b/crates/perry-runtime/src/object/method_site.rs @@ -76,9 +76,9 @@ //! //! # Agents //! -//! Entries are primed only on the primary agent, like the chain-store -//! verdicts. A program with workers emits thread-local site slots (#10399), so -//! a worker never reads a primary-heap closure through a site. +//! Inherited entries prime only on the primary agent. The first worker start +//! clears them and prevents further inherited primes, so a worker cannot +//! follow a holder from the primary heap through a process-global site. use crate::object::ObjectHeader; @@ -296,9 +296,8 @@ fn stats_report_enabled() -> bool { } let (hd, ha, hr) = read_holder::read_holder_stats(); eprintln!( - "[method-site] primes_own={a} primes_inherited={b} primes_function={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_holder_refused={hr} marked_value_write_bumps={}{refused}", - method_site_function_primes(), - crate::object::proto_validity::marked_value_write_bumps() + "[method-site] primes_own={a} primes_inherited={b} primes_function={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_holder_refused={hr}{refused}", + method_site_function_primes() ); } unsafe { libc::atexit(report) }; @@ -978,7 +977,7 @@ unsafe fn next_prototype(obj: *const ObjectHeader) -> *const ObjectHeader { super::class_prototype_object(class_id) } -/// Root scan: every inherited entry's closure is marked and rewritten. +/// Root scan: every inherited entry's holder is marked and rewritten. pub(crate) fn scan_method_site_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { if let Ok(sites) = METHOD_SITES.lock() { for &site in sites.iter() { diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index ac037f7710..3d998b28ea 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -381,13 +381,12 @@ pub(crate) unsafe fn prime_read_holder( let recv = ordinary_receiver(obj as usize)?; // Cheap pre-walk: a receiver the entry could never describe keeps the // caller's path and pays nothing for the getter below. A site with no - // cache yet stays without one, so the slow entry keeps asking the - // inherited-read cache for it. + // cache yet stays without one and uses the generic getter. // // A walk that ends at the default link needs `%Object.prototype%`, which // is materialized lazily: while it is unresolved the walk cannot pin it, - // and refusing here would leave the site to the inherited-read cache for - // good (the getter below is what resolves it). So an unresolved realm + // and refusing here would leave the site on the generic path (the getter + // below is what resolves it). So an unresolved realm // does not decide the pre-walk; the walk after the getter does. let realm_pending = crate::array::object_prototype_addr_if_resolved() == 0; if !holder_name_admitted(name) @@ -403,7 +402,7 @@ pub(crate) unsafe fn prime_read_holder( let scope = crate::gc::RuntimeHandleScope::new(); let handle = scope.root_raw_mut_ptr(obj as *mut ObjectHeader); let (value, obj) = handle.across_mut::(|| { - crate::object::field_get_set::get_field_by_name_past_inherited_cache(obj, key) + crate::object::field_get_set::get_field_by_name_after_site_miss(obj, key) }); // From here a refusal has already run the getter, so the site latches: // the next miss must not walk and run it again only to refuse again. diff --git a/crates/perry-runtime/src/object/mod.rs b/crates/perry-runtime/src/object/mod.rs index 203c20d612..871643b5ca 100644 --- a/crates/perry-runtime/src/object/mod.rs +++ b/crates/perry-runtime/src/object/mod.rs @@ -136,7 +136,6 @@ pub(crate) use global_fetch::scan_pending_fetch_signal_root_mut; pub(crate) mod chain_store; mod global_this; pub mod handle_expando; -pub(crate) mod inherited_read_cache; pub(crate) mod prop_plan; pub(crate) mod proto_validity; pub(crate) use global_this::{ diff --git a/crates/perry-runtime/src/object/proto_validity.rs b/crates/perry-runtime/src/object/proto_validity.rs index 2848fb72d2..8b2c44e843 100644 --- a/crates/perry-runtime/src/object/proto_validity.rs +++ b/crates/perry-runtime/src/object/proto_validity.rs @@ -1,103 +1,16 @@ -//! Prototype-mutation validity: one word that stands for "no object anybody -//! inherits from has changed structurally since you looked". +//! Prototype-mutation validity for cached key-add store verdicts. //! -//! # The problem this replaces +//! A chain-store verdict records that no inherited setter or non-writable +//! property blocks a key add. It may depend on prototype hops, so the +//! [[Prototype]] installation funnel and chain-store prime mark those objects. +//! A structural mutation of a marked hop, or a semantic property event, +//! advances the global word. The receiver's own ShapeId covers changes to +//! its own keys and prototype edge. A plain value overwrite does not change +//! the chain-store verdict; readers of inherited values load the holder slot +//! under holder-shape guards and do not use this word. //! -//! A mutation of an object that is used as a prototype is invisible to the -//! objects that inherit from it. Perry does not record that an ordinary object -//! is somebody's prototype, so `proto.b = 1` transitions `proto`'s shape and -//! nothing else: no epoch moves, no instance is touched, and a cache that -//! remembers "key `a` lives on `proto` at slot 3" has no way to notice. -//! -//! The inherited-read cache's first answer (#10834) was to re-prove the chain -//! on every hit: one ShapeId compare per hop, up to four dependent loads -//! through prototype objects that are usually cold. That is correct, and it -//! has two costs. It is proportional to the DEPTH of the chain, so a deep -//! chain pays for its depth on every read; and, decisively for the emitted -//! sequence this campaign is aiming at, it is a LOOP — compiled code at a -//! property-read site cannot emit a variable number of compares, so as long as -//! the chain check is per-hop the hit can only live behind a call. -//! -//! # The mechanism -//! -//! This is V8's prototype validity cell, collapsed to one global counter: -//! -//! 1. An object is MARKED (`OBJECT_META_FLAG_IS_PROTOTYPE` in its -//! `ObjectMeta`) by the `[[Prototype]]` INSTALL funnel — every link kind in -//! `prototype_chain::object_set_static_prototype_impl`, plus -//! `class_prototype_object_root_store`. The inherited-read cache does NOT -//! mark: it REFUSES to record a hop that is not already marked. -//! -//! The cache's prime is the SAFETY NET, not the authority: when it meets an -//! unmarked hop it marks that hop and ABANDONS the walk without recording -//! anything, so the next read of the pair primes normally. Coverage is -//! therefore self-healing — an install route this funnel misses costs one -//! declined read, not a permanent loss — while the invariant that matters -//! still holds absolutely: **no entry is ever recorded through a hop that -//! was not already marked before the walk began.** -//! -//! That polarity is the point. The first version of this flag was set BY -//! the prime, for the hop it was about to record, which made the coverage -//! argument trivial but meant any way of losing a mark produced a STALE -//! VALUE. It lived in `GcHeader::_reserved` bit 13 — which -//! `layout::set_layout_state` CLEARS on transitions that have nothing to do -//! with prototypes, so marks were erasable and the failure was invisible. -//! Both halves of that are fixed here: the flag moved to `ObjectMeta`, -//! where nothing else writes it, and the prime no longer records through a -//! hop it marked in the same breath. -//! 2. Any STRUCTURAL mutation of a marked object bumps [`proto_validity`]. -//! Structural means "the shape word changed": key add, key delete, -//! descriptor install, attribute change, `setPrototypeOf`. Every one of -//! those publishes through -//! `shapes::stamp_object_shape_id_with_carrier_note`, the runtime's single -//! structural-mutation publication funnel, where [`note_object_shape_stamped`] -//! sits. -//! 3. The same counter is bumped by `prop_plan::prop_plan_epoch_bump`, so it -//! also stands for everything the semantic property epoch stands for -//! (descriptor installs and clears, `delete`, per-instance prototype -//! recording, class-prototype-object registration, parent-static linking). -//! A descriptor install or clear, or a `delete`, bumps it only when its -//! owner can be a hop of a recorded chain — a MARKED ordinary object, or -//! one that cannot be classified ([`mutation_owner_may_be_a_recorded_hop`]). -//! Every consumer records through marked hops only, and the receiver's own -//! such mutations transition its ShapeId, so an unmarked object's -//! descriptors cannot change a recorded answer. Without that gate every -//! `Function.prototype.bind` (which names its result through descriptor -//! installs) invalidated every cached verdict in the process. Folding the two -//! into one word is what lets a cached entry re-prove itself with ONE load -//! and ONE compare instead of two of each. -//! -//! 4. A plain value store to an EXISTING key of a marked object bumps it too -//! ([`note_marked_value_write`], owner decision D3(b)): an inherited -//! method-site entry (`object::method_site`) memoizes the method CLOSURE, -//! not (holder, slot), so a replaced value must invalidate it. Store caches -//! never learn a marked object's shape ([`store_cache_may_learn`]), so every -//! such store reaches a runtime funnel that calls it. Entries that record -//! (holder, slot) and load the value on every hit do not need this, and -//! must not assume the word stays put across such a store either. -//! -//! # Why the counter is global, and what that costs -//! -//! Per-prototype validity would need a word per prototype object and a load of -//! it per hop — which is the per-hop walk again, one indirection shallower. A -//! single global word is one load, covers a chain of ANY depth, and is the -//! only form an emitted inline check can use. -//! -//! What it buys in over-invalidation is real but small, because the bump is -//! gated on the mark: mutating an ordinary object — the overwhelming majority -//! of all mutation — bumps nothing. Only mutating an object that something -//! actually inherits from does, and then it invalidates every cached inherited -//! read rather than the ones that name it. Measured (`/root/pv`, one -//! structural mutation of an UNRELATED prototype per 64 inherited reads, which -//! a per-prototype design would not invalidate at all): see the PR body. -//! -//! # Cost when nothing is marked -//! -//! [`note_object_shape_stamped`] is on the structural-mutation path, which a -//! program that builds objects in a loop runs hot. Until something is marked -//! it is one relaxed `bool` load and a predictable not-taken branch: the -//! latch is only ever set by a prime of the inherited-read cache, so a program -//! with no inherited reads never pays the header load. +//! The remaining global word can be replaced by per-hop shape facts in the +//! key-add store path (design decision D-A2). use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -134,7 +47,7 @@ pub(crate) fn proto_validity() -> u64 { PROTO_VALIDITY.load(Ordering::Relaxed) } -/// Invalidate every cached inherited read. Callers are rare, cold paths by +/// Invalidate every cached chain-store verdict. Callers are rare, cold paths by /// construction: a structural mutation of an object used as a prototype, or a /// semantic property event. #[inline] @@ -353,10 +266,8 @@ pub(crate) unsafe fn object_is_marked_prototype(obj: usize) -> bool { /// Can a descriptor install or clear, or a `delete`, on `owner` change what /// a cached chain verdict answers? /// -/// Every consumer of [`proto_validity`] records a verdict only through -/// prototype hops that are MARKED ordinary objects (the inherited-read cache -/// refuses an unmarked or non-object hop; `object::chain_store` marks every -/// hop its verdict depends on before it records). What such a mutation can +/// The chain-store verdict records through MARKED ordinary prototype hops; +/// `object::chain_store` marks each hop it depends on before recording. What such a mutation can /// change on its RECEIVER is covered by the receiver's own ShapeId, which /// every descriptor install, clear and delete transitions. So the mutation /// matters to a verdict only when `owner` can be one of those hops: a marked @@ -385,7 +296,7 @@ pub(crate) unsafe fn mutation_owner_may_be_a_recorded_hop(owner: usize) -> bool /// The hook in the structural-mutation publication funnel /// (`shapes::stamp_object_shape_id_with_carrier_note`): a shape word that -/// CHANGED on a MARKED object invalidates every cached inherited read. +/// CHANGED on a MARKED object invalidates every cached chain-store verdict. /// /// `previous == published` is the re-publication of an unchanged descriptor — /// the read-side `lookup_ways` restamp, and a birth stamp that agrees with the @@ -407,46 +318,3 @@ pub(crate) unsafe fn note_object_shape_stamped(obj: usize, previous: u32, publis #[cfg(test)] #[path = "proto_validity_tests.rs"] mod tests; - -static MARKED_VALUE_WRITE_BUMPS: AtomicU64 = AtomicU64::new(0); - -/// Bumps taken by [`note_marked_value_write`] (diagnostics, tests). -pub(crate) fn marked_value_write_bumps() -> u64 { - MARKED_VALUE_WRITE_BUMPS.load(Ordering::Relaxed) -} - -/// A value was written into an EXISTING slot of `obj`. When `obj` is a marked -/// prototype this invalidates every cached inherited verdict: a method-call -/// site memoizes the CLOSURE an inherited key resolves to (owner decision -/// D3(b)), so a plain store over it must move the word just as a structural -/// change does. Unmarked objects — nearly every store — pay the latch load, -/// and a meta-null test once anything is marked. -/// -/// # Safety -/// `obj` is a live `ObjectHeader`. -#[inline] -pub(crate) unsafe fn note_marked_value_write(obj: *const crate::object::ObjectHeader) { - if !any_prototype_marked() { - return; - } - let meta = (*obj).meta; - if !meta.is_null() && (*meta).flags & crate::object::OBJECT_META_FLAG_IS_PROTOTYPE != 0 { - MARKED_VALUE_WRITE_BUMPS.fetch_add(1, Ordering::Relaxed); - bump_proto_validity(); - } -} - -/// Whether a store cache may LEARN `obj`'s shape: never for a marked -/// prototype, so every write to one reaches a runtime funnel that calls -/// [`note_marked_value_write`]. -/// -/// # Safety -/// `obj` is a live `ObjectHeader`. -#[inline] -pub(crate) unsafe fn store_cache_may_learn(obj: *const crate::object::ObjectHeader) -> bool { - if !any_prototype_marked() { - return true; - } - let meta = (*obj).meta; - meta.is_null() || (*meta).flags & crate::object::OBJECT_META_FLAG_IS_PROTOTYPE == 0 -} diff --git a/crates/perry-runtime/src/object/slot_store.rs b/crates/perry-runtime/src/object/slot_store.rs index 5c8d937e41..9165b8bf59 100644 --- a/crates/perry-runtime/src/object/slot_store.rs +++ b/crates/perry-runtime/src/object/slot_store.rs @@ -1,9 +1,7 @@ //! The two runtime funnels that store a JS value into an object's inline //! slot (split out of `object/mod.rs`, which is at the file-size cap). -//! Every one also tells `proto_validity` about writes to prototype-marked -//! objects (owner decision D3(b)), and runs the field-representation store -//! check (charter step 5, `field_rep_store::checked_slot_bits`) before the -//! value reaches the slot. +//! Every one runs the field-representation store check +//! (`field_rep_store::checked_slot_bits`) before the value reaches the slot. use super::ObjectHeader; #[inline] @@ -12,7 +10,6 @@ pub(crate) unsafe fn store_object_field_slot( field_index: usize, value_bits: u64, ) { - super::proto_validity::note_marked_value_write(obj); let value_bits = super::field_rep_store::checked_slot_bits(obj, field_index, value_bits); let fields_ptr = (obj as *mut u8).add(std::mem::size_of::()) as *mut u64; let slot = fields_ptr.add(field_index); diff --git a/crates/perry-runtime/src/object/spill.rs b/crates/perry-runtime/src/object/spill.rs index abcbd4a0fb..f0062fa96e 100644 --- a/crates/perry-runtime/src/object/spill.rs +++ b/crates/perry-runtime/src/object/spill.rs @@ -599,9 +599,6 @@ pub(crate) fn learned_inline_field_count(class_id: u32) -> u32 { /// overflow slots fill in sequence. #[inline] pub(crate) fn overflow_set(obj_ptr: usize, field_index: usize, vbits: u64) { - unsafe { - crate::object::proto_validity::note_marked_value_write(obj_ptr as *const ObjectHeader) - }; if object_spill_enabled() && field_index < SPILL_MAX_FIELD_INDEX && unsafe { spill_capable_owner(obj_ptr) } diff --git a/crates/perry-runtime/src/proxy/put_value.rs b/crates/perry-runtime/src/proxy/put_value.rs index 239762afd7..44583fa067 100644 --- a/crates/perry-runtime/src/proxy/put_value.rs +++ b/crates/perry-runtime/src/proxy/put_value.rs @@ -179,21 +179,6 @@ pub extern "C" fn js_put_value_set( if unsafe { crate::object::try_existing_own_data_overwrite(obj, key_ptr, value) } { return value; } - // Charter step 3: a key this receiver shape inherits as an accessor - // runs its setter from the inherited-access table. - if crate::value::addr_class::is_above_handle_band(obj as usize) { - if let Some(interned) = unsafe { - crate::object::chain_store::interned_key_for_store(f64::from_bits(key_bits)) - } { - if unsafe { - crate::object::inherited_read_cache::inherited_write_through( - obj, interned, value, - ) - } { - return value; - } - } - } } let scope = crate::gc::RuntimeHandleScope::new(); @@ -655,11 +640,6 @@ pub extern "C" fn js_put_value_set_ic_miss( // The descriptor above already proves this stamp is live, so the // token comes from the header word rather than from a second full // lookup-and-copy of the same id (see `dyn_ic_try_store`). - // D3(b): a marked prototype's shape is never learned by a store cache, - // so every write to it reaches a funnel that bumps PERRY_PROTO_VALIDITY. - if !crate::object::proto_validity::store_cache_may_learn(obj) { - return result; - } let shape_token = crate::object::shapes::PIC_ID_TOKEN_BIT | crate::object::shapes::object_shape_stamp(obj) as u64; @@ -1286,11 +1266,6 @@ pub extern "C" fn js_put_value_set_dyn_ic_miss( // The descriptor above already proves this stamp is live, so the // token comes from the header word rather than from a second full // lookup-and-copy of the same id (see `dyn_ic_try_store`). - // D3(b): a marked prototype's shape is never learned by a store cache, - // so every write to it reaches a funnel that bumps PERRY_PROTO_VALIDITY. - if !crate::object::proto_validity::store_cache_may_learn(obj) { - return result; - } let shape_token = crate::object::shapes::PIC_ID_TOKEN_BIT | crate::object::shapes::object_shape_stamp(obj) as u64; let key_bits = key.to_bits() as i64; diff --git a/crates/perry-runtime/src/proxy/put_value/packed_set.rs b/crates/perry-runtime/src/proxy/put_value/packed_set.rs index 3682cf4c8d..58071df2c3 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_set.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_set.rs @@ -177,24 +177,6 @@ pub extern "C" fn js_put_value_set_packed_miss( } } - // Charter step 3: a key this receiver shape inherits as an accessor runs - // its setter from the inherited-access table (the same entries reads use), - // ahead of the key interning, chain proof and rooting below, which it - // would pay for nothing. - { - let tb = target.to_bits(); - if tb & !crate::value::POINTER_MASK == crate::value::POINTER_TAG && !key.is_null() { - let obj = (tb & crate::value::POINTER_MASK) as *const crate::ObjectHeader; - if crate::value::addr_class::is_above_handle_band(obj as usize) - && unsafe { - crate::object::inherited_read_cache::inherited_write_through(obj, key, value) - } - { - return value; - } - } - } - // Inherited-access lane: a key-adding store whose chain this site has // already proved clear takes the transition append (`object::chain_store`). // Allocation-free on a decline. @@ -384,7 +366,6 @@ unsafe fn prime_packed_set( || !crate::object::shapes::store_kind::shape_admits_plain_store( crate::object::shapes::object_shape_stamp(obj), ) - || !crate::object::proto_validity::store_cache_may_learn(obj) { return; } diff --git a/crates/perry-runtime/src/typed_feedback/guards.rs b/crates/perry-runtime/src/typed_feedback/guards.rs index 4c43d607f0..510c324eb3 100644 --- a/crates/perry-runtime/src/typed_feedback/guards.rs +++ b/crates/perry-runtime/src/typed_feedback/guards.rs @@ -845,7 +845,7 @@ fn class_field_get_one_path( } let key = (key as u64 & crate::value::POINTER_MASK) as *const crate::StringHeader; if probe_mru { - if let Some(value) = unsafe { class_field_get_from_shape(bits, key, cache_slot, false) } { + if let Some(value) = unsafe { class_field_get_from_shape(bits, cache_slot) } { return value; } } @@ -861,24 +861,14 @@ fn class_field_get_one_path( /// The answers the receiver's shape gives without the ladder, in the order /// the emitted generic read asks them: the site's own word, the site's holder -/// entry, then (on its declined edge) the inherited-read cache. `None` for -/// everything else. -/// -/// `leaf`: the caller is the S2 GC-leaf entry, so an inherited ACCESSOR entry -/// (which runs a getter) is declined, as `js_inherited_read_cache_hit_f64` -/// declines it for the emitted read. Otherwise the cache is asked once, as -/// the ladder's own first question (`get_field_ic_miss_impl`'s hook A) asks -/// it, getter included. +/// entry. `None` for everything else. /// /// # Safety -/// `cache_slot` is null or the site's live read cache; `key` is the interned -/// key with its tag masked off. +/// `cache_slot` is null or the site's live read cache. #[inline(always)] unsafe fn class_field_get_from_shape( bits: u64, - key: *const crate::StringHeader, cache_slot: *mut crate::object::PicCacheSlot, - leaf: bool, ) -> Option { // POINTER tag above the handle band: the receiver test every emitted // generic read makes before either lookup. @@ -900,22 +890,7 @@ unsafe fn class_field_get_from_shape( crate::hot_diag::recv_route_note_runtime(crate::hot_diag::RT_ROUTE_CLASS_MISS_SHAPE); return Some(value); } - let value = if leaf { - let value = - crate::object::inherited_read_cache::js_inherited_read_cache_hit_f64(handle, key); - (value.to_bits() != crate::value::TAG_HOLE).then_some(value) - } else { - match crate::object::inherited_read_cache::inherited_read_cache_lookup(handle, key) { - crate::object::inherited_read_cache::Lookup::Hit(value) => { - Some(f64::from_bits(value.bits())) - } - _ => None, - } - }; - if value.is_some() { - crate::hot_diag::recv_route_note_runtime(crate::hot_diag::RT_ROUTE_CLASS_MISS_SHAPE); - } - value + None } /// `js_class_field_get_ic`'s guard-FAIL arm with typed feedback on. @@ -1050,9 +1025,8 @@ pub extern "C" fn js_class_field_get_ic_fast( if !typed_feedback_enabled() && !crate::value::JSValue::from_bits(key as u64).is_short_string() { - let key = (key as u64 & crate::value::POINTER_MASK) as *const crate::StringHeader; if let Some(value) = - unsafe { class_field_get_from_shape(receiver.to_bits(), key, cache_slot, true) } + unsafe { class_field_get_from_shape(receiver.to_bits(), cache_slot) } { return value; } diff --git a/crates/perry-runtime/src/value/addr_class.rs b/crates/perry-runtime/src/value/addr_class.rs index f519aeba04..f727899553 100644 --- a/crates/perry-runtime/src/value/addr_class.rs +++ b/crates/perry-runtime/src/value/addr_class.rs @@ -285,15 +285,9 @@ pub(crate) unsafe fn try_read_gc_header_known_plausible(addr: usize) -> Option<& // "misaligned pointer dereference" abort that takes the whole test binary // down. `try_read_tracked_gc_header` has always checked this. // - // This guard lives HERE rather than in `try_read_gc_header` because that - // function delegates to this one, while three sites in - // `object/inherited_read_cache.rs` call this one DIRECTLY. Those callers - // satisfy this function's documented precondition — they proved - // `is_plausible_heap_addr` — but that predicate is - // `is_above_handle_band && is_valid_obj_ptr` and says nothing about - // alignment, so proving it does not transfer the guarantee. Guarding the - // delegate covers every path with one check instead of one check and one - // gap. Costs one AND on a path that then dereferences. + // Keep the guard in this delegate: its documented precondition only + // proves `is_plausible_heap_addr`, which says nothing about alignment. + // Direct callers remain safe, as do callers of `try_read_gc_header`. if !addr.is_multiple_of(std::mem::align_of::()) { return None; } @@ -646,11 +640,8 @@ mod known_plausible_alignment_tests { /// The alignment guard must be reachable from the DIRECT call path, not /// only through [`try_read_gc_header`]. /// - /// `try_read_gc_header` checks plausibility then delegates, so a guard - /// placed in *its* body covers its own callers and misses the three sites - /// in `object/inherited_read_cache.rs` that call the delegate directly. - /// Those sites satisfy the delegate's documented precondition — they proved - /// `is_plausible_heap_addr` — but that predicate is + /// `try_read_gc_header` checks plausibility then delegates. The delegate's + /// documented precondition is only `is_plausible_heap_addr`, which is /// `is_above_handle_band && is_valid_obj_ptr`, which says nothing about /// alignment, so satisfying it does not transfer the guarantee. /// diff --git a/crates/perry/tests/method_site.rs b/crates/perry/tests/method_site.rs index e5ffcdae84..2201c342d0 100644 --- a/crates/perry/tests/method_site.rs +++ b/crates/perry/tests/method_site.rs @@ -28,6 +28,19 @@ fn run(source: &str) -> (String, u64, u64, u64) { /// Compile and run `source`; return stdout and a reader of the site counters. fn run_counted(source: &str) -> (String, impl Fn(&str) -> u64) { + let (stdout, stderr) = compile_and_run(source, &[]); + let count = move |name: &str| stat(&stderr, name); + (stdout, count) +} + +fn stat(stderr: &str, name: &str) -> u64 { + stderr + .split_whitespace() + .find_map(|w| w.strip_prefix(name)?.strip_prefix('=')?.parse().ok()) + .unwrap_or(0) +} + +fn compile_and_run(source: &str, envs: &[(&str, &str)]) -> (String, String) { let dir = tempfile::tempdir().expect("tempdir"); let entry = dir.path().join("main.ts"); let output = dir.path().join("main_bin"); @@ -47,26 +60,23 @@ fn run_counted(source: &str) -> (String, impl Fn(&str) -> u64) { String::from_utf8_lossy(&compile.stdout), String::from_utf8_lossy(&compile.stderr) ); - let run = Command::new(&output) + let mut command = Command::new(&output); + command .current_dir(dir.path()) - .env("PERRY_METHOD_SITE_STATS", "1") - .output() - .expect("run compiled binary"); + .env("PERRY_METHOD_SITE_STATS", "1"); + for &(key, value) in envs { + command.env(key, value); + } + let run = command.output().expect("run compiled binary"); let stderr = String::from_utf8_lossy(&run.stderr).into_owned(); assert!( run.status.success(), "binary failed ({:?})\nstderr:\n{stderr}", run.status ); - let count = move |name: &str| -> u64 { - stderr - .split_whitespace() - .find_map(|w| w.strip_prefix(name)?.strip_prefix('=')?.parse().ok()) - .unwrap_or(0) - }; ( String::from_utf8_lossy(&run.stdout).trim().to_owned(), - count, + stderr, ) } @@ -194,6 +204,7 @@ for (let i = 0; i < 6000; i++) { unrelated.y = i; sum += o.m(i); } + console.log(sum, unrelated.y, child.y); "#); assert_eq!(stdout, "18003000 5999 5999"); @@ -203,6 +214,91 @@ console.log(sum, unrelated.y, child.y); ); } +/// The inherited entry holds the direct prototype as a strong, rewriteable +/// root. The holder is young when the site primes; a forced copying minor +/// must actually relocate objects, then the same site must keep calling the +/// live method through the moved holder. Dropping the method-site root scan +/// makes this fail under poisoned from-space. +#[test] +fn an_inherited_method_holder_survives_a_moving_collection() { + let (stdout, stderr) = compile_and_run( + r#"function call(o: any, x: number): number { return o.m(x); } +const p: any = { m(x: number) { return x + 1; } }; +const o: any = Object.create(p); +let sum = 0; +for (let i = 0; i < 3000; i++) { + if (i === 1000) { + (globalThis as any).gc(); + const junk: any[] = []; + for (let j = 0; j < 20000; j++) junk.push({ j }); + } + if (i === 2000) p.m = function (x: number) { return x * 3; }; + sum += call(o, i); +} +console.log(sum, call(o, 7)); +"#, + &[ + ("PERRY_GC_FORCE_EVACUATE", "1"), + ("PERRY_GC_VERIFY_EVACUATION", "1"), + ("PERRY_GC_POISON_FROMSPACE", "1"), + ("PERRY_GC_DIAG", "1"), + ], + ); + assert_eq!(stdout, "9499500 21", "{stderr}"); + assert!( + stat(&stderr, "primes_inherited") > 0, + "site never primed: {stderr}" + ); + assert!(stat(&stderr, "misses") < 50, "site did not hit: {stderr}"); + let moved: u64 = stderr + .lines() + .filter_map(|line| line.strip_prefix("[gc-copy-minor] ran ")) + .filter(|line| !line.split_whitespace().any(|f| f == "in_place=true")) + .flat_map(|line| line.split_whitespace()) + .filter_map(|f| { + f.strip_prefix("copied_objects=") + .and_then(|v| v.parse::().ok()) + }) + .sum(); + assert!(moved > 0, "no copying minor relocated an object: {stderr}"); +} + +/// Starting a worker clears inherited entries whose holders live in the +/// primary heap. The worker must execute the same call site and get its own +/// prototype's method; subsequent primary calls must also remain correct. +#[test] +fn a_worker_never_uses_the_primary_heaps_inherited_holder() { + let (stdout, stderr) = compile_and_run( + r#"import { spawn } from "perry/thread"; +function call(o: any, x: number): number { return o.m(x); } +async function main(): Promise { + const p: any = { m(x: number) { return x + 1; } }; + const o: any = Object.create(p); + let before = 0; + for (let i = 0; i < 1000; i++) before += call(o, i); + const worker = await spawn(() => { + const wp: any = { m(x: number) { return x * 2; } }; + const wo: any = Object.create(wp); + let total = 0; + for (let i = 0; i < 1000; i++) total += call(wo, i); + return total; + }); + console.log(before, worker, call(o, 5)); +} +main(); +"#, + &[], + ); + assert_eq!(stdout, "500500 999000 6", "{stderr}"); + assert!( + stat(&stderr, "primes_inherited") > 0, + "primary site never primed: {stderr}" + ); + assert!( + stat(&stderr, "misses") >= 1000, + "worker did not take the inherited miss path: {stderr}" + ); +} /// What the prime must refuse (rest, `arguments`, bound) and what the hit must keep (arity padding, /// per-object captures, `this` after a throw, a GC-moved inherited closure). Sabotage: the own hit /// skips the code-pointer compare -> another object's method runs. diff --git a/scripts/gc_root_dominance_check.py b/scripts/gc_root_dominance_check.py index fc77baad36..662bfcbad9 100755 --- a/scripts/gc_root_dominance_check.py +++ b/scripts/gc_root_dominance_check.py @@ -466,10 +466,6 @@ def build_cfg(f): # `perry-codegen/src/gc_call_effects.rs` (`GcCallEffect::CannotCollect`). "js_write_barrier_root_nanbox", "perry_transition_cache_base", "js_transition_ic_note_hit", - # object/inherited_read_cache.rs `js_inherited_read_cache_hit_f64`: a - # per-thread table probe plus one load through the holder; no allocation, - # no user code, no chain walk (declines answer TAG_HOLE). - "js_inherited_read_cache_hit_f64", # S2 GC-leaf IC hits (`expr/ic_fast_split.rs`); audit in gc_call_effects.rs. "js_object_get_field_ic_fast", "js_class_field_get_ic_fast", diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index d5933d9ee1..7c54ffe072 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -360,7 +360,7 @@ "sources": { "crates/perry-runtime/src/gc/census.rs": "4a611bfe5615559642b0e6e1eaf0f25440c5e228db5302d67dba43e577e0a1b6", "crates/perry-runtime/src/gc/cycle.rs": "9b992d27700382c84758f6f4a5574115a9862234bed2e18e0c5a3cca25c25236", - "crates/perry-runtime/src/gc/mod.rs": "3658d3cad2e24948105648997b559a161e3efed06081251597f4c1def5772943", + "crates/perry-runtime/src/gc/mod.rs": "5314f692ce92c67a659e88709e9a8ff253ab375d388082821f961a6a3c188f03", "crates/perry-runtime/src/gc/policy.rs": "84a869e0aa09e932a4d2b6601186260129ba851b4482f29579047713c60fc95e", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } diff --git a/scripts/global_sink_asserted_baseline.txt b/scripts/global_sink_asserted_baseline.txt index 3e86811f44..e96bec1bbf 100644 --- a/scripts/global_sink_asserted_baseline.txt +++ b/scripts/global_sink_asserted_baseline.txt @@ -38,7 +38,6 @@ crates/perry-runtime/src/object/class_meta_registry.rs::PARENT_DENSE_INCOMPLETE crates/perry-runtime/src/object/delete_rest.rs::ON crates/perry-runtime/src/object/field_get_set/enumeration.rs::ON crates/perry-runtime/src/object/field_get_set/ic_miss.rs::ON -crates/perry-runtime/src/object/inherited_read_cache.rs::ON crates/perry-runtime/src/object/map_set_subclass.rs::ITERATOR_PROTOCOL_TOUCHED crates/perry-runtime/src/object/native_module_registry.rs::NM_DISPATCH_REGISTRY crates/perry-runtime/src/object/prop_plan.rs::ON diff --git a/scripts/thread_exit_address_globals.json b/scripts/thread_exit_address_globals.json index 926c0555d2..37db749e8f 100644 --- a/scripts/thread_exit_address_globals.json +++ b/scripts/thread_exit_address_globals.json @@ -4001,6 +4001,14 @@ "verdict": "per_thread", "why": "`#[thread_local]` static (not `thread_local!`, because generated code names the TLS symbol itself): every thread has its own AgentPtrs block, so no address outlives the thread that published it; const-initialised to null with no drop glue." }, + { + "file": "crates/perry-runtime/src/object/method_site.rs", + "names": [ + "METHOD_SITES" + ], + "verdict": "process_global_allocation", + "why": "The values are addresses of MethodSite records allocated by pic_slot_resolve_init from the process-lifetime PicArena (std::alloc chunks, never freed), not from any thread's GC arena. Each record's primary-heap holder is visited by scan_method_site_roots_mut and all inherited entries are emptied before the first worker executes (note_worker_agent), after which inherited primes are refused." + }, { "file": "crates/perry-runtime/src/object/method_site.rs", "names": [ diff --git a/test-files/test_parity_inherited_read_cache.ts b/test-files/test_parity_inherited_read_cache.ts index 99f9ecfc1f..da9e5cd016 100644 --- a/test-files/test_parity_inherited_read_cache.ts +++ b/test-files/test_parity_inherited_read_cache.ts @@ -76,9 +76,8 @@ function warm(read: () => unknown, n: number): unknown { // `Object.setPrototypeOf(o, null)` belongs here and is deliberately absent: // perry answers it from the prototype `o` was BORN with, because the generic // read falls back to the class registry when the recorded chain misses - // (#10827). That is a pre-existing divergence -- `PERRY_INHERITED_IC=0` - // prints the same wrong value -- and this cache declines the case, so - // asserting it here would make this file red for someone else's bug. Case 8 + // (#10827). That is a pre-existing divergence in the generic getter, so + // asserting it here would make this file red for another bug. Case 8 // covers a null prototype the receiver was born with, and 4c already proves // `setPrototypeOf` on the receiver invalidates an entry. } From 8ba860651f826c94a7d02571aac1b5f0aa9ff4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 10:50:54 +0200 Subject: [PATCH 03/40] fix: gate process-global read sites when workers start --- crates/perry-codegen/src/expr/method_site.rs | 13 ++- .../src/runtime_decls/objects.rs | 4 + .../perry-runtime/src/object/method_site.rs | 82 ++++++++++++------- .../src/object/method_site/read_holder.rs | 49 +++++------ crates/perry/tests/method_site.rs | 40 +++++++-- scripts/thread_exit_address_globals.json | 4 +- 6 files changed, 125 insertions(+), 67 deletions(-) diff --git a/crates/perry-codegen/src/expr/method_site.rs b/crates/perry-codegen/src/expr/method_site.rs index b4ffb77732..8ce6e4be49 100644 --- a/crates/perry-codegen/src/expr/method_site.rs +++ b/crates/perry-codegen/src/expr/method_site.rs @@ -30,7 +30,7 @@ //! exactly the behaviour it had. use super::FnCtx; -use crate::types::{DOUBLE, I1, I32, I64, PTR}; +use crate::types::{DOUBLE, I1, I32, I64, I8, PTR}; /// Is the One Path method site available for this call? /// `PERRY_METHOD_SITE=0` at compile time keeps the old dispatcher (A/B). @@ -147,7 +147,16 @@ pub(crate) fn emit_method_site( fused.biased }; ctx.current_block = object_idx; - ctx.block().cond_br(&ic.present, &deref_l, &miss_l); + // Site records are process-global, and inherited holders belong to the + // primary heap. A worker's first startup publishes this sticky gate + // before executing user code; afterward every agent takes the generic + // path. No worker reads a primary holder or races a primary site update. + let workers = ctx + .block() + .load_atomic_seq_cst(I8, "@PERRY_METHOD_SITE_WORKERS_PRESENT", 1); + let no_workers = ctx.block().icmp_eq(I8, &workers, "0"); + let site_enabled = ctx.block().and(I1, &no_workers, &ic.present); + ctx.block().cond_br(&site_enabled, &deref_l, &miss_l); // deref: the receiver word against each entry's word, in order. ctx.current_block = deref_idx; diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index ad320d6c80..2e917612de 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -71,6 +71,10 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // The key-add hit's chain-verdict generation and the store census // (expr/put_value_store_ic.rs, expr/store_census.rs). module.add_external_global("PERRY_PROTO_VALIDITY", I64); + // Sticky worker-start gate for process-global method sites. After it + // becomes nonzero, emitted sites use ordinary dispatch without touching + // primary-heap holder entries. + module.add_external_global("PERRY_METHOD_SITE_WORKERS_PRESENT", I8); module.add_external_global("PERRY_STORE_CENSUS", I64); // #10943: has ANY named property ever been installed on a non-ordinary // cell in this process? Zero is the own-override guard's own proof that a diff --git a/crates/perry-runtime/src/object/method_site.rs b/crates/perry-runtime/src/object/method_site.rs index db26e46e47..01ecb8bdb9 100644 --- a/crates/perry-runtime/src/object/method_site.rs +++ b/crates/perry-runtime/src/object/method_site.rs @@ -76,9 +76,12 @@ //! //! # Agents //! -//! Inherited entries prime only on the primary agent. The first worker start -//! clears them and prevents further inherited primes, so a worker cannot -//! follow a holder from the primary heap through a process-global site. +//! The first worker start atomically gates every emitted method site and +//! prevents further primes. It does not rewrite a live site while primary +//! code may be reading it. Existing inherited entries are no longer read or +//! traced by any agent after the gate; their stale words are inert and the +//! holders can be collected by the primary GC. The cost thereafter is +//! ordinary method dispatch at every site. use crate::object::ObjectHeader; @@ -179,37 +182,33 @@ const _: () = { /// The emitted `@perry_ic_N = private global ptr null` for a method site. pub type MethodSiteSlot = *mut MethodSite; -/// Every site that holds (or held) an inherited entry, for the root scan and -/// for clearing primary-heap holders when the first worker starts. +/// Every site that holds (or held) an inherited entry, for the primary agent's +/// root scan until a worker starts. The sites are process-lifetime +/// allocations, but their holders belong to the primary heap. static METHOD_SITES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); -/// Set when the first `perry/thread` worker agent starts. Site memos are -/// process-global and an inherited entry holds a primary-heap holder, so from -/// then on no inherited entry is primed and every existing one is emptied. -/// Own entries hold no heap reference (ShapeIds are -/// process-unique; the call passes the receiver's own closure). -static WORKER_AGENTS_EXIST: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); +/// Sticky process-wide gate: a worker cannot read a primary-heap holder from +/// a process-global method or read site. Emitted method sites read this byte +/// atomically and take the generic miss once it is set. Keeping the old words +/// intact avoids racing a worker startup write against a primary inline hit. +/// After the gate, no agent reads or traces the stale entries, so they do not +/// retain their primary-heap holders. +#[cfg_attr(not(test), export_name = "PERRY_METHOD_SITE_WORKERS_PRESENT")] +pub(crate) static WORKER_AGENTS_EXIST: std::sync::atomic::AtomicU8 = + std::sync::atomic::AtomicU8::new(0); /// Called by `agent::enter_worker_agent` before the worker runs any code. pub fn note_worker_agent() { - if !WORKER_AGENTS_EXIST.swap(true, Ordering::SeqCst) { + // Publish the gate under the same lock as `publish`: every in-flight + // write finishes before a worker can run emitted code, and all later + // publishes decline. No site word is written at worker startup. + let _sites = METHOD_SITES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let first = WORKER_AGENTS_EXIST.swap(1, Ordering::SeqCst) == 0; + drop(_sites); + if first { super::proto_validity::bump_proto_validity(); - if let Ok(sites) = METHOD_SITES.lock() { - for &site in sites.iter() { - // A worker must not read a primary-heap holder through a - // process-wide site. No inherited entry is published again. - unsafe { - for entry in (*(site as *mut MethodSite)).entries.iter_mut() { - if entry.slot & METHOD_SITE_INHERITED != 0 { - entry.word = METHOD_SITE_EMPTY; - entry.closure = 0; - } - } - } - } - } - read_holder::empty_read_holder_entries(); } } @@ -327,6 +326,17 @@ pub unsafe extern "C-unwind" fn js_method_site_miss( else { return f64::from_bits(crate::value::TAG_UNDEFINED); }; + if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { + refuse(11); + return crate::typed_feedback::js_typed_feedback_native_call_method( + site_id, + recv, + name_ref.ptr as *const i8, + name_ref.len, + args_ptr, + argc, + ); + } // Only an ordinary heap object can prime. Everything else (primitives, // handles, functions, arrays) dispatches with no extra work at all. let megamorphic = site_is_megamorphic(slot); @@ -448,7 +458,7 @@ unsafe fn publish(slot: *mut MethodSiteSlot, entry: MethodEntry) -> bool { let Ok(mut sites) = METHOD_SITES.lock() else { return false; }; - if entry.slot & METHOD_SITE_INHERITED != 0 && WORKER_AGENTS_EXIST.load(Ordering::SeqCst) { + if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return false; } let site = site_of(slot); @@ -510,6 +520,7 @@ unsafe fn prime(slot: *mut MethodSiteSlot, recv: f64, name: &[u8], argc: usize) if slot.is_null() || name_refused(name) || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT + || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return; } @@ -851,7 +862,7 @@ unsafe fn prime_inherited( name: &[u8], argc: usize, ) { - if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) { + if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { refuse(11); return; } @@ -977,8 +988,17 @@ unsafe fn next_prototype(obj: *const ObjectHeader) -> *const ObjectHeader { super::class_prototype_object(class_id) } -/// Root scan: every inherited entry's holder is marked and rewritten. +/// Root scan: before workers exist, every inherited entry's holder is marked +/// and rewritten. After the sticky gate, no emitted or runtime path reads an +/// entry; returning here lets otherwise-dead holders collect. A primary +/// inline hit begun before the gate cannot safepoint between its entry read +/// and method call, so no primary GC can observe an in-flight holder read. pub(crate) fn scan_method_site_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + if crate::agent::current_agent() != crate::agent::PRIMARY_AGENT + || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 + { + return; + } if let Ok(sites) = METHOD_SITES.lock() { for &site in sites.iter() { for e in unsafe { (*(site as *mut MethodSite)).entries.iter_mut() } { diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 3d998b28ea..a544c9212a 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -29,16 +29,18 @@ //! (`field_get_set::ic_miss::read_confirm::js_object_get_field_ic_front`) //! asks [`entry_answer`] after the ways, the spill entry and a latched site's //! confirm, so an own-key read pays nothing for it. A decline is `TAG_HOLE` -//! and the site continues to the collecting slow call. +//! and the site continues to the collecting slow call. Once a worker starts, +//! the front declines before reading any holder-entry word: the entry belongs +//! to the primary heap and no agent's GC scans it after the gate. //! //! # Priming //! -//! Only from the read miss handler, which already knows the key is not own, -//! and only after the generic getter has produced the answer: the entry is +//! Only from the primary agent's read miss handler, which already knows the key +//! is not own, and only after the generic getter has produced the answer: the entry is //! recorded only when what the shapes say equals what the getter returned //! (names the runtime synthesizes, lazily materialized intrinsics and -//! `constructor` refuse there). Primary agent only; a worker agent's start -//! empties every entry. +//! `constructor` refuse there). A worker agent's start gates all further +//! holder hits and primes; stale entries stop being roots and can collect. //! //! A miss whose receiver the live entry already answers is served from the //! entry and primes nothing (a caller that does not emit the check, such as @@ -94,9 +96,9 @@ const HOLDER_ABSENT_BIT: u64 = 1 << 62; const HOLDER_DEPTH_SHIFT: u32 = 32; const HOLDER_MAX_DEPTH: usize = 4; -/// Every cache that holds (or held) a holder entry, for the root scan and for -/// emptying the entries when the first worker agent starts. The entries are -/// in the per-site caches; this is only where the scan finds them. +/// Every cache that holds (or held) a holder entry, for the primary agent's +/// root scan until a worker starts. The entries are in the per-site caches; +/// this is only where the scan finds them. static HOLDER_SITES: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); per_test_global! { @@ -138,6 +140,9 @@ unsafe fn refuse_and_latch(cache: *mut PicCache) { /// whose ShapeId still matches has not had the slot cleared. #[inline(always)] pub(crate) unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { + if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { + return None; + } if c[HOLDER_RECV] != token || token == 0 { return None; } @@ -357,7 +362,7 @@ pub(crate) unsafe fn prime_read_holder( ) -> Option { if cache_slot.is_null() || key.is_null() - || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) + || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT { return None; @@ -404,6 +409,9 @@ pub(crate) unsafe fn prime_read_holder( let (value, obj) = handle.across_mut::(|| { crate::object::field_get_set::get_field_by_name_after_site_miss(obj, key) }); + if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { + return Some(value); + } // From here a refusal has already run the getter, so the site latches: // the next miss must not walk and run it again only to refuse again. let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); @@ -481,8 +489,15 @@ unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk) { super::stats_report_enabled(); } -/// Root scan: every live entry's holder and hops are marked and rewritten. +/// Root scan: live entries are primary roots only until a worker starts. Once +/// the sticky gate is set, `entry_answer` declines before touching any entry +/// word, and no agent needs to retain the old primary objects. pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + if crate::agent::current_agent() != crate::agent::PRIMARY_AGENT + || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 + { + return; + } let Ok(sites) = HOLDER_SITES.lock() else { return; }; @@ -499,17 +514,3 @@ pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVis } } } - -/// The first worker agent's start: a holder entry names a primary-heap object, -/// so every entry is emptied, and none is primed again (`WORKER_AGENTS_EXIST`). -pub(crate) fn empty_read_holder_entries() { - let Ok(sites) = HOLDER_SITES.lock() else { - return; - }; - for &site in sites.iter() { - // SAFETY: as in `scan_read_holder_roots_mut`; one aligned word store. - unsafe { - std::ptr::write_volatile(&mut (*(site as *mut PicCache))[HOLDER_RECV], 0); - } - } -} diff --git a/crates/perry/tests/method_site.rs b/crates/perry/tests/method_site.rs index 2201c342d0..fd0adb752e 100644 --- a/crates/perry/tests/method_site.rs +++ b/crates/perry/tests/method_site.rs @@ -263,9 +263,10 @@ console.log(sum, call(o, 7)); assert!(moved > 0, "no copying minor relocated an object: {stderr}"); } -/// Starting a worker clears inherited entries whose holders live in the -/// primary heap. The worker must execute the same call site and get its own -/// prototype's method; subsequent primary calls must also remain correct. +/// Starting a worker gates every process-global site without clearing a +/// primary holder while primary code may be reading it. Both agents execute +/// the same call site while the worker is live. Their copying collections +/// must scan only the roots in their own heaps. #[test] fn a_worker_never_uses_the_primary_heaps_inherited_holder() { let (stdout, stderr) = compile_and_run( @@ -276,20 +277,39 @@ async function main(): Promise { const o: any = Object.create(p); let before = 0; for (let i = 0; i < 1000; i++) before += call(o, i); - const worker = await spawn(() => { - const wp: any = { m(x: number) { return x * 2; } }; + const sab = new SharedArrayBuffer(8); + const gate = new Int32Array(sab); + const pending = spawn(() => { + const workerGate = new Int32Array(sab); + const wp: any = {}; + wp.m = (x: number) => x * 2; const wo: any = Object.create(wp); + (globalThis as any).gc(); + Atomics.store(workerGate, 0, 1); + Atomics.notify(workerGate, 0); + if (Atomics.wait(workerGate, 1, 0, 10000) === 'timed-out') throw new Error('primary did not overlap worker'); let total = 0; for (let i = 0; i < 1000; i++) total += call(wo, i); return total; }); - console.log(before, worker, call(o, 5)); + if (Atomics.wait(gate, 0, 0, 10000) === 'timed-out') throw new Error('worker did not start'); + let overlap = 0; + for (let i = 0; i < 1000; i++) overlap += call(o, i); + Atomics.store(gate, 1, 1); + Atomics.notify(gate, 1); + const worker = await pending; + (globalThis as any).gc(); + console.log(before, worker, overlap, call(o, 5)); } main(); "#, - &[], + &[ + ("PERRY_GC_FORCE_EVACUATE", "1"), + ("PERRY_GC_VERIFY_EVACUATION", "1"), + ("PERRY_GC_POISON_FROMSPACE", "1"), + ], ); - assert_eq!(stdout, "500500 999000 6", "{stderr}"); + assert_eq!(stdout, "500500 999000 500500 6", "{stderr}"); assert!( stat(&stderr, "primes_inherited") > 0, "primary site never primed: {stderr}" @@ -298,6 +318,10 @@ main(); stat(&stderr, "misses") >= 1000, "worker did not take the inherited miss path: {stderr}" ); + assert!( + stat(&stderr, "refused.inh_workers") > 0, + "site miss did not refuse admission after worker startup: {stderr}" + ); } /// What the prime must refuse (rest, `arguments`, bound) and what the hit must keep (arity padding, /// per-object captures, `this` after a throw, a GC-moved inherited closure). Sabotage: the own hit diff --git a/scripts/thread_exit_address_globals.json b/scripts/thread_exit_address_globals.json index 37db749e8f..7fe94cb5a4 100644 --- a/scripts/thread_exit_address_globals.json +++ b/scripts/thread_exit_address_globals.json @@ -4007,7 +4007,7 @@ "METHOD_SITES" ], "verdict": "process_global_allocation", - "why": "The values are addresses of MethodSite records allocated by pic_slot_resolve_init from the process-lifetime PicArena (std::alloc chunks, never freed), not from any thread's GC arena. Each record's primary-heap holder is visited by scan_method_site_roots_mut and all inherited entries are emptied before the first worker executes (note_worker_agent), after which inherited primes are refused." + "why": "The values are addresses of MethodSite records allocated by pic_slot_resolve_init from the process-lifetime PicArena (std::alloc chunks, never freed), not from any thread's GC arena. Before workers start, primary-heap holders are visited by the primary agent's scan_method_site_roots_mut. Worker startup atomically gates all site reads and later publishes without writing live entries; thereafter no agent reads or scans the stale holder words, allowing their objects to collect." }, { "file": "crates/perry-runtime/src/object/method_site.rs", @@ -4023,7 +4023,7 @@ "HOLDER_SITES" ], "verdict": "process_global_allocation", - "why": "The addresses of read-site `PicCache`s, each allocated by `field_get_set::ic_slot::pic_arena_alloc` from `std::alloc::alloc_zeroed` chunks that are never freed and belong to no thread's arena, so thread exit cannot free or reuse them. The primary-heap holder and hop addresses the caches hold are written only by the primary agent (`prime_read_holder` checks `current_agent() == PRIMARY_AGENT` and `WORKER_AGENTS_EXIST`), are strong roots visited by `scan_read_holder_roots_mut`, and are emptied by `empty_read_holder_entries` when the first worker agent starts." + "why": "The addresses of read-site `PicCache`s, each allocated by `field_get_set::ic_slot::pic_arena_alloc` from `std::alloc::alloc_zeroed` chunks that are never freed and belong to no thread's arena, so thread exit cannot free or reuse them. Primary-heap holder and hop addresses are written only by the primary agent and traced by its scan_read_holder_roots_mut until the atomic worker-start gate is set. Thereafter no agent reads or scans stale holder words, allowing their objects to collect." }, { "file": "crates/perry-stdlib/src/common/handle_lifecycle.rs", From cec534747525ed186e916b1dae5c2d4e709fa9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 11:02:56 +0200 Subject: [PATCH 04/40] test: refuse class prototype identities at read-holder sites --- .../src/object/method_site/read_holder.rs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index a544c9212a..4fb0b1a5fe 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -514,3 +514,37 @@ pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVis } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A class instance has a valid, stamped ShapeId, but its prototype is + /// resolved through the class vtable. The holder walk must refuse it even + /// when the shape and the object's current prototype id agree. + #[test] + fn class_prototype_identity_is_refused_by_read_holder() { + let _lock = crate::gc::global_side_table_test_lock(); + const CLASS_ID: u32 = 0x0C3C_79A2; + let packed = b"holder_class_key"; + let keys = crate::object::js_build_class_keys_array( + CLASS_ID, + 1, + packed.as_ptr(), + packed.len() as u32, + ); + let shape_id = crate::object::shapes::js_object_shape_id_for_class_keys( + keys as usize as u64, + 1, + CLASS_ID, + ); + let obj = crate::object::js_object_alloc_class_inline_keys_stamped( + CLASS_ID, 0, 1, keys, shape_id, + ); + let claimed = shape_proto_id(shape_id).expect("class shape must be stamped"); + assert_eq!(claimed, crate::object::shapes::class_proto_id(CLASS_ID)); + assert_eq!(unsafe { object_proto_id(obj) }, claimed); + assert!(claimed >= crate::object::shapes::PROTO_ID_CLASS); + assert_eq!(unsafe { admitted_proto_id(obj) }, None); + } +} From 19e8a24d43321775fdb3a565852dfc3cb382c889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 09:17:58 +0000 Subject: [PATCH 05/40] test: reconcile A2 root-holder inventory after P4 --- scripts/gc_runtime_root_holders.json | 89 +++++++++++++++++++++------- 1 file changed, 68 insertions(+), 21 deletions(-) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 7c54ffe072..002a3453b8 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -184,6 +184,13 @@ "verdict": "not_a_gc_pointer", "why": "Monotonic counter of live async-resource handles. Holds no address at all; the resource objects live in RESOURCES, which scan_async_hooks_roots_mut visits." }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "BOX_YOUNG_ROOTS", + "verdict": "covered_elsewhere", + "why": "#9976: the minor remembered set for box roots — a YoungLog of box addresses whose payload may matter to a minor. Every address in it is also in the box REGISTRY, which the module's own doc calls the authoritative full/major root set and which `scan_box_roots_mut` walks. The log is an accelerator over that set, not an independent holder: an address dropped from it is still reached through the registry.", + "scanner": "box::scan_box_roots_mut (crates/perry-runtime/src/box.rs), registered by reg_scanner! in crates/perry-runtime/src/gc/mod.rs" + }, { "file": "crates/perry-runtime/src/buffer/header.rs", "name": "BUFFER_ADDR_RANGE", @@ -343,7 +350,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes — in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 (charter step 5, P0): the field-representation census adds `mod census_field_repr;` to `gc/mod.rs` and, in `census.rs`, one Rust-owned accumulator fed from `visit_object` inside `take_census`, which runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED; it reads slots and the per-object layout maps (try_borrow), allocates nothing on the JS heap and alters no mark/sweep control flow. The window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 (class constructors as function objects, #11414): `gc/mod.rs` gains one `reg_scanner!` registration (`object::class_value::scan_class_value_roots_mut`, the per-agent class function-object table, visited and rewritten) — a root-scanner registration that alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 after #11659 added `verify::verify_array_hole_tails_at_collection()` to `gc/mod.rs` and `gc/policy.rs`. Every call sits at a collection or budgeted-cycle ENTRY, beside `roots::ensure_stack_maps_built()`, i.e. before mark begins and so before `census_pass1_if_armed` opens the window. The call is a read-only walk of array headers (debug builds, or release with PERRY_GC_VERIFY_ARRAY_HOLES): it neither allocates GC memory, relocates, runs JS, nor reads or writes PASS1_MARKED, and it panics rather than continuing on a violation. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Pin re-audited 2026-09-29 after PR #11646 added one call inside that window, in `cycle.rs` beside `census_take_if_armed_at_full_sweep_start`: `object::shapes::store_kind::audit_heap_at_full_sweep_start()`. It compiles to nothing unless the `shape-fact-audit` feature is on; when on it walks the arena read-only (`gc::for_each_live_object_at_sweep_start`, in `gc/verify.rs`) and reads each marked object header and its shape record. It allocates no GC object, moves nothing, runs no JS callback and holds no address past the walk, so PASS1_MARKED stays valid across it; a disagreement panics rather than continuing. Re-audited 2026-09-29 after the pinned-roots fix touched `gc/cycle.rs` and `gc/mod.rs`: the block-persistence live-block predicate now reads GC_FLAG_MARKED alone (pinned objects are marked as roots, so the set of live blocks is unchanged), and `gc/mod.rs` adds one `reg_scanner!` registration. Neither relocates, runs JS, or moves the mark-complete to sweep-entry window. Re-audited again 2026-09-29 (same fix, copying-minor follow-up): `BlockPersistCycleState`'s force-mark in `gc/cycle.rs` no longer skips an unmarked pinned header, so it marks and pushes it like any other object of a live block. That is marking inside the budgeted cycle's persistence step: nothing relocates, no JS runs, and the budgeted path still skips both census boundaries. The window is unchanged. Re-audited 2026-09-29 for this-as-a-parameter stage 3: `gc/mod.rs` registers `scan_dispatch_binding_roots_mut` in place of `scan_implicit_this_roots_mut` (the implicit-`this` cell is deleted; the same scanner body keeps `new.target`, the static-`this` override and the static private-owner stack) and rewords its comment; a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 for the JsFunctionInfo lane: `gc/census.rs`'s `side_tables()` loses its `closure_registry_census()` row (the closure-body registry is deleted); a census report row, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 for the read-site holder entry: `gc/mod.rs` adds one `reg_scanner!` registration, `read_holder::scan_read_holder_roots_mut`, which visits the holder and hop words of registered read-site caches during ROOT SCAN, before mark propagation completes. It rewrites root slots only through the visitor (as every registered scanner does), runs no JS, and nothing it does executes between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. The window is unchanged. Re-audited 2026-09-30 for Step 5 P4: census.rs only removes the typed-layout count, and gc/mod.rs removes shape-install registration/re-export and an init-time typed-intact verifier gate. The snapshot remains stored after mark propagation and taken at sweep entry; none of these edits relocates objects or invokes callbacks in that interval. Re-audited 2026-09-29 (#11549 trace cost): `gc/mod.rs` gains one `mod copying_object_scan;` declaration, the copying minor's plain-object slot walk. It is reached only from the copying drain, never from a full cycle, and changes no mark or sweep control flow. The window is unchanged. Re-audited 2026-09-30 after rebasing P4 over #11549/#11676: gc/mod.rs retains the copying_object_scan declaration and P4 removes typed-layout registration; both are module wiring outside the synchronous full mark-complete to sweep-entry window. The census boundaries and the window owner in cycle.rs are unchanged. Re-audited 2026-09-30 for the scope-context change: gc/mod.rs only removes the box root-scanner registration and exit-time box statistics. The former ran during root scan, before mark completion, and the latter runs at process exit. Neither changes the synchronous mark-complete to sweep-entry window.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete \u2192 sweep-entry window of a synchronous full \u2014 where PASS1_MARKED is populated and consumed within one `run_to_completion` \u2014 is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize \u2014 INSIDE the window \u2014 the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes \u2014 in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 (charter step 5, P0): the field-representation census adds `mod census_field_repr;` to `gc/mod.rs` and, in `census.rs`, one Rust-owned accumulator fed from `visit_object` inside `take_census`, which runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED; it reads slots and the per-object layout maps (try_borrow), allocates nothing on the JS heap and alters no mark/sweep control flow. The window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 (class constructors as function objects, #11414): `gc/mod.rs` gains one `reg_scanner!` registration (`object::class_value::scan_class_value_roots_mut`, the per-agent class function-object table, visited and rewritten) \u2014 a root-scanner registration that alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 after #11659 added `verify::verify_array_hole_tails_at_collection()` to `gc/mod.rs` and `gc/policy.rs`. Every call sits at a collection or budgeted-cycle ENTRY, beside `roots::ensure_stack_maps_built()`, i.e. before mark begins and so before `census_pass1_if_armed` opens the window. The call is a read-only walk of array headers (debug builds, or release with PERRY_GC_VERIFY_ARRAY_HOLES): it neither allocates GC memory, relocates, runs JS, nor reads or writes PASS1_MARKED, and it panics rather than continuing on a violation. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Pin re-audited 2026-09-29 after PR #11646 added one call inside that window, in `cycle.rs` beside `census_take_if_armed_at_full_sweep_start`: `object::shapes::store_kind::audit_heap_at_full_sweep_start()`. It compiles to nothing unless the `shape-fact-audit` feature is on; when on it walks the arena read-only (`gc::for_each_live_object_at_sweep_start`, in `gc/verify.rs`) and reads each marked object header and its shape record. It allocates no GC object, moves nothing, runs no JS callback and holds no address past the walk, so PASS1_MARKED stays valid across it; a disagreement panics rather than continuing. Re-audited 2026-09-29 after the pinned-roots fix touched `gc/cycle.rs` and `gc/mod.rs`: the block-persistence live-block predicate now reads GC_FLAG_MARKED alone (pinned objects are marked as roots, so the set of live blocks is unchanged), and `gc/mod.rs` adds one `reg_scanner!` registration. Neither relocates, runs JS, or moves the mark-complete to sweep-entry window. Re-audited again 2026-09-29 (same fix, copying-minor follow-up): `BlockPersistCycleState`'s force-mark in `gc/cycle.rs` no longer skips an unmarked pinned header, so it marks and pushes it like any other object of a live block. That is marking inside the budgeted cycle's persistence step: nothing relocates, no JS runs, and the budgeted path still skips both census boundaries. The window is unchanged. Re-audited 2026-09-29 for this-as-a-parameter stage 3: `gc/mod.rs` registers `scan_dispatch_binding_roots_mut` in place of `scan_implicit_this_roots_mut` (the implicit-`this` cell is deleted; the same scanner body keeps `new.target`, the static-`this` override and the static private-owner stack) and rewords its comment; a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 for the JsFunctionInfo lane: `gc/census.rs`'s `side_tables()` loses its `closure_registry_census()` row (the closure-body registry is deleted); a census report row, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 for the read-site holder entry: `gc/mod.rs` adds one `reg_scanner!` registration, `read_holder::scan_read_holder_roots_mut`, which visits the holder and hop words of registered read-site caches during ROOT SCAN, before mark propagation completes. It rewrites root slots only through the visitor (as every registered scanner does), runs no JS, and nothing it does executes between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. The window is unchanged. Re-audited 2026-09-30 for Step 5 P4: census.rs only removes the typed-layout count, and gc/mod.rs removes shape-install registration/re-export and an init-time typed-intact verifier gate. The snapshot remains stored after mark propagation and taken at sweep entry; none of these edits relocates objects or invokes callbacks in that interval. Re-audited 2026-09-29 (#11549 trace cost): `gc/mod.rs` gains one `mod copying_object_scan;` declaration, the copying minor's plain-object slot walk. It is reached only from the copying drain, never from a full cycle, and changes no mark or sweep control flow. The window is unchanged. Re-audited 2026-09-30 after rebasing P4 over #11549/#11676: gc/mod.rs retains the copying_object_scan declaration and P4 removes typed-layout registration; both are module wiring outside the synchronous full mark-complete to sweep-entry window. The census boundaries and the window owner in cycle.rs are unchanged. Re-audited for A2 inherited-read-cache deletion: gc/mod.rs removes only its old root-scanner registration and adjacent comments. The method-site and read-holder scanners still run during root scan before mark completion; no collection phase or callback was added inside the mark-complete to sweep-entry window. Re-audited 2026-09-30 for the scope-context change: gc/mod.rs also removes the box root-scanner registration and exit-time box statistics. The former ran during root scan before mark completion, and the latter at process exit. Neither changes the synchronous mark-complete to sweep-entry window.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -910,18 +917,6 @@ "verdict": "not_a_gc_pointer", "why": "A bool: set while this thread builds its %Object% / %Object.prototype% pair, so the build's own stores do not re-enter the build. No address is ever stored in it." }, - { - "file": "crates/perry-runtime/src/object/inherited_read_cache.rs", - "name": "BY_NAME_SEEN", - "verdict": "not_a_gc_pointer", - "why": "#10495/#10753: per-thread direct-mapped filter of u32 words, each a FINGERPRINT of a (receiver ShapeId, key address) pair plus a 2-bit sighting count, so hook D primes the inherited-read cache on a pair's second or third by-name miss only and then stops trying. A fingerprint is a hash, never dereferenced or compared as an address; a moved or recycled key can only cost one extra or one skipped prime." - }, - { - "file": "crates/perry-runtime/src/object/inherited_read_cache.rs", - "name": "NOT_ANON_MEMO", - "verdict": "not_a_gc_pointer", - "why": "#10495: per-thread direct-mapped memo of u32 CLASS IDS proved not to be an anonymous literal shape's (is_anon_shape_class_id == false). Class ids are integers from the class registry, never heap addresses." - }, { "file": "crates/perry-runtime/src/object/mod.rs", "name": "OBJECT_INTRINSIC_PROTO_PTR_SLOT", @@ -2874,6 +2869,58 @@ "file": "crates/perry-runtime/src/async_hooks.rs", "name": "REGISTERED" }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "ASYNC_BOX_ACTIVATION_FREE_HEAD" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "ASYNC_PENDING_RELEASES" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "ASYNC_RELEASED_CELLS" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "BOOL_BOX_FREE_HEAD" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "BOOL_BOX_REGISTRY" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "BOOL_BOX_RELEASE_QUARANTINE" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "BOX_FREE_HEAD" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "BOX_REGISTRY" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "BOX_RELEASE_QUARANTINE" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "I32_BOX_FREE_HEAD" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "I32_BOX_REGISTRY" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "I32_BOX_RELEASE_QUARANTINE" + }, + { + "file": "crates/perry-runtime/src/box.rs", + "name": "NEXT_ASYNC_BOX_ACTIVATION_ID" + }, { "file": "crates/perry-runtime/src/box.rs", "name": "TDZ_SUPPRESS_DEPTH" @@ -2930,6 +2977,14 @@ "file": "crates/perry-runtime/src/child_process/v8_serde.rs", "name": "SERIALIZERS" }, + { + "file": "crates/perry-runtime/src/closure/box_captures.rs", + "name": "BOX_CAPTURE_COUNTS" + }, + { + "file": "crates/perry-runtime/src/closure/box_captures.rs", + "name": "CLOSURE_BOX_CELLS" + }, { "file": "crates/perry-runtime/src/closure/dispatch/errors.rs", "name": "THROW_NOT_CALLABLE_COUNT" @@ -4264,14 +4319,6 @@ { "file": "crates/perry-stdlib/src/zlib.rs", "name": "ZLIB_GC_REGISTERED" - }, - { - "file": "crates/perry-runtime/src/box/activation.rs", - "name": "ASYNC_BOX_ACTIVATION_FREE_HEAD" - }, - { - "file": "crates/perry-runtime/src/box/activation.rs", - "name": "NEXT_ASYNC_BOX_ACTIVATION_ID" } ] } From 7f4bd978293b438a9af16c1c28c25330cf80c8de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 09:27:45 +0000 Subject: [PATCH 06/40] docs: note inherited-read single-path change --- changelog.d/inherited-read-one-shape.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/inherited-read-one-shape.md diff --git a/changelog.d/inherited-read-one-shape.md b/changelog.d/inherited-read-one-shape.md new file mode 100644 index 0000000000..a53b9119e0 --- /dev/null +++ b/changelog.d/inherited-read-one-shape.md @@ -0,0 +1 @@ +**Objects:** inherited reads and method calls use receiver and holder shape facts; the old inherited-read cache and its GC roots are removed. Method sites take ordinary dispatch after worker startup. From 44f20d67d4798b87c35399b05039215e9bc49aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 09:47:33 +0000 Subject: [PATCH 07/40] test: require relocated inherited method holder root --- crates/perry-runtime/src/object/method_site.rs | 10 +++++++--- crates/perry/tests/method_site.rs | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site.rs b/crates/perry-runtime/src/object/method_site.rs index 01ecb8bdb9..f416617fb9 100644 --- a/crates/perry-runtime/src/object/method_site.rs +++ b/crates/perry-runtime/src/object/method_site.rs @@ -245,6 +245,7 @@ fn refuse(reason: usize) { per_test_global! { static PRIMES_OWN: AtomicU64 = AtomicU64::new(0); static PRIMES_INHERITED: AtomicU64 = AtomicU64::new(0); + static HOLDER_REWRITES: AtomicU64 = AtomicU64::new(0); static MISSES: AtomicU64 = AtomicU64::new(0); static PRIMES_FUNCTION: AtomicU64 = AtomicU64::new(0); } @@ -295,8 +296,9 @@ fn stats_report_enabled() -> bool { } let (hd, ha, hr) = read_holder::read_holder_stats(); eprintln!( - "[method-site] primes_own={a} primes_inherited={b} primes_function={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_holder_refused={hr}{refused}", - method_site_function_primes() + "[method-site] primes_own={a} primes_inherited={b} primes_function={} holder_rewrites={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_holder_refused={hr}{refused}", + method_site_function_primes(), + HOLDER_REWRITES.load(Ordering::Relaxed) ); } unsafe { libc::atexit(report) }; @@ -1003,7 +1005,9 @@ pub(crate) fn scan_method_site_roots_mut(visitor: &mut crate::gc::RuntimeRootVis for &site in sites.iter() { for e in unsafe { (*(site as *mut MethodSite)).entries.iter_mut() } { if e.closure != 0 { - visitor.visit_tagged_usize_slot(&mut e.closure, crate::value::POINTER_TAG); + if visitor.visit_tagged_usize_slot(&mut e.closure, crate::value::POINTER_TAG) { + HOLDER_REWRITES.fetch_add(1, Ordering::Relaxed); + } } } } diff --git a/crates/perry/tests/method_site.rs b/crates/perry/tests/method_site.rs index fd0adb752e..11c9c27a60 100644 --- a/crates/perry/tests/method_site.rs +++ b/crates/perry/tests/method_site.rs @@ -250,6 +250,10 @@ console.log(sum, call(o, 7)); "site never primed: {stderr}" ); assert!(stat(&stderr, "misses") < 50, "site did not hit: {stderr}"); + assert!( + stat(&stderr, "holder_rewrites") > 0, + "the method-site holder was not relocated: {stderr}" + ); let moved: u64 = stderr .lines() .filter_map(|line| line.strip_prefix("[gc-copy-minor] ran ")) From 79cfd603970ece994056384b4e8f3a90df78aee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 10:42:58 +0000 Subject: [PATCH 08/40] fix: acquire method-site slot publication before worker gate --- crates/perry-codegen/src/expr/method_site.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/perry-codegen/src/expr/method_site.rs b/crates/perry-codegen/src/expr/method_site.rs index 8ce6e4be49..35b369b254 100644 --- a/crates/perry-codegen/src/expr/method_site.rs +++ b/crates/perry-codegen/src/expr/method_site.rs @@ -134,7 +134,18 @@ pub(crate) fn emit_method_site( // takes the universal dispatcher directly, with its string and primitive // arms, exactly as without a site. A heap object takes the site: its memo // if the site has one, else the miss, which primes it. - let ic = crate::expr::emit_inline_cache_slot(ctx, &cache_name); + // The runtime publishes this process-global slot with an AtomicPtr CAS. + // A worker may enter the site just as the primary agent first publishes + // it, before the sticky worker gate below is loaded. Pair the load with + // that publication even though the worker will then take the miss path. + let slot_ref = format!("@{cache_name}"); + let cache = ctx.block().load_atomic_acquire(PTR, &slot_ref, 8); + let present = ctx.block().icmp_ne(PTR, &cache, "null"); + let ic = crate::expr::InlineCacheSlot { + slot_ref, + cache, + present, + }; let prim_idx = ctx.new_block("msite.primitive"); let object_idx = ctx.new_block("msite.object"); let prim_l = ctx.block_label(prim_idx); From b411d7382216368c3986d48e86f6e5897f405298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 10:51:01 +0000 Subject: [PATCH 09/40] test: adapt class-prototype admission fixture to birth rep --- crates/perry-runtime/src/object/method_site/read_holder.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 4fb0b1a5fe..c69b187f68 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -532,14 +532,16 @@ mod tests { 1, packed.as_ptr(), packed.len() as u32, + 0, ); let shape_id = crate::object::shapes::js_object_shape_id_for_class_keys( keys as usize as u64, 1, CLASS_ID, + 0, ); let obj = crate::object::js_object_alloc_class_inline_keys_stamped( - CLASS_ID, 0, 1, keys, shape_id, + CLASS_ID, 0, 1, keys, shape_id, 0, ); let claimed = shape_proto_id(shape_id).expect("class shape must be stamped"); assert_eq!(claimed, crate::object::shapes::class_proto_id(CLASS_ID)); From 5921301fdef063ade357c509cbe1615adf4b31cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 12:13:27 +0000 Subject: [PATCH 10/40] Cache direct class prototype getters at read sites --- .../perry-runtime/src/object/field_get_set.rs | 13 +- .../src/object/field_get_set/accessors.rs | 4 + .../src/object/field_get_set/ic_miss.rs | 9 + .../perry-runtime/src/object/method_site.rs | 6 +- .../src/object/method_site/read_holder.rs | 204 ++++++++++++++++-- crates/perry-runtime/src/object/shapes.rs | 4 +- crates/perry/tests/read_holder_accessor.rs | 122 +++++++++++ 7 files changed, 339 insertions(+), 23 deletions(-) create mode 100644 crates/perry/tests/read_holder_accessor.rs diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index 76a14153cd..b8e2134e85 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -261,12 +261,13 @@ impl FieldLookupCaches { // reach the cross-module helpers via their own `use super::*;`. pub use accessors::js_object_get_field; pub(crate) use accessors::{ - accessor_receiver_override_begin, accessor_receiver_override_end, - accessor_receiver_override_take, array_prototype_property_value, - builtin_reflection_accessor_read, invoke_accessor_getter, invoke_accessor_setter, - is_typed_array_prototype, object_field_at_with_live, ordinary_object_prototype_property_value, - own_data_field_by_name, primitive_builtin_prototype_property, - primitive_object_prototype_accessor, primitive_tagged_prototype_property, string_index_value, + accessor_receiver_override_armed, accessor_receiver_override_begin, + accessor_receiver_override_end, accessor_receiver_override_take, + array_prototype_property_value, builtin_reflection_accessor_read, invoke_accessor_getter, + invoke_accessor_setter, is_typed_array_prototype, object_field_at_with_live, + ordinary_object_prototype_property_value, own_data_field_by_name, + primitive_builtin_prototype_property, primitive_object_prototype_accessor, + primitive_tagged_prototype_property, string_index_value, }; pub(crate) use class_object_props::{ class_evaluation_prototype_class_id, class_object_materialized_prototype, diff --git a/crates/perry-runtime/src/object/field_get_set/accessors.rs b/crates/perry-runtime/src/object/field_get_set/accessors.rs index 6e364dc32c..7dd0d526d6 100644 --- a/crates/perry-runtime/src/object/field_get_set/accessors.rs +++ b/crates/perry-runtime/src/object/field_get_set/accessors.rs @@ -446,6 +446,10 @@ pub(crate) fn accessor_receiver_override_take() -> Option { ACCESSOR_RECEIVER_OVERRIDE.with(|c| c.take()) } +pub(crate) fn accessor_receiver_override_armed() -> bool { + ACCESSOR_RECEIVER_OVERRIDE.with(|c| c.get().is_some()) +} + pub(crate) fn accessor_receiver_override_end(prev: Option) { ACCESSOR_RECEIVER_OVERRIDE.with(|c| c.set(prev)); } diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index 8893588a93..7d19773c20 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -803,6 +803,15 @@ pub(super) fn get_field_ic_miss_impl( // let gc_header = unsafe { crate::value::addr_class::try_read_gc_header(obj as usize) }; let gc_kind = gc_header.map(|h| h.obj_type); + // An accessor can run JS and collect, so this lives in the collecting + // miss handler. The leaf front only recognizes data and absent entries. + if gc_kind == Some(crate::gc::GC_TYPE_OBJECT) { + if let Some(value) = unsafe { + crate::object::method_site::read_holder::try_cached_class_accessor(obj, cache_slot) + } { + return f64::from_bits(value.bits()); + } + } if crate::value::addr_class::is_above_handle_band(obj as usize) { // # The receiver-classification ladder runs only for NON-object kinds // diff --git a/crates/perry-runtime/src/object/method_site.rs b/crates/perry-runtime/src/object/method_site.rs index f416617fb9..870387123b 100644 --- a/crates/perry-runtime/src/object/method_site.rs +++ b/crates/perry-runtime/src/object/method_site.rs @@ -295,10 +295,12 @@ fn stats_report_enabled() -> bool { } } let (hd, ha, hr) = read_holder::read_holder_stats(); + let (ap, ah) = read_holder::read_accessor_stats(); eprintln!( - "[method-site] primes_own={a} primes_inherited={b} primes_function={} holder_rewrites={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_holder_refused={hr}{refused}", + "[method-site] primes_own={a} primes_inherited={b} primes_function={} holder_rewrites={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_accessor_primes={ap} read_accessor_hits={ah} read_holder_rewrites={} read_holder_refused={hr}{refused}", method_site_function_primes(), - HOLDER_REWRITES.load(Ordering::Relaxed) + HOLDER_REWRITES.load(Ordering::Relaxed), + read_holder::read_holder_rewrites() ); } unsafe { libc::atexit(report) }; diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index c69b187f68..fffdc65c0e 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -53,7 +53,7 @@ use super::{key_may_be_accessor, next_prototype, ordinary_receiver, WORKER_AGENTS_EXIST}; use crate::object::shapes::{ object_proto_id, object_shape_descriptor, object_shape_stamp, shape_proto_id, PIC_ID_TOKEN_BIT, - PROTO_ID_DEFAULT, PROTO_ID_NULL, + PROTO_ID_DEFAULT, PROTO_ID_MIXED, PROTO_ID_NULL, PROTO_ID_UNIQUE, }; use crate::object::{ObjectHeader, PicCache, PicCacheSlot}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -93,6 +93,9 @@ const MAX_REPRIMES: i64 = 4; pub const HOLDER_ABSENT_DEPTH1: i64 = crate::codegen_abi::PIC_HOLDER_ABSENT_DEPTH1; pub const HOLDER_STUB: u64 = 1 << 63; const HOLDER_ABSENT_BIT: u64 = 1 << 62; +/// A direct class-prototype accessor. It can collect and therefore never +/// answers from the GC-leaf front call. +const HOLDER_ACCESSOR: u64 = 1 << 61; const HOLDER_DEPTH_SHIFT: u32 = 32; const HOLDER_MAX_DEPTH: usize = 4; @@ -105,6 +108,9 @@ per_test_global! { static PRIMES_HOLDER: AtomicU64 = AtomicU64::new(0); static PRIMES_ABSENT: AtomicU64 = AtomicU64::new(0); static REFUSED_HOLDER: AtomicU64 = AtomicU64::new(0); + static PRIMES_ACCESSOR: AtomicU64 = AtomicU64::new(0); + static HITS_ACCESSOR: AtomicU64 = AtomicU64::new(0); + static HOLDER_REWRITES: AtomicU64 = AtomicU64::new(0); } /// `(data primes, absent primes, refusals)`. @@ -116,6 +122,17 @@ pub fn read_holder_stats() -> (u64, u64, u64) { ) } +pub fn read_accessor_stats() -> (u64, u64) { + ( + PRIMES_ACCESSOR.load(Ordering::Relaxed), + HITS_ACCESSOR.load(Ordering::Relaxed), + ) +} + +pub fn read_holder_rewrites() -> u64 { + HOLDER_REWRITES.load(Ordering::Relaxed) +} + #[inline] fn refuse() { REFUSED_HOLDER.fetch_add(1, Ordering::Relaxed); @@ -147,6 +164,9 @@ pub(crate) unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { return None; } let kind = c[HOLDER_KIND]; + if kind as u64 & HOLDER_ACCESSOR != 0 { + return None; + } let (depth, absent, slot) = if kind >= 0 { (1, kind == HOLDER_ABSENT_DEPTH1, kind as u32) } else { @@ -272,6 +292,142 @@ pub(super) unsafe fn admitted_proto_id(obj: *const ObjectHeader) -> Option (object_proto_id(obj) == pid).then_some(pid) } +/// The class-instance form records both a class and a serial prototype link. +/// A bare CLASS identity cannot name one holder: the class registry may still +/// resolve its prototype lazily, and is deliberately refused. +unsafe fn class_link(recv: *const ObjectHeader) -> Option<*const ObjectHeader> { + let pid = shape_proto_id(object_shape_stamp(recv))?; + if !(PROTO_ID_MIXED..PROTO_ID_UNIQUE).contains(&pid) || object_proto_id(recv) != pid { + return None; + } + let holder = next_prototype(recv); + (!holder.is_null() && holder != recv).then_some(holder) +} + +struct ClassAccessor { + holder: usize, + shape: u32, + slot: u32, + raw_get: usize, +} + +/// Only the direct prototype's compiled class accessor is admitted. Other +/// accessor forms keep the generic path and its receiver-override semantics. +unsafe fn class_accessor_walk(recv: *const ObjectHeader, name: &[u8]) -> Option { + if !holder_name_admitted(name) + || crate::object::field_get_set::accessor_receiver_override_armed() + || crate::object::prototype_chain::resolution_stack_savepoint() != 0 + { + return None; + } + let holder = class_link(recv)? as usize; + if !crate::value::addr_class::is_above_handle_band(holder) + || !super::address_is_prime_stable(holder) + { + return None; + } + let header = crate::value::addr_class::try_read_gc_header(holder)?; + if header.obj_type != crate::gc::GC_TYPE_OBJECT + || header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || crate::object::dictionary::is_dictionary(holder as *const ObjectHeader) + { + return None; + } + let shape = object_shape_descriptor(holder as *const ObjectHeader)?; + if !shape.object_kind.is_ordinary_layout() { + return None; + } + let keys = shape.keys as usize as *const crate::array::ArrayHeader; + if keys.is_null() { + return None; + } + let slot = + crate::object::keys_find_slot_by_bytes_resolved(keys, shape.logical_key_count, name)?; + if slot >= shape.live_inline_slot_count + || crate::object::key_attrs::keys_entry(keys, slot) + & crate::object::key_attrs::ENTRY_ACCESSOR + == 0 + { + return None; + } + let acc = crate::object::accessor_pair::pair_of_value(slot_bits(holder, slot))?; + if acc.raw_get == 0 && acc.raw_set == 0 { + return None; + } + Some(ClassAccessor { + holder, + shape: object_shape_stamp(holder as *const ObjectHeader), + slot, + raw_get: acc.raw_get, + }) +} + +unsafe fn invoke_class_getter(recv: *const ObjectHeader, raw_get: usize) -> crate::value::JSValue { + if raw_get == 0 { + return crate::value::JSValue::from_bits(crate::value::TAG_UNDEFINED); + } + let scope = crate::gc::RuntimeHandleScope::new(); + let receiver = scope.root_raw_mut_ptr(recv as *mut ObjectHeader); + let f = crate::closure::body_call::js_method_body_fn!(raw_get as *const u8;); + let bits = receiver.with_mut_ptr::(|ptr| { + let this = crate::value::js_nanbox_pointer(ptr as i64); + f(f64::from_bits(this.to_bits())).to_bits() + }); + crate::value::JSValue::from_bits(bits) +} + +/// Collecting read-miss arm. The GC-leaf front always declines this kind. +/// Every hit confirms the receiver's shape and live link, the rooted holder's +/// shape, and the accessor descriptor before invoking with the ORIGINAL receiver. +pub(crate) unsafe fn try_cached_class_accessor( + recv: *const ObjectHeader, + cache_slot: *mut PicCacheSlot, +) -> Option { + if cache_slot.is_null() + || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 + || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT + || crate::object::field_get_set::accessor_receiver_override_armed() + || crate::object::prototype_chain::resolution_stack_savepoint() != 0 + { + return None; + } + let cache = crate::object::field_get_set::pic_slot_peek::(cache_slot); + if cache.is_null() { + return None; + } + let c = &*cache; + let stamp = object_shape_stamp(recv); + if stamp == 0 + || c[HOLDER_RECV] != (u64::from(stamp) | PIC_ID_TOKEN_BIT) as i64 + || c[HOLDER_KIND] as u64 & HOLDER_ACCESSOR == 0 + || class_link(recv)? as usize != c[HOLDER_OBJ] as usize + { + return None; + } + let holder = c[HOLDER_OBJ] as usize; + if shape_word(holder) != c[HOLDER_SHAPE] as u32 { + return None; + } + let slot = c[HOLDER_KIND] as u32; + let shape = object_shape_descriptor(holder as *const ObjectHeader)?; + let keys = shape.keys as usize as *const crate::array::ArrayHeader; + if keys.is_null() + || slot >= shape.live_inline_slot_count + || crate::object::key_attrs::keys_entry(keys, slot) + & crate::object::key_attrs::ENTRY_ACCESSOR + == 0 + { + return None; + } + let acc = crate::object::accessor_pair::pair_of_value(slot_bits(holder, slot))?; + if acc.raw_get == 0 && acc.raw_set == 0 { + return None; + } + HITS_ACCESSOR.fetch_add(1, Ordering::Relaxed); + super::stats_report_enabled(); + Some(invoke_class_getter(recv, acc.raw_get)) +} + unsafe fn walk(recv: *const ObjectHeader, name: &[u8]) -> Option { let mut w = Walk { holder: 0, @@ -384,6 +540,20 @@ pub(crate) unsafe fn prime_read_holder( } let name = crate::string::header_str_checked(key)?.as_bytes(); let recv = ordinary_receiver(obj as usize)?; + if let Some(acc) = class_accessor_walk(recv, name) { + let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); + if !cache.is_null() { + let w = Walk { + holder: acc.holder, + holder_shape: acc.shape, + slot: Some(acc.slot), + hops: [(0, 0); HOLDER_MAX_DEPTH - 1], + depth: 1, + }; + publish(cache, recv, &w, true); + } + return Some(invoke_class_getter(recv, acc.raw_get)); + } // Cheap pre-walk: a receiver the entry could never describe keeps the // caller's path and pays nothing for the getter below. A site with no // cache yet stays without one and uses the generic getter. @@ -437,11 +607,11 @@ pub(crate) unsafe fn prime_read_holder( if cache.is_null() { return Some(value); } - publish(cache, recv, &w); + publish(cache, recv, &w, false); Some(value) } -unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk) { +unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk, accessor: bool) { let c = &mut *cache; let token = (u64::from(object_shape_stamp(recv)) | PIC_ID_TOKEN_BIT) as i64; if c[HOLDER_RECV] != 0 && c[HOLDER_RECV] != token { @@ -459,14 +629,18 @@ unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk) { c[HOLDER_RECV] = 0; c[HOLDER_OBJ] = w.holder as i64; c[HOLDER_SHAPE] = (u64::from(w.holder_shape) | u64::from(w.hops[2].1) << 32) as i64; - c[HOLDER_KIND] = match (w.depth, w.slot) { - (1, Some(s)) => i64::from(s), - (1, None) => HOLDER_ABSENT_DEPTH1, - (d, s) => { - (HOLDER_STUB - | if s.is_none() { HOLDER_ABSENT_BIT } else { 0 } - | (d as u64) << HOLDER_DEPTH_SHIFT - | u64::from(s.unwrap_or(0))) as i64 + c[HOLDER_KIND] = if accessor { + (HOLDER_ACCESSOR | u64::from(w.slot.expect("class accessor has slot"))) as i64 + } else { + match (w.depth, w.slot) { + (1, Some(s)) => i64::from(s), + (1, None) => HOLDER_ABSENT_DEPTH1, + (d, s) => { + (HOLDER_STUB + | if s.is_none() { HOLDER_ABSENT_BIT } else { 0 } + | (d as u64) << HOLDER_DEPTH_SHIFT + | u64::from(s.unwrap_or(0))) as i64 + } } }; for i in 0..HOLDER_MAX_DEPTH - 1 { @@ -481,7 +655,9 @@ unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk) { } // Last: the entry is live only once every other word is written. c[HOLDER_RECV] = token; - if w.slot.is_some() { + if accessor { + PRIMES_ACCESSOR.fetch_add(1, Ordering::Relaxed); + } else if w.slot.is_some() { PRIMES_HOLDER.fetch_add(1, Ordering::Relaxed); } else { PRIMES_ABSENT.fetch_add(1, Ordering::Relaxed); @@ -508,7 +684,9 @@ pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVis if c[HOLDER_RECV] == 0 { continue; } - visitor.visit_i64_slot(&mut c[HOLDER_OBJ]); + if visitor.visit_i64_slot(&mut c[HOLDER_OBJ]) { + HOLDER_REWRITES.fetch_add(1, Ordering::Relaxed); + } for i in 0..HOLDER_MAX_DEPTH - 1 { visitor.visit_i64_slot(&mut c[HOLDER_HOPS + i]); } diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index ec2ae47852..b36b24c654 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -3379,8 +3379,8 @@ pub(crate) const PROTO_ID_DEFAULT: u64 = 0; pub(crate) const PROTO_ID_NULL: u64 = u64::MAX; const PROTO_ID_TAG_SHIFT: u32 = 62; pub(crate) const PROTO_ID_CLASS: u64 = 1 << PROTO_ID_TAG_SHIFT; -const PROTO_ID_MIXED: u64 = 2 << PROTO_ID_TAG_SHIFT; -const PROTO_ID_UNIQUE: u64 = 3 << PROTO_ID_TAG_SHIFT; +pub(crate) const PROTO_ID_MIXED: u64 = 2 << PROTO_ID_TAG_SHIFT; +pub(crate) const PROTO_ID_UNIQUE: u64 = 3 << PROTO_ID_TAG_SHIFT; /// The prototype identity of a shape that answers nothing about its receiver /// (a dictionary-kind shape shared by many receivers): `UNIQUE | 0`, which /// [`fresh_unique_proto_id`] never hands out (its counter starts at 1). diff --git a/crates/perry/tests/read_holder_accessor.rs b/crates/perry/tests/read_holder_accessor.rs new file mode 100644 index 0000000000..e18919dd4b --- /dev/null +++ b/crates/perry/tests/read_holder_accessor.rs @@ -0,0 +1,122 @@ +//! A class getter uses the read site's collecting accessor entry. The entry +//! must keep the original receiver, follow a changed prototype and descriptor, +//! survive a moving holder, and stop at worker startup. +use std::path::PathBuf; +use std::process::Command; + +const SOURCE: &str = r#"import { spawn } from 'perry/thread'; +class HolderA { get path(): number { return (this as any).n + 1; } } +class HolderB { get path(): number { return (this as any).n + 7; } } +class Host { n = 1; } +function read(o: any): number { return o.path; } +async function main(): Promise { + const o: any = new Host(); + const a: any = HolderA.prototype; + const b: any = HolderB.prototype; + // An explicit, serial prototype link gives Host a MIXED identity. + Object.setPrototypeOf(o, a); + let first = 0; + for (let i = 0; i < 1000; i++) first += read(o); + let keep: any[] = []; + for (let i = 0; i < 20000; i++) keep.push({ i }); + (globalThis as any).gc(); + keep = []; + let moved = 0; + for (let i = 0; i < 1000; i++) moved += read(o); + // Both prototypes declare the same accessor key. A receiver-link check, + // not holder shape alone, must reject a stale answer from A. + Object.setPrototypeOf(o, b); + let replaced = 0; + for (let i = 0; i < 1000; i++) replaced += read(o); + Object.defineProperty(b, 'path', { + configurable: true, + get() { return (this as any).n + 30; } + }); + let mutated = 0; + for (let i = 0; i < 1000; i++) mutated += read(o); + const sab = new SharedArrayBuffer(8); + const gate = new Int32Array(sab); + const pending = spawn(() => { + const workerGate = new Int32Array(sab); + class WorkerHolder { get path(): number { return (this as any).n + 100; } } + const w: any = { n: 3 }; + Object.setPrototypeOf(w, WorkerHolder.prototype); + (globalThis as any).gc(); + Atomics.store(workerGate, 0, 1); + Atomics.notify(workerGate, 0); + if (Atomics.wait(workerGate, 1, 0, 10000) === 'timed-out') throw new Error('primary did not overlap worker'); + let total = 0; + for (let i = 0; i < 1000; i++) total += read(w); + return total; + }); + if (Atomics.wait(gate, 0, 0, 10000) === 'timed-out') throw new Error('worker did not start'); + let overlap = 0; + for (let i = 0; i < 1000; i++) overlap += read(o); + Atomics.store(gate, 1, 1); + Atomics.notify(gate, 1); + const worker = await pending; + (globalThis as any).gc(); + console.log(first, moved, replaced, mutated, overlap, worker, read(o)); +} +main(); +"#; + +fn stat(stderr: &str, name: &str) -> u64 { + let line = stderr + .lines() + .find(|l| l.starts_with("[method-site]")) + .unwrap_or_else(|| panic!("missing method-site stats: {stderr}")); + line.split_whitespace() + .find_map(|w| w.strip_prefix(name).and_then(|v| v.strip_prefix('='))) + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| panic!("missing {name}: {line}")) +} + +#[test] +fn class_accessor_entry_collects_with_original_receiver_and_stops_for_workers() { + let dir = tempfile::tempdir().expect("tempdir"); + let entry = dir.path().join("main.ts"); + let output = dir.path().join("main_bin"); + std::fs::write(&entry, SOURCE).expect("write entry"); + let compile = Command::new(PathBuf::from(env!("CARGO_BIN_EXE_perry"))) + .current_dir(dir.path()) + .arg("compile") + .arg(&entry) + .arg("-o") + .arg(&output) + .env("PERRY_NO_CACHE", "1") + .output() + .expect("compile"); + assert!( + compile.status.success(), + "compile failed:\n{}", + String::from_utf8_lossy(&compile.stderr) + ); + let run = Command::new(&output) + .current_dir(dir.path()) + .env("PERRY_METHOD_SITE_STATS", "1") + .env("PERRY_GC_FORCE_EVACUATE", "1") + .env("PERRY_GC_VERIFY_EVACUATION", "1") + .env("PERRY_GC_POISON_FROMSPACE", "1") + .output() + .expect("run"); + let stderr = String::from_utf8_lossy(&run.stderr).into_owned(); + assert!(run.status.success(), "run failed:\n{stderr}"); + assert_eq!( + String::from_utf8_lossy(&run.stdout).trim(), + "2000 2000 8000 31000 31000 103000 31", + "{stderr}" + ); + assert!( + stat(&stderr, "read_accessor_primes") > 0, + "accessor entry never primed: {stderr}" + ); + assert!( + stat(&stderr, "read_accessor_hits") > 0, + "accessor entry never hit: {stderr}" + ); + assert!( + stat(&stderr, "read_holder_rewrites") > 0, + "holder never moved: {stderr}" + ); +} From 6c6467957c6675ef447c8eeca33be69ae45f9cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 12:35:58 +0000 Subject: [PATCH 11/40] Admit live-linked declared class accessors --- .../perry-runtime/src/object/method_site.rs | 7 +- .../src/object/method_site/read_holder.rs | 64 +++++++++++++++---- .../fixtures/read_holder_accessor_parity.ts | 41 ++++++++++++ crates/perry/tests/read_holder_accessor.rs | 23 +++++-- 4 files changed, 116 insertions(+), 19 deletions(-) create mode 100644 crates/perry/tests/fixtures/read_holder_accessor_parity.ts diff --git a/crates/perry-runtime/src/object/method_site.rs b/crates/perry-runtime/src/object/method_site.rs index 870387123b..eb2370816b 100644 --- a/crates/perry-runtime/src/object/method_site.rs +++ b/crates/perry-runtime/src/object/method_site.rs @@ -297,10 +297,13 @@ fn stats_report_enabled() -> bool { let (hd, ha, hr) = read_holder::read_holder_stats(); let (ap, ah) = read_holder::read_accessor_stats(); eprintln!( - "[method-site] primes_own={a} primes_inherited={b} primes_function={} holder_rewrites={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_accessor_primes={ap} read_accessor_hits={ah} read_holder_rewrites={} read_holder_refused={hr}{refused}", + "[method-site] primes_own={a} primes_inherited={b} primes_function={} holder_rewrites={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_accessor_primes={ap} read_accessor_hits={ah} read_accessor_class_primes={} read_holder_rewrites={} read_accessor_rewrites={} read_accessor_same_shape_relinks={} read_holder_refused={hr}{refused}", method_site_function_primes(), HOLDER_REWRITES.load(Ordering::Relaxed), - read_holder::read_holder_rewrites() + read_holder::read_accessor_class_primes(), + read_holder::read_holder_rewrites(), + read_holder::read_accessor_rewrites(), + read_holder::read_accessor_same_shape_relinks() ); } unsafe { libc::atexit(report) }; diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index fffdc65c0e..5d88a8ad2a 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -53,7 +53,7 @@ use super::{key_may_be_accessor, next_prototype, ordinary_receiver, WORKER_AGENTS_EXIST}; use crate::object::shapes::{ object_proto_id, object_shape_descriptor, object_shape_stamp, shape_proto_id, PIC_ID_TOKEN_BIT, - PROTO_ID_DEFAULT, PROTO_ID_MIXED, PROTO_ID_NULL, PROTO_ID_UNIQUE, + PROTO_ID_CLASS, PROTO_ID_DEFAULT, PROTO_ID_MIXED, PROTO_ID_NULL, PROTO_ID_UNIQUE, }; use crate::object::{ObjectHeader, PicCache, PicCacheSlot}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -111,6 +111,9 @@ per_test_global! { static PRIMES_ACCESSOR: AtomicU64 = AtomicU64::new(0); static HITS_ACCESSOR: AtomicU64 = AtomicU64::new(0); static HOLDER_REWRITES: AtomicU64 = AtomicU64::new(0); + static ACCESSOR_REWRITES: AtomicU64 = AtomicU64::new(0); + static SAME_SHAPE_RELINKS: AtomicU64 = AtomicU64::new(0); + static CLASS_PRIMES: AtomicU64 = AtomicU64::new(0); } /// `(data primes, absent primes, refusals)`. @@ -133,6 +136,18 @@ pub fn read_holder_rewrites() -> u64 { HOLDER_REWRITES.load(Ordering::Relaxed) } +pub fn read_accessor_rewrites() -> u64 { + ACCESSOR_REWRITES.load(Ordering::Relaxed) +} + +pub fn read_accessor_same_shape_relinks() -> u64 { + SAME_SHAPE_RELINKS.load(Ordering::Relaxed) +} + +pub fn read_accessor_class_primes() -> u64 { + CLASS_PRIMES.load(Ordering::Relaxed) +} + #[inline] fn refuse() { REFUSED_HOLDER.fetch_add(1, Ordering::Relaxed); @@ -292,15 +307,21 @@ pub(super) unsafe fn admitted_proto_id(obj: *const ObjectHeader) -> Option (object_proto_id(obj) == pid).then_some(pid) } -/// The class-instance form records both a class and a serial prototype link. -/// A bare CLASS identity cannot name one holder: the class registry may still -/// resolve its prototype lazily, and is deliberately refused. +/// A MIXED identity records an explicit serial link. A bare CLASS identity +/// does not pin its registry-resolved prototype, so the accessor-only hit must +/// compare the LIVE declared-prototype pointer with the cached holder. unsafe fn class_link(recv: *const ObjectHeader) -> Option<*const ObjectHeader> { let pid = shape_proto_id(object_shape_stamp(recv))?; - if !(PROTO_ID_MIXED..PROTO_ID_UNIQUE).contains(&pid) || object_proto_id(recv) != pid { + if object_proto_id(recv) != pid { return None; } - let holder = next_prototype(recv); + let holder = if (PROTO_ID_MIXED..PROTO_ID_UNIQUE).contains(&pid) { + next_prototype(recv) + } else if (PROTO_ID_CLASS..PROTO_ID_MIXED).contains(&pid) { + crate::object::class_decl_prototype_object((*recv).class_id) + } else { + return None; + }; (!holder.is_null() && holder != recv).then_some(holder) } @@ -383,12 +404,7 @@ pub(crate) unsafe fn try_cached_class_accessor( recv: *const ObjectHeader, cache_slot: *mut PicCacheSlot, ) -> Option { - if cache_slot.is_null() - || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 - || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT - || crate::object::field_get_set::accessor_receiver_override_armed() - || crate::object::prototype_chain::resolution_stack_savepoint() != 0 - { + if cache_slot.is_null() || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return None; } let cache = crate::object::field_get_set::pic_slot_peek::(cache_slot); @@ -396,10 +412,16 @@ pub(crate) unsafe fn try_cached_class_accessor( return None; } let c = &*cache; + if c[HOLDER_KIND] as u64 & HOLDER_ACCESSOR == 0 + || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT + || crate::object::field_get_set::accessor_receiver_override_armed() + || crate::object::prototype_chain::resolution_stack_savepoint() != 0 + { + return None; + } let stamp = object_shape_stamp(recv); if stamp == 0 || c[HOLDER_RECV] != (u64::from(stamp) | PIC_ID_TOKEN_BIT) as i64 - || c[HOLDER_KIND] as u64 & HOLDER_ACCESSOR == 0 || class_link(recv)? as usize != c[HOLDER_OBJ] as usize { return None; @@ -626,6 +648,14 @@ unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk, acc return; } } + if accessor + && c[HOLDER_RECV] != 0 + && c[HOLDER_KIND] as u64 & HOLDER_ACCESSOR != 0 + && c[HOLDER_OBJ] as usize != w.holder + && c[HOLDER_SHAPE] as u32 == w.holder_shape + { + SAME_SHAPE_RELINKS.fetch_add(1, Ordering::Relaxed); + } c[HOLDER_RECV] = 0; c[HOLDER_OBJ] = w.holder as i64; c[HOLDER_SHAPE] = (u64::from(w.holder_shape) | u64::from(w.hops[2].1) << 32) as i64; @@ -657,6 +687,11 @@ unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk, acc c[HOLDER_RECV] = token; if accessor { PRIMES_ACCESSOR.fetch_add(1, Ordering::Relaxed); + if shape_proto_id(object_shape_stamp(recv)) + .is_some_and(|pid| (PROTO_ID_CLASS..PROTO_ID_MIXED).contains(&pid)) + { + CLASS_PRIMES.fetch_add(1, Ordering::Relaxed); + } } else if w.slot.is_some() { PRIMES_HOLDER.fetch_add(1, Ordering::Relaxed); } else { @@ -686,6 +721,9 @@ pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVis } if visitor.visit_i64_slot(&mut c[HOLDER_OBJ]) { HOLDER_REWRITES.fetch_add(1, Ordering::Relaxed); + if c[HOLDER_KIND] as u64 & HOLDER_ACCESSOR != 0 { + ACCESSOR_REWRITES.fetch_add(1, Ordering::Relaxed); + } } for i in 0..HOLDER_MAX_DEPTH - 1 { visitor.visit_i64_slot(&mut c[HOLDER_HOPS + i]); diff --git a/crates/perry/tests/fixtures/read_holder_accessor_parity.ts b/crates/perry/tests/fixtures/read_holder_accessor_parity.ts new file mode 100644 index 0000000000..634afccb3a --- /dev/null +++ b/crates/perry/tests/fixtures/read_holder_accessor_parity.ts @@ -0,0 +1,41 @@ +// Pinned Node 26.5.1 parity for the non-worker half of read_holder_accessor.rs. +// Expected: 2000 2000 8000 31000 42000 31 +class HolderA { get path(): number { return (this as any).n + 1; } } +class HolderB { get path(): number { return (this as any).n + 7; } } +class Host { n = 1; } +function read(o: any): number { return o.path; } +async function main(): Promise { + const o: any = new Host(); + const a: any = HolderA.prototype; + const b: any = HolderB.prototype; + // An explicit, serial prototype link gives Host a MIXED identity. + Object.setPrototypeOf(o, a); + let first = 0; + for (let i = 0; i < 1000; i++) first += read(o); + let keep: any[] = []; + for (let i = 0; i < 20000; i++) keep.push({ i }); + (globalThis as any).gc(); + keep = []; + let moved = 0; + for (let i = 0; i < 1000; i++) moved += read(o); + // Both prototypes declare the same accessor key. A receiver-link check, + // not holder shape alone, must reject a stale answer from A. + Object.setPrototypeOf(o, b); + let replaced = 0; + for (let i = 0; i < 1000; i++) replaced += read(o); + Object.defineProperty(b, 'path', { + configurable: true, + get() { return (this as any).n + 30; } + }); + let mutated = 0; + for (let i = 0; i < 1000; i++) mutated += read(o); + // A declared class with no user override is the bare CLASS identity that + // resolves through the class registry, as Zod's ParseInputLazyPath does. + class Bare { n = 2; get path(): number { return this.n + 40; } } + function bareRead(x: any): number { return x.path; } + const bare: any = new Bare(); + let declared = 0; + for (let i = 0; i < 1000; i++) declared += bareRead(bare); + console.log(first, moved, replaced, mutated, declared, read(o)); +} +main(); diff --git a/crates/perry/tests/read_holder_accessor.rs b/crates/perry/tests/read_holder_accessor.rs index e18919dd4b..9723fb8625 100644 --- a/crates/perry/tests/read_holder_accessor.rs +++ b/crates/perry/tests/read_holder_accessor.rs @@ -34,6 +34,13 @@ async function main(): Promise { }); let mutated = 0; for (let i = 0; i < 1000; i++) mutated += read(o); + // A declared class with no user override is the bare CLASS identity that + // resolves through the class registry, as Zod's ParseInputLazyPath does. + class Bare { n = 2; get path(): number { return this.n + 40; } } + function bareRead(x: any): number { return x.path; } + const bare: any = new Bare(); + let declared = 0; + for (let i = 0; i < 1000; i++) declared += bareRead(bare); const sab = new SharedArrayBuffer(8); const gate = new Int32Array(sab); const pending = spawn(() => { @@ -56,7 +63,7 @@ async function main(): Promise { Atomics.notify(gate, 1); const worker = await pending; (globalThis as any).gc(); - console.log(first, moved, replaced, mutated, overlap, worker, read(o)); + console.log(first, moved, replaced, mutated, declared, overlap, worker, read(o)); } main(); "#; @@ -104,7 +111,7 @@ fn class_accessor_entry_collects_with_original_receiver_and_stops_for_workers() assert!(run.status.success(), "run failed:\n{stderr}"); assert_eq!( String::from_utf8_lossy(&run.stdout).trim(), - "2000 2000 8000 31000 31000 103000 31", + "2000 2000 8000 31000 42000 31000 103000 31", "{stderr}" ); assert!( @@ -116,7 +123,15 @@ fn class_accessor_entry_collects_with_original_receiver_and_stops_for_workers() "accessor entry never hit: {stderr}" ); assert!( - stat(&stderr, "read_holder_rewrites") > 0, - "holder never moved: {stderr}" + stat(&stderr, "read_accessor_class_primes") > 0, + "bare CLASS path never primed: {stderr}" + ); + assert!( + stat(&stderr, "read_accessor_rewrites") > 0, + "accessor holder never moved: {stderr}" + ); + assert!( + stat(&stderr, "read_accessor_same_shape_relinks") > 0, + "prototype replacement did not preserve holder shape: {stderr}" ); } From 1aedbb66febf6cecb4ddbfcfb0d0de8479f0058f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 12:28:31 +0000 Subject: [PATCH 12/40] Cache multiple receiver shapes for one absent read holder --- .../src/object/method_site/read_holder.rs | 182 +++++++++++++++++- 1 file changed, 179 insertions(+), 3 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 5d88a8ad2a..581b82f5e7 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -96,6 +96,11 @@ const HOLDER_ABSENT_BIT: u64 = 1 << 62; /// A direct class-prototype accessor. It can collect and therefore never /// answers from the GC-leaf front call. const HOLDER_ACCESSOR: u64 = 1 << 61; +/// Depth-1 ABSENT entries can share one terminal holder across several +/// receiver shapes. The spare hop words hold ShapeIds, never GC pointers. +const HOLDER_MULTI_ABSENT: u64 = 1 << 60; +const MULTI_ABSENT_EXTRA_IDS: usize = 9; +const MULTI_ABSENT_NEXT_MASK: u64 = 0xf; const HOLDER_DEPTH_SHIFT: u32 = 32; const HOLDER_MAX_DEPTH: usize = 4; @@ -175,13 +180,27 @@ pub(crate) unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return None; } - if c[HOLDER_RECV] != token || token == 0 { + if token == 0 { return None; } let kind = c[HOLDER_KIND]; if kind as u64 & HOLDER_ACCESSOR != 0 { return None; } + if kind as u64 & HOLDER_MULTI_ABSENT != 0 { + if c[HOLDER_RECV] == 0 + || kind as u64 & HOLDER_ABSENT_BIT == 0 + || (c[HOLDER_RECV] != token + && !(0..MULTI_ABSENT_EXTRA_IDS).any(|i| multi_absent_id(c, i) == token as u32)) + { + return None; + } + return (shape_word(c[HOLDER_OBJ] as usize) == c[HOLDER_SHAPE] as u32) + .then_some(crate::value::TAG_UNDEFINED); + } + if c[HOLDER_RECV] != token { + return None; + } let (depth, absent, slot) = if kind >= 0 { (1, kind == HOLDER_ABSENT_DEPTH1, kind as u32) } else { @@ -213,6 +232,34 @@ pub(crate) unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { Some(slot_bits(holder, slot)) } +/// Spare depth-1 ABSENT words: the upper half of the holder-shape word and +/// four words that otherwise hold intermediate hop addresses/shapes. +/// The root scanner visits only HOLDER_OBJ for this entry kind. +#[inline] +fn multi_absent_id(c: &PicCache, i: usize) -> u32 { + debug_assert!(i < MULTI_ABSENT_EXTRA_IDS); + if i == 0 { + (c[HOLDER_SHAPE] as u64 >> 32) as u32 + } else { + let word = HOLDER_HOPS + (i - 1) / 2; + (c[word] as u64 >> (32 * ((i - 1) % 2))) as u32 + } +} + +#[inline] +fn set_multi_absent_id(c: &mut PicCache, i: usize, id: u32) { + debug_assert!(i < MULTI_ABSENT_EXTRA_IDS); + if i == 0 { + c[HOLDER_SHAPE] = + ((c[HOLDER_SHAPE] as u64 & u64::from(u32::MAX)) | (u64::from(id) << 32)) as i64; + } else { + let word = HOLDER_HOPS + (i - 1) / 2; + let shift = 32 * ((i - 1) % 2); + let mask = u64::from(u32::MAX) << shift; + c[word] = ((c[word] as u64 & !mask) | (u64::from(id) << shift)) as i64; + } +} + /// The site's holder entry asked for `handle`, without priming: what the /// emitted tower's holder check answers, for a runtime caller that asks the /// site's words itself (`typed_feedback::guards`' class-field miss arm). @@ -636,6 +683,39 @@ pub(crate) unsafe fn prime_read_holder( unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk, accessor: bool) { let c = &mut *cache; let token = (u64::from(object_shape_stamp(recv)) | PIC_ID_TOKEN_BIT) as i64; + // A default-prototype optional field is often absent from many receiver + // shapes at ONE read site (TypeScript AST's `.name` is the canonical + // case). Keep the already-confirmed receiver shapes in this site's spare + // words when they all share the same terminal object and terminal shape. + // Every new token is admitted only after the caller's generic getter + // returned `undefined` and `walk` proved the complete chain absent. + let old_kind = c[HOLDER_KIND] as u64; + if !accessor + && w.depth == 1 + && w.slot.is_none() + && c[HOLDER_RECV] != 0 + && c[HOLDER_RECV] != token + && (old_kind == HOLDER_ABSENT_DEPTH1 as u64 || old_kind & HOLDER_MULTI_ABSENT != 0) + && c[HOLDER_OBJ] as usize == w.holder + && c[HOLDER_SHAPE] as u32 == w.holder_shape + { + let next = if old_kind & HOLDER_MULTI_ABSENT == 0 { + 0 + } else { + (old_kind & MULTI_ABSENT_NEXT_MASK) as usize + }; + debug_assert!(next < MULTI_ABSENT_EXTRA_IDS); + let previous = c[HOLDER_RECV] as u32; + c[HOLDER_RECV] = 0; + set_multi_absent_id(c, next, previous); + c[HOLDER_KIND] = (HOLDER_ABSENT_BIT + | HOLDER_MULTI_ABSENT + | ((next + 1) % MULTI_ABSENT_EXTRA_IDS) as u64) as i64; + c[HOLDER_RECV] = token; + PRIMES_ABSENT.fetch_add(1, Ordering::Relaxed); + super::stats_report_enabled(); + return; + } if c[HOLDER_RECV] != 0 && c[HOLDER_RECV] != token { // The site's non-own receivers take more than one shape. One entry // cannot hold them; after a few replacements the site stops priming. @@ -725,8 +805,10 @@ pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVis ACCESSOR_REWRITES.fetch_add(1, Ordering::Relaxed); } } - for i in 0..HOLDER_MAX_DEPTH - 1 { - visitor.visit_i64_slot(&mut c[HOLDER_HOPS + i]); + if c[HOLDER_KIND] as u64 & HOLDER_MULTI_ABSENT == 0 { + for i in 0..HOLDER_MAX_DEPTH - 1 { + visitor.visit_i64_slot(&mut c[HOLDER_HOPS + i]); + } } } } @@ -735,6 +817,100 @@ pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVis mod tests { use super::*; + #[test] + fn ten_receiver_shapes_share_one_confirmed_absent_terminal() { + let _lock = crate::gc::global_side_table_test_lock(); + let gate = WORKER_AGENTS_EXIST.swap(0, Ordering::SeqCst); + let base = crate::object::shapes::SHAPE_ID_BASE; + let holder = Box::new(ObjectHeader { + class_id: 0, + parent_class_id: base + 100, + meta: std::ptr::null_mut(), + }); + let other = Box::new(ObjectHeader { + class_id: 0, + parent_class_id: base + 101, + meta: std::ptr::null_mut(), + }); + let mut cache = [0; crate::object::PIC_CACHE_WORDS]; + // The stack cache is not a process-lifetime PIC allocation. Skip + // registration; this test exercises only the published words. + cache[HOLDER_STATE] = STATE_REGISTERED; + let mut recv = ObjectHeader { + class_id: 0, + parent_class_id: base, + meta: std::ptr::null_mut(), + }; + let absent = Walk { + holder: (&*holder as *const ObjectHeader) as usize, + holder_shape: base + 100, + slot: None, + hops: [(0, 0); HOLDER_MAX_DEPTH - 1], + depth: 1, + }; + for i in 0..11 { + recv.parent_class_id = base + i; + unsafe { publish(&mut cache, &recv, &absent, false) }; + } + assert_ne!(cache[HOLDER_KIND] as u64 & HOLDER_MULTI_ABSENT, 0); + assert_eq!( + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base)) as i64) }, + None, + "the oldest of eleven shapes must leave a ten-shape site" + ); + for i in 1..11 { + let token = (PIC_ID_TOKEN_BIT | u64::from(base + i)) as i64; + assert_eq!( + unsafe { entry_answer(&cache, token) }, + Some(crate::value::TAG_UNDEFINED) + ); + } + // A new shape after an own-key shadow has no entry, while a terminal + // mutation invalidates every receiver shape in the shared entry. + assert_eq!( + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64) }, + None + ); + let mut moved = Box::new(ObjectHeader { + class_id: 0, + parent_class_id: base + 100, + meta: std::ptr::null_mut(), + }); + cache[HOLDER_OBJ] = (&mut *moved as *mut ObjectHeader) as i64; + assert_eq!( + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64) }, + Some(crate::value::TAG_UNDEFINED) + ); + moved.parent_class_id = base + 102; + assert_eq!( + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64) }, + None + ); + // A different terminal never inherits the old entry's receiver set. + let distinct = Walk { + holder: (&*other as *const ObjectHeader) as usize, + holder_shape: base + 101, + ..absent + }; + recv.parent_class_id = base + 11; + unsafe { publish(&mut cache, &recv, &distinct, false) }; + assert_eq!(cache[HOLDER_KIND], HOLDER_ABSENT_DEPTH1); + assert_eq!( + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64) }, + None + ); + assert_eq!( + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64) }, + Some(crate::value::TAG_UNDEFINED) + ); + WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); + assert_eq!( + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64) }, + None + ); + WORKER_AGENTS_EXIST.store(gate, Ordering::SeqCst); + } + /// A class instance has a valid, stamped ShapeId, but its prototype is /// resolved through the class vtable. The holder walk must refuse it even /// when the shape and the object's current prototype id agree. From 0dda83ef2b86640cb7d79a9b25b73603f5bd337f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 12:37:59 +0000 Subject: [PATCH 13/40] test: cover polymorphic absent reads across prototype and GC changes --- .../one_shape_multi_absent/expected.txt | 6 ++ tests/fixtures/one_shape_multi_absent/main.ts | 55 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 tests/fixtures/one_shape_multi_absent/expected.txt create mode 100644 tests/fixtures/one_shape_multi_absent/main.ts diff --git a/tests/fixtures/one_shape_multi_absent/expected.txt b/tests/fixtures/one_shape_multi_absent/expected.txt new file mode 100644 index 0000000000..d27beea412 --- /dev/null +++ b/tests/fixtures/one_shape_multi_absent/expected.txt @@ -0,0 +1,6 @@ +all-absent 0 +own-shadow 250 +terminal-add 3750 +terminal-value 4750 +terminal-delete 250 +different-terminal 3575 diff --git a/tests/fixtures/one_shape_multi_absent/main.ts b/tests/fixtures/one_shape_multi_absent/main.ts new file mode 100644 index 0000000000..a813351411 --- /dev/null +++ b/tests/fixtures/one_shape_multi_absent/main.ts @@ -0,0 +1,55 @@ +// Eleven receiver shapes at one read site, sharing one absent null-prototype +// terminal. The last shape rotates the ten-shape site and must remain correct. +const terminal: any = Object.create(null); +const receivers: any[] = []; +for (let n = 0; n < 11; n++) { + const o: any = Object.create(terminal); + for (let k = 0; k < n; k++) o["field" + k] = k; + receivers.push(o); +} + +function read(o: any): number { + const value = o.missing; + return value === undefined ? 0 : value; +} + +let sum = 0; +for (let i = 0; i < 550; i++) sum += read(receivers[i % receivers.length]); +console.log("all-absent", sum); + +// The terminal is young when the site primes. A forced collection must keep +// and rewrite the site's rooted holder; a later own-key shadow cannot reuse +// the same receiver ShapeId and must win over the absent entry. +let churn: any[] = []; +for (let i = 0; i < 20000; i++) churn.push({ i }); +(globalThis as any).gc(); +churn = []; +receivers[3].missing = 5; +sum = 0; +for (let i = 0; i < 550; i++) sum += read(receivers[i % receivers.length]); +console.log("own-shadow", sum); + +// A new terminal shape invalidates every stored receiver shape. Reassigning +// the terminal value without changing its shape must be observed as well. +terminal.missing = 7; +sum = 0; +for (let i = 0; i < 550; i++) sum += read(receivers[i % receivers.length]); +console.log("terminal-add", sum); +terminal.missing = 9; +sum = 0; +for (let i = 0; i < 550; i++) sum += read(receivers[i % receivers.length]); +console.log("terminal-value", sum); +delete terminal.missing; +sum = 0; +for (let i = 0; i < 550; i++) sum += read(receivers[i % receivers.length]); +console.log("terminal-delete", sum); + +// Two receivers with the same own key list can have different prototype +// identities. Their absent facts must not be shared through the site. +const otherTerminal: any = Object.create(null); +otherTerminal.missing = 13; +const otherReceiver: any = Object.create(otherTerminal); +otherReceiver.field0 = 0; +sum = 0; +for (let i = 0; i < 550; i++) sum += read(i % 2 ? otherReceiver : receivers[1]); +console.log("different-terminal", sum); From 398133b5a20820e61f78aaa137b88971c58942e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 13:20:05 +0000 Subject: [PATCH 14/40] Warm lazy class getter site without latching and root read key --- .../src/object/method_site/read_holder.rs | 150 ++++++++++++++++-- .../fixtures/read_holder_accessor_parity.ts | 30 +++- crates/perry/tests/read_holder_accessor.rs | 55 +++++-- 3 files changed, 207 insertions(+), 28 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 581b82f5e7..98988b1bc5 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -4,7 +4,9 @@ //! * The receiver's ShapeId `S` vouches that `k` is not own, that the receiver //! is an ordinary object, and its [[Prototype]] identity. Only a serial //! identity or `PROTO_ID_DEFAULT` (the realm's `Object.prototype`) pins ONE -//! object, so only those admit. +//! object for the GC-leaf data/absent path. A collecting accessor entry may +//! also use a declared class identity because it rechecks the live registry +//! prototype pointer on every hit. //! * The holder's ShapeId `SH` vouches that `k` is an own inline data slot of //! the holder `H` — or, for an ABSENT entry, that the terminal object lacks //! `k` and has a null [[Prototype]]. @@ -36,10 +38,10 @@ //! # Priming //! //! Only from the primary agent's read miss handler, which already knows the key -//! is not own, and only after the generic getter has produced the answer: the entry is -//! recorded only when what the shapes say equals what the getter returned -//! (names the runtime synthesizes, lazily materialized intrinsics and -//! `constructor` refuse there). A worker agent's start gates all further +//! is not own. Data and absent entries are recorded only after the generic +//! getter's answer agrees with the shapes. A class accessor is published only +//! after its compiled pair is validated, then its getter runs once. A worker +//! agent's start gates all further //! holder hits and primes; stale entries stop being roots and can collect. //! //! A miss whose receiver the live entry already answers is served from the @@ -71,6 +73,8 @@ pub const HOLDER_SHAPE: usize = crate::codegen_abi::PIC_HOLDER_SHAPE_WORD; /// |---|---| /// | `0 ..= u32::MAX` | depth 1, the value is the holder's inline slot | /// | [`HOLDER_ABSENT_DEPTH1`] | depth 1, absent: the answer is `undefined` | +/// | [`HOLDER_ACCESSOR`] + slot | direct class-prototype accessor; collecting hit only | +/// | [`HOLDER_MULTI_ABSENT`] | depth-1 absent for up to ten receiver shapes | /// | negative | [`HOLDER_STUB`] set: depth 2..=4 and/or a deep absent entry | pub const HOLDER_KIND: usize = crate::codegen_abi::PIC_HOLDER_KIND_WORD; /// First of three intermediate hop addresses (depth 2..=4). @@ -623,6 +627,18 @@ pub(crate) unsafe fn prime_read_holder( } return Some(invoke_class_getter(recv, acc.raw_get)); } + // A declared class prototype is created lazily. The first getter read + // can reach its vtable while the holder object still does not exist. Let + // that ONE generic read materialize it without latching the site; the + // next miss can validate the real accessor pair and publish. We never + // call the getter twice or infer its first answer from post-call state. + let pending_class_accessor = holder_name_admitted(name) + && shape_proto_id(object_shape_stamp(recv)) + .is_some_and(|pid| (PROTO_ID_CLASS..PROTO_ID_MIXED).contains(&pid)) + && crate::object::class_decl_prototype_object((*recv).class_id).is_null() + && std::str::from_utf8(name).ok().is_some_and(|name| { + crate::object::class_chain_has_instance_accessor((*recv).class_id, name) + }); // Cheap pre-walk: a receiver the entry could never describe keeps the // caller's path and pays nothing for the getter below. A site with no // cache yet stays without one and uses the generic getter. @@ -633,9 +649,10 @@ pub(crate) unsafe fn prime_read_holder( // below is what resolves it). So an unresolved realm // does not decide the pre-walk; the walk after the getter does. let realm_pending = crate::array::object_prototype_addr_if_resolved() == 0; - if !holder_name_admitted(name) - || key_may_be_accessor(recv, name) - || (walk(recv, name).is_none() && !realm_pending) + if !pending_class_accessor + && (!holder_name_admitted(name) + || key_may_be_accessor(recv, name) + || (walk(recv, name).is_none() && !realm_pending)) { refuse_and_latch(existing); return None; @@ -645,16 +662,24 @@ pub(crate) unsafe fn prime_read_holder( // so the receiver is rooted across it and everything is re-read after. let scope = crate::gc::RuntimeHandleScope::new(); let handle = scope.root_raw_mut_ptr(obj as *mut ObjectHeader); + let key_handle = scope.root_string_ptr(key); let (value, obj) = handle.across_mut::(|| { crate::object::field_get_set::get_field_by_name_after_site_miss(obj, key) }); if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return Some(value); } + if pending_class_accessor { + if crate::object::class_decl_prototype_object((*obj).class_id).is_null() { + let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); + refuse_and_latch(cache); + } + return Some(value); + } // From here a refusal has already run the getter, so the site latches: // the next miss must not walk and run it again only to refuse again. let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); - let name = crate::string::header_str_checked(key)?.as_bytes(); + let name = crate::string::header_str_checked(key_handle.get_raw_const_ptr())?.as_bytes(); let Some(recv) = ordinary_receiver(obj as usize) else { refuse_and_latch(cache); return Some(value); @@ -817,6 +842,113 @@ pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVis mod tests { use super::*; + extern "C" fn getter_two(_this: f64) -> f64 { + 2.0 + } + extern "C" fn getter_eight(_this: f64) -> f64 { + 8.0 + } + + /// Two holders with exactly one ShapeId but different compiled getters. + /// Replacing a declared class's registry pointer leaves the receiver's + /// bare CLASS ShapeId unchanged; the collecting hit must compare the live + /// link, rather than trust receiver and holder shapes alone. + #[test] + fn class_accessor_rechecks_same_shape_holder_link() { + let _lock = crate::gc::global_side_table_test_lock(); + const CID: u32 = 0x0C3C_79A3; + let scope = crate::gc::RuntimeHandleScope::new(); + let p1 = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 2)); + let p2 = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 2)); + for (holder, raw_get) in [ + (&p1, getter_two as *const () as usize), + (&p2, getter_eight as *const () as usize), + ] { + holder.with_mut_ptr::(|ptr| { + crate::object::set_builtin_accessor_pair( + ptr as usize, + "path".to_owned(), + crate::object::accessor_pair::Accessor { + raw_get, + ..Default::default() + }, + crate::object::PropertyAttrs::new(true, false, true), + ); + }); + } + let p1_addr = p1.get_raw_const_ptr::() as usize; + let p2_addr = p2.get_raw_const_ptr::() as usize; + let shape = unsafe { object_shape_stamp(p1_addr as *const ObjectHeader) }; + assert_ne!(p1_addr, p2_addr); + assert_eq!(shape, unsafe { + object_shape_stamp(p2_addr as *const ObjectHeader) + }); + + let packed = b"holder_class_key"; + let keys = crate::object::js_build_class_keys_array( + CID, + 1, + packed.as_ptr(), + packed.len() as u32, + 0, + ); + let recv_shape = crate::object::shapes::js_object_shape_id_for_class_keys( + keys as usize as u64, + 1, + CID, + 0, + ); + let recv = crate::object::js_object_alloc_class_inline_keys_stamped( + CID, 0, 1, keys, recv_shape, 0, + ); + let recv = scope.root_raw_mut_ptr(recv); + let receiver = recv.get_raw_const_ptr::(); + assert_eq!( + unsafe { shape_proto_id(object_shape_stamp(receiver)) }, + Some(PROTO_ID_CLASS | u64::from(CID)) + ); + + crate::object::test_seed_class_decl_prototype_object_root(CID, p1_addr); + let first = unsafe { class_accessor_walk(receiver, b"path") }.expect("first accessor"); + let cache: &'static mut PicCache = + Box::leak(Box::new([0; crate::codegen_abi::PIC_CACHE_WORDS])); + let mut slot: PicCacheSlot = cache; + let w = Walk { + holder: first.holder, + holder_shape: first.shape, + slot: Some(first.slot), + hops: [(0, 0); HOLDER_MAX_DEPTH - 1], + depth: 1, + }; + unsafe { publish(cache, receiver, &w, true) }; + assert_eq!( + unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), + Some(2.0) + ); + + let old_relinks = read_accessor_same_shape_relinks(); + crate::object::test_seed_class_decl_prototype_object_root(CID, p2_addr); + assert_eq!(unsafe { object_shape_stamp(receiver) }, recv_shape); + assert!( + unsafe { try_cached_class_accessor(receiver, &mut slot) }.is_none(), + "stale getter was served after registry replacement" + ); + let second = unsafe { class_accessor_walk(receiver, b"path") }.expect("second accessor"); + let w = Walk { + holder: second.holder, + holder_shape: second.shape, + slot: Some(second.slot), + hops: [(0, 0); HOLDER_MAX_DEPTH - 1], + depth: 1, + }; + unsafe { publish(cache, receiver, &w, true) }; + assert!(read_accessor_same_shape_relinks() > old_relinks); + assert_eq!( + unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), + Some(8.0) + ); + } + #[test] fn ten_receiver_shapes_share_one_confirmed_absent_terminal() { let _lock = crate::gc::global_side_table_test_lock(); diff --git a/crates/perry/tests/fixtures/read_holder_accessor_parity.ts b/crates/perry/tests/fixtures/read_holder_accessor_parity.ts index 634afccb3a..d46561f414 100644 --- a/crates/perry/tests/fixtures/read_holder_accessor_parity.ts +++ b/crates/perry/tests/fixtures/read_holder_accessor_parity.ts @@ -1,17 +1,31 @@ // Pinned Node 26.5.1 parity for the non-worker half of read_holder_accessor.rs. // Expected: 2000 2000 8000 31000 42000 31 -class HolderA { get path(): number { return (this as any).n + 1; } } -class HolderB { get path(): number { return (this as any).n + 7; } } -class Host { n = 1; } +function makeHolder() { + return class { get path(): number { + if ((globalThis as any).accessorCollect) { + (globalThis as any).accessorCollect = false; + (globalThis as any).gc(); + } + return (this as any).child.value + (this as any).factor; + } }; +} +const HolderA = makeHolder(); +const HolderB = makeHolder(); +class Host { n = 1; child = { value: 1 }; } function read(o: any): number { return o.path; } async function main(): Promise { const o: any = new Host(); const a: any = HolderA.prototype; const b: any = HolderB.prototype; + a.factor = 1; + b.factor = 7; // An explicit, serial prototype link gives Host a MIXED identity. Object.setPrototypeOf(o, a); let first = 0; - for (let i = 0; i < 1000; i++) first += read(o); + for (let i = 0; i < 1000; i++) { + if (i === 500) (globalThis as any).accessorCollect = true; + first += read(o); + } let keep: any[] = []; for (let i = 0; i < 20000; i++) keep.push({ i }); (globalThis as any).gc(); @@ -25,7 +39,13 @@ async function main(): Promise { for (let i = 0; i < 1000; i++) replaced += read(o); Object.defineProperty(b, 'path', { configurable: true, - get() { return (this as any).n + 30; } + get() { + if ((globalThis as any).accessorCollect) { + (globalThis as any).accessorCollect = false; + (globalThis as any).gc(); + } + return (this as any).child.value + 30; + } }); let mutated = 0; for (let i = 0; i < 1000; i++) mutated += read(o); diff --git a/crates/perry/tests/read_holder_accessor.rs b/crates/perry/tests/read_holder_accessor.rs index 9723fb8625..9366357e25 100644 --- a/crates/perry/tests/read_holder_accessor.rs +++ b/crates/perry/tests/read_holder_accessor.rs @@ -5,32 +5,51 @@ use std::path::PathBuf; use std::process::Command; const SOURCE: &str = r#"import { spawn } from 'perry/thread'; -class HolderA { get path(): number { return (this as any).n + 1; } } -class HolderB { get path(): number { return (this as any).n + 7; } } -class Host { n = 1; } +function makeHolder() { + return class { get path(): number { + if ((globalThis as any).accessorCollect) { + (globalThis as any).accessorCollect = false; + (globalThis as any).gc(); + } + return (this as any).child.value + (this as any).factor; + } }; +} +const HolderA = makeHolder(); +const HolderB = makeHolder(); +class Host { n = 1; child = { value: 1 }; } function read(o: any): number { return o.path; } async function main(): Promise { const o: any = new Host(); const a: any = HolderA.prototype; const b: any = HolderB.prototype; + a.factor = 1; + b.factor = 7; // An explicit, serial prototype link gives Host a MIXED identity. Object.setPrototypeOf(o, a); let first = 0; - for (let i = 0; i < 1000; i++) first += read(o); + for (let i = 0; i < 1000; i++) { + if (i === 500) (globalThis as any).accessorCollect = true; + first += read(o); + } let keep: any[] = []; for (let i = 0; i < 20000; i++) keep.push({ i }); (globalThis as any).gc(); keep = []; let moved = 0; for (let i = 0; i < 1000; i++) moved += read(o); - // Both prototypes declare the same accessor key. A receiver-link check, - // not holder shape alone, must reject a stale answer from A. + // A live receiver-link check rejects A after the prototype is replaced. Object.setPrototypeOf(o, b); let replaced = 0; for (let i = 0; i < 1000; i++) replaced += read(o); Object.defineProperty(b, 'path', { configurable: true, - get() { return (this as any).n + 30; } + get() { + if ((globalThis as any).accessorCollect) { + (globalThis as any).accessorCollect = false; + (globalThis as any).gc(); + } + return (this as any).child.value + 30; + } }); let mutated = 0; for (let i = 0; i < 1000; i++) mutated += read(o); @@ -45,7 +64,13 @@ async function main(): Promise { const gate = new Int32Array(sab); const pending = spawn(() => { const workerGate = new Int32Array(sab); - class WorkerHolder { get path(): number { return (this as any).n + 100; } } + class WorkerHolder { get path(): number { + if ((globalThis as any).workerAccessorCollect) { + (globalThis as any).workerAccessorCollect = false; + (globalThis as any).gc(); + } + return (this as any).n + 100; + } } const w: any = { n: 3 }; Object.setPrototypeOf(w, WorkerHolder.prototype); (globalThis as any).gc(); @@ -53,12 +78,18 @@ async function main(): Promise { Atomics.notify(workerGate, 0); if (Atomics.wait(workerGate, 1, 0, 10000) === 'timed-out') throw new Error('primary did not overlap worker'); let total = 0; - for (let i = 0; i < 1000; i++) total += read(w); + for (let i = 0; i < 1000; i++) { + if (i === 500) (globalThis as any).workerAccessorCollect = true; + total += read(w); + } return total; }); if (Atomics.wait(gate, 0, 0, 10000) === 'timed-out') throw new Error('worker did not start'); let overlap = 0; - for (let i = 0; i < 1000; i++) overlap += read(o); + for (let i = 0; i < 1000; i++) { + if (i === 500) (globalThis as any).accessorCollect = true; + overlap += read(o); + } Atomics.store(gate, 1, 1); Atomics.notify(gate, 1); const worker = await pending; @@ -130,8 +161,4 @@ fn class_accessor_entry_collects_with_original_receiver_and_stops_for_workers() stat(&stderr, "read_accessor_rewrites") > 0, "accessor holder never moved: {stderr}" ); - assert!( - stat(&stderr, "read_accessor_same_shape_relinks") > 0, - "prototype replacement did not preserve holder shape: {stderr}" - ); } From 8d4d48b705d338921cf7dae0d1b58510f3c702df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 13:18:25 +0000 Subject: [PATCH 15/40] Add bounded collecting class read memo for absent and inherited data --- .../src/object/field_get_set/ic_miss.rs | 5 + .../perry-runtime/src/object/method_site.rs | 3 +- .../src/object/method_site/read_holder.rs | 63 ++- .../method_site/read_holder/class_read.rs | 378 ++++++++++++++++++ 4 files changed, 433 insertions(+), 16 deletions(-) create mode 100644 crates/perry-runtime/src/object/method_site/read_holder/class_read.rs diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index 7d19773c20..36f1fa1160 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -806,6 +806,11 @@ pub(super) fn get_field_ic_miss_impl( // An accessor can run JS and collect, so this lives in the collecting // miss handler. The leaf front only recognizes data and absent entries. if gc_kind == Some(crate::gc::GC_TYPE_OBJECT) { + if let Some(value) = unsafe { + crate::object::method_site::read_holder::try_cached_class_read(obj, cache_slot) + } { + return f64::from_bits(value.bits()); + } if let Some(value) = unsafe { crate::object::method_site::read_holder::try_cached_class_accessor(obj, cache_slot) } { diff --git a/crates/perry-runtime/src/object/method_site.rs b/crates/perry-runtime/src/object/method_site.rs index eb2370816b..f07dfe0adf 100644 --- a/crates/perry-runtime/src/object/method_site.rs +++ b/crates/perry-runtime/src/object/method_site.rs @@ -296,8 +296,9 @@ fn stats_report_enabled() -> bool { } let (hd, ha, hr) = read_holder::read_holder_stats(); let (ap, ah) = read_holder::read_accessor_stats(); + let (cp, ch, cr) = read_holder::class_read_stats(); eprintln!( - "[method-site] primes_own={a} primes_inherited={b} primes_function={} holder_rewrites={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_accessor_primes={ap} read_accessor_hits={ah} read_accessor_class_primes={} read_holder_rewrites={} read_accessor_rewrites={} read_accessor_same_shape_relinks={} read_holder_refused={hr}{refused}", + "[method-site] primes_own={a} primes_inherited={b} primes_function={} holder_rewrites={} misses={c} read_holder_primes={hd} read_absent_primes={ha} read_accessor_primes={ap} read_accessor_hits={ah} read_accessor_class_primes={} class_read_primes={cp} class_read_hits={ch} class_read_root_rewrites={cr} read_holder_rewrites={} read_accessor_rewrites={} read_accessor_same_shape_relinks={} read_holder_refused={hr}{refused}", method_site_function_primes(), HOLDER_REWRITES.load(Ordering::Relaxed), read_holder::read_accessor_class_primes(), diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 98988b1bc5..4520a79f9a 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -60,6 +60,8 @@ use crate::object::shapes::{ use crate::object::{ObjectHeader, PicCache, PicCacheSlot}; use std::sync::atomic::{AtomicU64, Ordering}; +mod class_read; + /// The receiver's ShapeId as a PIC token (`ShapeId | PIC_ID_TOKEN_BIT`), or 0 /// for an empty entry. A zeroed cache is therefore an empty one: no token is 0. pub const HOLDER_RECV: usize = crate::codegen_abi::PIC_HOLDER_RECV_WORD; @@ -141,6 +143,10 @@ pub fn read_accessor_stats() -> (u64, u64) { ) } +pub fn class_read_stats() -> (u64, u64, u64) { + class_read::stats() +} + pub fn read_holder_rewrites() -> u64 { HOLDER_REWRITES.load(Ordering::Relaxed) } @@ -448,6 +454,14 @@ unsafe fn invoke_class_getter(recv: *const ObjectHeader, raw_get: usize) -> crat crate::value::JSValue::from_bits(bits) } +/// Collecting-path class data/absence memo; the leaf front never consults it. +pub(crate) unsafe fn try_cached_class_read( + recv: *const ObjectHeader, + cache_slot: *mut PicCacheSlot, +) -> Option { + class_read::try_hit(recv, cache_slot) +} + /// Collecting read-miss arm. The GC-leaf front always declines this kind. /// Every hit confirms the receiver's shape and live link, the rooted holder's /// shape, and the accessor descriptor before invoking with the ORIGINAL receiver. @@ -501,7 +515,7 @@ pub(crate) unsafe fn try_cached_class_accessor( Some(invoke_class_getter(recv, acc.raw_get)) } -unsafe fn walk(recv: *const ObjectHeader, name: &[u8]) -> Option { +unsafe fn walk(recv: *const ObjectHeader, name: &[u8], class_first: bool) -> Option { let mut w = Walk { holder: 0, holder_shape: 0, @@ -518,6 +532,18 @@ unsafe fn walk(recv: *const ObjectHeader, name: &[u8]) -> Option { let terminal = depth > 1 && current as usize == object_prototype; let pid = if terminal { PROTO_ID_NULL + } else if depth == 1 && class_first { + // A bare CLASS ShapeId does not pin the registry's live + // C.prototype. The collecting class-read hit compares that + // pointer on every use; this walk records it as the first hop. + crate::object::shapes::PROTO_ID_CLASS + } else if class_first { + let pid = shape_proto_id(object_shape_stamp(current))?; + if (PROTO_ID_MIXED..PROTO_ID_UNIQUE).contains(&pid) && object_proto_id(current) == pid { + pid + } else { + admitted_proto_id(current)? + } } else { admitted_proto_id(current)? }; @@ -533,7 +559,9 @@ unsafe fn walk(recv: *const ObjectHeader, name: &[u8]) -> Option { w.depth = depth - 1; return Some(w); } - let next = if pid == PROTO_ID_DEFAULT { + let next = if depth == 1 && class_first { + class_link(recv)? + } else if pid == PROTO_ID_DEFAULT { object_prototype as *const ObjectHeader } else { next_prototype(current) @@ -639,6 +667,11 @@ pub(crate) unsafe fn prime_read_holder( && std::str::from_utf8(name).ok().is_some_and(|name| { crate::object::class_chain_has_instance_accessor((*recv).class_id, name) }); + if !pending_class_accessor { + if let Some(value) = class_read::prime(recv, key, cache_slot, name) { + return Some(value); + } + } // Cheap pre-walk: a receiver the entry could never describe keeps the // caller's path and pays nothing for the getter below. A site with no // cache yet stays without one and uses the generic getter. @@ -652,7 +685,7 @@ pub(crate) unsafe fn prime_read_holder( if !pending_class_accessor && (!holder_name_admitted(name) || key_may_be_accessor(recv, name) - || (walk(recv, name).is_none() && !realm_pending)) + || (walk(recv, name, false).is_none() && !realm_pending)) { refuse_and_latch(existing); return None; @@ -684,7 +717,7 @@ pub(crate) unsafe fn prime_read_holder( refuse_and_latch(cache); return Some(value); }; - let Some(w) = walk(recv, name) else { + let Some(w) = walk(recv, name, false) else { refuse_and_latch(cache); return Some(value); }; @@ -821,20 +854,20 @@ pub(crate) fn scan_read_holder_roots_mut(visitor: &mut crate::gc::RuntimeRootVis // SAFETY: registered caches are PIC-arena allocations // (`pic_arena_alloc`), which are never freed. let c = unsafe { &mut *(site as *mut PicCache) }; - if c[HOLDER_RECV] == 0 { - continue; - } - if visitor.visit_i64_slot(&mut c[HOLDER_OBJ]) { - HOLDER_REWRITES.fetch_add(1, Ordering::Relaxed); - if c[HOLDER_KIND] as u64 & HOLDER_ACCESSOR != 0 { - ACCESSOR_REWRITES.fetch_add(1, Ordering::Relaxed); + if c[HOLDER_RECV] != 0 { + if visitor.visit_i64_slot(&mut c[HOLDER_OBJ]) { + HOLDER_REWRITES.fetch_add(1, Ordering::Relaxed); + if c[HOLDER_KIND] as u64 & HOLDER_ACCESSOR != 0 { + ACCESSOR_REWRITES.fetch_add(1, Ordering::Relaxed); + } } - } - if c[HOLDER_KIND] as u64 & HOLDER_MULTI_ABSENT == 0 { - for i in 0..HOLDER_MAX_DEPTH - 1 { - visitor.visit_i64_slot(&mut c[HOLDER_HOPS + i]); + if c[HOLDER_KIND] as u64 & HOLDER_MULTI_ABSENT == 0 { + for i in 0..HOLDER_MAX_DEPTH - 1 { + visitor.visit_i64_slot(&mut c[HOLDER_HOPS + i]); + } } } + class_read::scan_roots(c, visitor); } } diff --git a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs new file mode 100644 index 0000000000..f142db4589 --- /dev/null +++ b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs @@ -0,0 +1,378 @@ +//! Bounded read-site facts for declared-class instances. +//! +//! A bare CLASS ShapeId identifies the receiver's own keys but does not pin +//! `C.prototype`: the class registry can replace that pointer without a +//! receiver restamp. These entries therefore run only from the collecting +//! miss arm, where they re-read the live direct prototype on every hit. +//! The entries belong to one PicCache site (word 2 points to its bounded +//! process-lifetime record), never to a process-global `(shape, key)` table. + +use super::*; + +const SITE_WORD: usize = 2; // the existing PIC's unused scratch word +const SITE_TAG: u64 = 0xA2C1_0000_0000_0000; +const STATE_CLASS_SITE: i64 = 4; +const WAYS: usize = 16; + +#[derive(Clone, Copy)] +struct Entry { + token: i64, + class_id: u32, + depth: u8, + absent: bool, + slot: u32, + holder: usize, + holder_shape: u32, + hops: [(usize, u32); HOLDER_MAX_DEPTH - 1], +} + +const EMPTY: Entry = Entry { + token: 0, + class_id: 0, + depth: 0, + absent: false, + slot: 0, + holder: 0, + holder_shape: 0, + hops: [(0, 0); HOLDER_MAX_DEPTH - 1], +}; + +struct Site { + entries: [Entry; WAYS], + next: usize, +} + +per_test_global! { + static PRIMES: AtomicU64 = AtomicU64::new(0); + static HITS: AtomicU64 = AtomicU64::new(0); + static ROOT_REWRITES: AtomicU64 = AtomicU64::new(0); +} + +pub(super) fn stats() -> (u64, u64, u64) { + ( + PRIMES.load(Ordering::Relaxed), + HITS.load(Ordering::Relaxed), + ROOT_REWRITES.load(Ordering::Relaxed), + ) +} + +#[inline] +unsafe fn site(c: &PicCache) -> Option<&Site> { + let word = c[SITE_WORD] as u64; + if c[HOLDER_STATE] & STATE_CLASS_SITE == 0 || word & !crate::value::POINTER_MASK != SITE_TAG { + return None; + } + Some(&*((word & crate::value::POINTER_MASK) as usize as *const Site)) +} + +/// The current link from an intermediate hop, computed by the same admitted +/// shape/prototype rule the prime walk used. A changed or exotic link declines. +unsafe fn admitted_next(hop: *const ObjectHeader) -> Option { + let pid = shape_proto_id(object_shape_stamp(hop))?; + if object_proto_id(hop) != pid { + return None; + } + if !(pid == PROTO_ID_DEFAULT + || pid == PROTO_ID_NULL + || (PROTO_ID_MIXED..PROTO_ID_UNIQUE).contains(&pid) + || (pid < crate::object::shapes::PROTO_ID_CLASS && pid != PROTO_ID_DEFAULT)) + { + return None; + } + let next = if pid == PROTO_ID_DEFAULT { + crate::array::object_prototype_addr_if_resolved() + } else if pid == PROTO_ID_NULL { + 0 + } else { + next_prototype(hop) as usize + }; + (next != 0).then_some(next) +} + +unsafe fn answer(e: &Entry, recv: *const ObjectHeader) -> Option { + if e.token == 0 + || e.class_id != (*recv).class_id + || e.token != (u64::from(object_shape_stamp(recv)) | PIC_ID_TOKEN_BIT) as i64 + { + return None; + } + let direct = if e.depth == 1 { e.holder } else { e.hops[0].0 }; + if class_link(recv)? as usize != direct { + return None; + } + let mut previous = 0usize; + for i in 0..(e.depth as usize).saturating_sub(1) { + let (addr, shape) = e.hops[i]; + if addr == 0 || shape_word(addr) != shape { + return None; + } + if i != 0 && admitted_next(previous as *const ObjectHeader)? != addr { + return None; + } + previous = addr; + } + if previous != 0 && admitted_next(previous as *const ObjectHeader)? != e.holder { + return None; + } + if e.holder == 0 || shape_word(e.holder) != e.holder_shape { + return None; + } + if e.absent { + Some(crate::value::TAG_UNDEFINED) + } else { + let bits = slot_bits(e.holder, e.slot); + // The generic inherited getter treats nullish/hole holder values as + // a miss and may continue to a farther prototype. The holder's + // ShapeId does not change on a value overwrite, so recheck each hit. + (bits != crate::value::TAG_UNDEFINED + && bits != crate::value::TAG_NULL + && bits != crate::value::TAG_HOLE) + .then_some(bits) + } +} + +/// A class entry is served on the collecting miss path only. The GC-leaf +/// front has no receiver-root/registry contract for bare CLASS prototypes. +pub(super) unsafe fn try_hit( + recv: *const ObjectHeader, + cache_slot: *mut PicCacheSlot, +) -> Option { + if cache_slot.is_null() + || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 + || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT + { + return None; + } + let cache = crate::object::field_get_set::pic_slot_peek::(cache_slot); + if cache.is_null() { + return None; + } + let s = site(&*cache)?; + for e in &s.entries { + if let Some(bits) = answer(e, recv) { + HITS.fetch_add(1, Ordering::Relaxed); + super::super::stats_report_enabled(); + return Some(crate::value::JSValue::from_bits(bits)); + } + } + None +} + +/// Prime only after the generic getter's result has been compared with the +/// same live chain and slot. Returns None if this receiver has no class link. +pub(super) unsafe fn prime( + obj: *const ObjectHeader, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + name: &[u8], +) -> Option { + if class_link(obj).is_none() + || !holder_name_admitted(name) + || key_may_be_accessor(obj, name) + || walk(obj, name, true).is_none() + { + return None; + } + let scope = crate::gc::RuntimeHandleScope::new(); + let handle = scope.root_raw_mut_ptr(obj as *mut ObjectHeader); + let key_handle = scope.root_string_ptr(key); + let (value, obj) = handle.across_mut::(|| { + crate::object::field_get_set::get_field_by_name_after_site_miss( + obj, + key_handle.get_raw_const_ptr::(), + ) + }); + if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { + return Some(value); + } + let key = key_handle.get_raw_const_ptr::(); + let name = crate::string::header_str_checked(key)?.as_bytes(); + let Some(obj) = ordinary_receiver(obj as usize) else { + return Some(value); + }; + let Some(w) = walk(obj, name, true) else { + return Some(value); + }; + let bits = value.bits(); + let confirmed = match w.slot { + None => bits == crate::value::TAG_UNDEFINED, + Some(slot) => { + bits == slot_bits(w.holder, slot) + && bits != crate::value::TAG_HOLE + && bits != crate::value::TAG_UNDEFINED + && bits != crate::value::TAG_NULL + } + }; + if confirmed { + let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); + if !cache.is_null() { + publish(cache, obj, &w); + } + } + Some(value) +} + +unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk) { + let c = &mut *cache; + // Other PIC users may leave scratch data in word 2. Only a marker that + // we published together with a tagged pointer grants dereference rights. + let tagged = c[SITE_WORD] as u64; + let has_site = + c[HOLDER_STATE] & STATE_CLASS_SITE != 0 && tagged & !crate::value::POINTER_MASK == SITE_TAG; + let s = if !has_site { + let new = Box::into_raw(Box::new(Site { + entries: [EMPTY; WAYS], + next: 0, + })); + if c[HOLDER_STATE] & STATE_REGISTERED == 0 { + let mut sites = HOLDER_SITES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + sites.push(cache as usize); + c[HOLDER_STATE] |= STATE_REGISTERED; + } + let addr = new as usize as u64; + assert_eq!(addr & !crate::value::POINTER_MASK, 0); + c[SITE_WORD] = (SITE_TAG | addr) as i64; + c[HOLDER_STATE] |= STATE_CLASS_SITE; + &mut *new + } else { + &mut *((c[SITE_WORD] as u64 & crate::value::POINTER_MASK) as usize as *mut Site) + }; + let token = (u64::from(object_shape_stamp(recv)) | PIC_ID_TOKEN_BIT) as i64; + let index = s + .entries + .iter() + .position(|e| e.token == token && e.class_id == (*recv).class_id) + .unwrap_or_else(|| { + let i = s.next; + s.next = (s.next + 1) % WAYS; + i + }); + s.entries[index] = Entry { + token, + class_id: (*recv).class_id, + depth: w.depth as u8, + absent: w.slot.is_none(), + slot: w.slot.unwrap_or(0), + holder: w.holder, + holder_shape: w.holder_shape, + hops: w.hops, + }; + PRIMES.fetch_add(1, Ordering::Relaxed); + super::super::stats_report_enabled(); +} + +/// The PIC arena retains every site; its class entries are strong roots +/// until workers start. Every address is rewritten in place after evacuation. +pub(super) fn scan_roots(c: &mut PicCache, visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + let word = c[SITE_WORD] as u64; + if c[HOLDER_STATE] & STATE_CLASS_SITE == 0 || word & !crate::value::POINTER_MASK != SITE_TAG { + return; + } + let s = unsafe { &mut *((word & crate::value::POINTER_MASK) as usize as *mut Site) }; + for e in &mut s.entries { + if e.token == 0 { + continue; + } + if visitor.visit_tagged_usize_slot(&mut e.holder, crate::value::POINTER_TAG) { + ROOT_REWRITES.fetch_add(1, Ordering::Relaxed); + } + for i in 0..(e.depth as usize).saturating_sub(1) { + if visitor.visit_tagged_usize_slot(&mut e.hops[i].0, crate::value::POINTER_TAG) { + ROOT_REWRITES.fetch_add(1, Ordering::Relaxed); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn foreign_scratch_word_is_not_a_site() { + let mut cache = [0i64; crate::object::PIC_CACHE_WORDS]; + cache[SITE_WORD] = 0xA11CE; + assert!(unsafe { site(&cache) }.is_none()); + cache[HOLDER_STATE] |= STATE_CLASS_SITE; + assert!(unsafe { site(&cache) }.is_none()); + } + + #[test] + fn bare_class_link_replacement_with_same_holder_shape_declines() { + let _lock = crate::gc::global_side_table_test_lock(); + const CID: u32 = 0x0C3C_79A3; + const PROTO_CID: u32 = 0x0C3C_79A4; + let proto_keys = + crate::object::js_build_class_keys_array(PROTO_CID, 1, b"marker".as_ptr(), 6, 0); + let proto_shape = crate::object::shapes::js_object_shape_id_for_class_keys( + proto_keys as usize as u64, + 1, + PROTO_CID, + 0, + ); + let a = crate::object::js_object_alloc_class_inline_keys_stamped( + PROTO_CID, + 0, + 1, + proto_keys, + proto_shape, + 0, + ); + crate::object::class_decl_prototype_object_root_store(CID, a); + let b = crate::object::js_object_alloc_class_inline_keys_stamped( + PROTO_CID, + 0, + 1, + proto_keys, + proto_shape, + 0, + ); + let a = crate::object::class_decl_prototype_object(CID); + assert_ne!(a, b); + assert_eq!(unsafe { object_shape_stamp(a) }, unsafe { + object_shape_stamp(b) + }); + let recv_keys = crate::object::js_build_class_keys_array(CID, 1, b"own".as_ptr(), 3, 0); + let recv_shape = crate::object::shapes::js_object_shape_id_for_class_keys( + recv_keys as usize as u64, + 1, + CID, + 0, + ); + let recv = crate::object::js_object_alloc_class_inline_keys_stamped( + CID, 0, 1, recv_keys, recv_shape, 0, + ); + assert_eq!(unsafe { class_link(recv) }, Some(a as *const ObjectHeader)); + let entry = Entry { + token: (PIC_ID_TOKEN_BIT | u64::from(recv_shape)) as i64, + class_id: CID, + depth: 1, + absent: true, + slot: 0, + holder: a as usize, + holder_shape: proto_shape, + hops: [(0, 0); HOLDER_MAX_DEPTH - 1], + }; + assert_eq!( + unsafe { answer(&entry, recv) }, + Some(crate::value::TAG_UNDEFINED) + ); + let mut data_entry = entry; + data_entry.absent = false; + unsafe { + let slot = (a as *mut u8).add(std::mem::size_of::()) as *mut u64; + std::ptr::write(slot, 42.0f64.to_bits()); + assert_eq!(answer(&data_entry, recv), Some(42.0f64.to_bits())); + std::ptr::write(slot, crate::value::TAG_UNDEFINED); + assert_eq!(answer(&data_entry, recv), None); + std::ptr::write(slot, crate::value::TAG_NULL); + assert_eq!(answer(&data_entry, recv), None); + std::ptr::write(slot, 43.0f64.to_bits()); + assert_eq!(answer(&data_entry, recv), Some(43.0f64.to_bits())); + } + crate::object::class_decl_prototype_object_root_store(CID, b); + assert_eq!(unsafe { answer(&entry, recv) }, None); + } +} From 0d1ab0466a1d5b354b17704fd1bed44dedd9d0b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 13:22:41 +0000 Subject: [PATCH 16/40] Add class-instance optional-read parity fixture --- .../one_shape_class_read/expected.txt | 12 ++++ tests/fixtures/one_shape_class_read/main.ts | 58 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 tests/fixtures/one_shape_class_read/expected.txt create mode 100644 tests/fixtures/one_shape_class_read/main.ts diff --git a/tests/fixtures/one_shape_class_read/expected.txt b/tests/fixtures/one_shape_class_read/expected.txt new file mode 100644 index 0000000000..4b55406efe --- /dev/null +++ b/tests/fixtures/one_shape_class_read/expected.txt @@ -0,0 +1,12 @@ +class-absent 0 +after-gc 0 +own-shadow 500 +prototype-add 8200 +prototype-value 10400 +prototype-delete 500 +object-prototype-add 12600 +object-prototype-delete 500 +first-terminal 500 +first-terminal-add 14800 +second-terminal 500 +second-terminal-add 19200 diff --git a/tests/fixtures/one_shape_class_read/main.ts b/tests/fixtures/one_shape_class_read/main.ts new file mode 100644 index 0000000000..b557900299 --- /dev/null +++ b/tests/fixtures/one_shape_class_read/main.ts @@ -0,0 +1,58 @@ +// A TypeScript AST-like declared class: optional fields are read through one +// polymorphic site while each instance has a different own-key layout. +class NodeObject { + kind = 1; +} + +const nodes: any[] = []; +for (let n = 0; n < 12; n++) { + const node: any = new NodeObject(); + for (let k = 0; k < n; k++) node["extra" + k] = k; + nodes.push(node); +} + +function read(node: any): number { + const value = node.optional; + return value === undefined ? 0 : value; +} + +function sumReads(): number { + let sum = 0; + for (let i = 0; i < 1200; i++) sum += read(nodes[i % nodes.length]); + return sum; +} + +console.log("class-absent", sumReads()); + +let churn: any[] = []; +for (let i = 0; i < 20000; i++) churn.push({ i }); +(globalThis as any).gc(); +churn = []; +console.log("after-gc", sumReads()); + +nodes[3].optional = 5; +console.log("own-shadow", sumReads()); + +(NodeObject.prototype as any).optional = 7; +console.log("prototype-add", sumReads()); +(NodeObject.prototype as any).optional = 9; +console.log("prototype-value", sumReads()); +delete (NodeObject.prototype as any).optional; +console.log("prototype-delete", sumReads()); + +(Object.prototype as any).optional = 11; +console.log("object-prototype-add", sumReads()); +delete (Object.prototype as any).optional; +console.log("object-prototype-delete", sumReads()); + +// Two same-layout terminals exercise live intermediate-link validation. +const first = { marker: 1 }; +const second = { marker: 2 }; +Object.setPrototypeOf(NodeObject.prototype, first); +console.log("first-terminal", sumReads()); +(first as any).optional = 13; +console.log("first-terminal-add", sumReads()); +Object.setPrototypeOf(NodeObject.prototype, second); +console.log("second-terminal", sumReads()); +(second as any).optional = 17; +console.log("second-terminal-add", sumReads()); From fd227ae049994edb626f4779d8e0db40155f9472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 13:27:08 +0000 Subject: [PATCH 17/40] Test class read memo worker gate on collecting hit --- .../method_site/read_holder/class_read.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs index f142db4589..e6a3234a6b 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs @@ -374,5 +374,24 @@ mod tests { } crate::object::class_decl_prototype_object_root_store(CID, b); assert_eq!(unsafe { answer(&entry, recv) }, None); + + crate::object::class_decl_prototype_object_root_store(CID, a); + let record = Box::into_raw(Box::new(Site { + entries: [entry; WAYS], + next: 0, + })); + let mut cache = [0i64; crate::object::PIC_CACHE_WORDS]; + cache[SITE_WORD] = (SITE_TAG | record as usize as u64) as i64; + cache[HOLDER_STATE] = STATE_CLASS_SITE; + let mut slot = &mut cache as *mut PicCache; + let gate = WORKER_AGENTS_EXIST.swap(0, Ordering::SeqCst); + assert_eq!( + unsafe { try_hit(recv, &mut slot) }.map(|v| v.bits()), + Some(crate::value::TAG_UNDEFINED) + ); + WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); + assert!(unsafe { try_hit(recv, &mut slot) }.is_none()); + WORKER_AGENTS_EXIST.store(gate, Ordering::SeqCst); + unsafe { drop(Box::from_raw(record)) }; } } From fd7d6389cba812164450dd7a2b54bdf99f22630f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 13:46:10 +0000 Subject: [PATCH 18/40] Add real-worker class read gate parity fixture --- .../one_shape_class_read_worker/check.sh | 35 +++++++++++++ .../expected-worker.txt | 5 ++ .../one_shape_class_read_worker/expected.txt | 3 ++ .../one_shape_class_read_worker/main.ts | 50 +++++++++++++++++++ .../one_shape_class_read_worker/worker.cjs | 31 ++++++++++++ 5 files changed, 124 insertions(+) create mode 100755 tests/fixtures/one_shape_class_read_worker/check.sh create mode 100644 tests/fixtures/one_shape_class_read_worker/expected-worker.txt create mode 100644 tests/fixtures/one_shape_class_read_worker/expected.txt create mode 100644 tests/fixtures/one_shape_class_read_worker/main.ts create mode 100644 tests/fixtures/one_shape_class_read_worker/worker.cjs diff --git a/tests/fixtures/one_shape_class_read_worker/check.sh b/tests/fixtures/one_shape_class_read_worker/check.sh new file mode 100755 index 0000000000..20b45bbe07 --- /dev/null +++ b/tests/fixtures/one_shape_class_read_worker/check.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +binary=$(realpath "${1:?pass the compiled fixture executable}") +fixture_dir=$(cd "$(dirname "$0")" && pwd) +repo_root=$(cd "$fixture_dir/../../.." && pwd) +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT +cd "$repo_root" + +PERRY_METHOD_SITE_STATS=1 PERRY_GC_FORCE_EVACUATE=1 \ + PERRY_GC_VERIFY_EVACUATION=1 PERRY_GC_DIAG=1 "$binary" \ + > "$tmp_dir/control.out" 2> "$tmp_dir/control.err" +A2_START_WORKER=1 PERRY_METHOD_SITE_STATS=1 PERRY_GC_FORCE_EVACUATE=1 \ + PERRY_GC_VERIFY_EVACUATION=1 PERRY_GC_DIAG=1 "$binary" \ + > "$tmp_dir/worker.out" 2> "$tmp_dir/worker.err" +cmp "$fixture_dir/expected.txt" "$tmp_dir/control.out" +cmp "$fixture_dir/expected-worker.txt" "$tmp_dir/worker.out" + +sum_counter() { + sed -n "s/.*$2=\([0-9][0-9]*\).*/\1/p" "$1" | + awk '{ sum += $1 } END { print sum + 0 }' +} +control_hits=$(sum_counter "$tmp_dir/control.err" class_read_hits) +worker_hits=$(sum_counter "$tmp_dir/worker.err" class_read_hits) +control_rewrites=$(sum_counter "$tmp_dir/control.err" class_read_root_rewrites) +control_copied=$(sum_counter "$tmp_dir/control.err" copied_objects) +worker_copied=$(sum_counter "$tmp_dir/worker.err" copied_objects) +test "$worker_hits" -gt 0 +test "$control_hits" -gt "$worker_hits" +test "$control_rewrites" -gt 0 +test "$control_copied" -gt 0 +test "$worker_copied" -gt 0 +printf 'class_read_hits control=%s worker=%s; root_rewrites=%s; copied_objects control=%s worker=%s\n' \ + "$control_hits" "$worker_hits" "$control_rewrites" "$control_copied" "$worker_copied" diff --git a/tests/fixtures/one_shape_class_read_worker/expected-worker.txt b/tests/fixtures/one_shape_class_read_worker/expected-worker.txt new file mode 100644 index 0000000000..1b68344d9d --- /dev/null +++ b/tests/fixtures/one_shape_class_read_worker/expected-worker.txt @@ -0,0 +1,5 @@ +before 0 +after 0 +after-again 0 +worker 0 +worker-exit 1 diff --git a/tests/fixtures/one_shape_class_read_worker/expected.txt b/tests/fixtures/one_shape_class_read_worker/expected.txt new file mode 100644 index 0000000000..7d973dea31 --- /dev/null +++ b/tests/fixtures/one_shape_class_read_worker/expected.txt @@ -0,0 +1,3 @@ +before 0 +after 0 +after-again 0 diff --git a/tests/fixtures/one_shape_class_read_worker/main.ts b/tests/fixtures/one_shape_class_read_worker/main.ts new file mode 100644 index 0000000000..94a8d24e1b --- /dev/null +++ b/tests/fixtures/one_shape_class_read_worker/main.ts @@ -0,0 +1,50 @@ +import { Worker } from "node:worker_threads"; + +// Compare this same read site with and without a real Worker start. Once any +// worker exists, class-read entries must decline on both agents. +class NodeObject { + kind = 1; +} +const nodes: any[] = []; +for (let n = 0; n < 12; n++) { + const node: any = new NodeObject(); + for (let k = 0; k < n; k++) node["extra" + k] = k; + nodes.push(node); +} +function read(node: any): number { + const value = node.optional; + return value === undefined ? 0 : value; +} +function sumReads(): number { + let sum = 0; + for (let i = 0; i < 1200; i++) sum += read(nodes[i % nodes.length]); + return sum; +} +function collect(): void { + let churn: any[] = []; + for (let i = 0; i < 20000; i++) churn.push({ i }); + (globalThis as any).gc(); + churn = []; +} + +console.log("before", sumReads()); +if (process.env.A2_START_WORKER !== "1") { + collect(); + console.log("after", sumReads()); + console.log("after-again", sumReads()); +} else { + process.chdir("tests/fixtures/one_shape_class_read_worker"); + const worker = new Worker("./worker.cjs"); + worker.on("message", (message: any) => { + if (message === "ready") { + collect(); + console.log("after", sumReads()); + console.log("after-again", sumReads()); + worker.postMessage("go"); + } else { + console.log("worker", message); + worker.terminate(); + } + }); + worker.on("exit", (code: number) => console.log("worker-exit", code)); +} diff --git a/tests/fixtures/one_shape_class_read_worker/worker.cjs b/tests/fixtures/one_shape_class_read_worker/worker.cjs new file mode 100644 index 0000000000..2d7532ca10 --- /dev/null +++ b/tests/fixtures/one_shape_class_read_worker/worker.cjs @@ -0,0 +1,31 @@ +const { parentPort } = require("node:worker_threads"); + +class WorkerNode { + constructor() { + this.kind = 1; + } +} +const nodes = []; +for (let n = 0; n < 12; n++) { + const node = new WorkerNode(); + for (let k = 0; k < n; k++) node["extra" + k] = k; + nodes.push(node); +} +function read(node) { + const value = node.optional; + return value === undefined ? 0 : value; +} +function sumReads() { + let sum = 0; + for (let i = 0; i < 1200; i++) sum += read(nodes[i % nodes.length]); + return sum; +} +parentPort.on("message", (message) => { + if (message !== "go") return; + let churn = []; + for (let i = 0; i < 20000; i++) churn.push({ i }); + global.gc(); + churn = []; + parentPort.postMessage(sumReads()); +}); +parentPort.postMessage("ready"); From fc84b409d36bc214f356cb80a8c481cfa94a3fd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 14:09:25 +0000 Subject: [PATCH 19/40] Keep multi-absent lookup off ordinary holder hits --- .../src/object/method_site/read_holder.rs | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 4520a79f9a..d5c9f34613 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -193,24 +193,20 @@ pub(crate) unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { if token == 0 { return None; } - let kind = c[HOLDER_KIND]; - if kind as u64 & HOLDER_ACCESSOR != 0 { - return None; + // The common data/stub entry checks its receiver token before decoding + // kinds. Only a token miss can search the extra ABSENT shapes; keeping + // that search off the ordinary inherited hit avoids taxing every read. + if c[HOLDER_RECV] != token { + return multi_absent_extra_answer(c, token); } - if kind as u64 & HOLDER_MULTI_ABSENT != 0 { - if c[HOLDER_RECV] == 0 - || kind as u64 & HOLDER_ABSENT_BIT == 0 - || (c[HOLDER_RECV] != token - && !(0..MULTI_ABSENT_EXTRA_IDS).any(|i| multi_absent_id(c, i) == token as u32)) - { + let kind = c[HOLDER_KIND]; + if kind as u64 & (HOLDER_ACCESSOR | HOLDER_MULTI_ABSENT) != 0 { + if kind as u64 & HOLDER_ACCESSOR != 0 || kind as u64 & HOLDER_ABSENT_BIT == 0 { return None; } return (shape_word(c[HOLDER_OBJ] as usize) == c[HOLDER_SHAPE] as u32) .then_some(crate::value::TAG_UNDEFINED); } - if c[HOLDER_RECV] != token { - return None; - } let (depth, absent, slot) = if kind >= 0 { (1, kind == HOLDER_ABSENT_DEPTH1, kind as u32) } else { @@ -242,6 +238,25 @@ pub(crate) unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { Some(slot_bits(holder, slot)) } +/// A second through tenth ABSENT receiver shape is a rare path relative to +/// one-token data hits. It shares the terminal holder but must still prove the +/// entry is live and its ShapeId has not changed. +#[cold] +#[inline(never)] +unsafe fn multi_absent_extra_answer(c: &PicCache, token: i64) -> Option { + let kind = c[HOLDER_KIND] as u64; + if c[HOLDER_RECV] == 0 + || kind & (HOLDER_MULTI_ABSENT | HOLDER_ABSENT_BIT) + != HOLDER_MULTI_ABSENT | HOLDER_ABSENT_BIT + || kind & HOLDER_ACCESSOR != 0 + || !(0..MULTI_ABSENT_EXTRA_IDS).any(|i| multi_absent_id(c, i) == token as u32) + { + return None; + } + (shape_word(c[HOLDER_OBJ] as usize) == c[HOLDER_SHAPE] as u32) + .then_some(crate::value::TAG_UNDEFINED) +} + /// Spare depth-1 ABSENT words: the upper half of the holder-shape word and /// four words that otherwise hold intermediate hop addresses/shapes. /// The root scanner visits only HOLDER_OBJ for this entry kind. From fc74ad454a136019120e18e5bae49de3acdd1303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 14:21:01 +0000 Subject: [PATCH 20/40] Scope read-holder key and test pointers to noncollecting use --- .../src/object/method_site/read_holder.rs | 155 +++++++++--------- 1 file changed, 82 insertions(+), 73 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index d5c9f34613..8920ad70e5 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -727,30 +727,32 @@ pub(crate) unsafe fn prime_read_holder( // From here a refusal has already run the getter, so the site latches: // the next miss must not walk and run it again only to refuse again. let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); - let name = crate::string::header_str_checked(key_handle.get_raw_const_ptr())?.as_bytes(); - let Some(recv) = ordinary_receiver(obj as usize) else { - refuse_and_latch(cache); - return Some(value); - }; - let Some(w) = walk(recv, name, false) else { - refuse_and_latch(cache); - return Some(value); - }; - // Confirm: what the shapes say must be what the getter returned. - let bits = value.bits(); - let confirmed = match w.slot { - None => bits == crate::value::TAG_UNDEFINED, - Some(s) => bits == slot_bits(w.holder, s) && bits != crate::value::TAG_HOLE, - }; - if !confirmed { - refuse_and_latch(cache); - return Some(value); - } - if cache.is_null() { - return Some(value); - } - publish(cache, recv, &w, false); - Some(value) + key_handle.with_const_ptr::(|key| { + let name = crate::string::header_str_checked(key)?.as_bytes(); + let Some(recv) = ordinary_receiver(obj as usize) else { + refuse_and_latch(cache); + return Some(value); + }; + let Some(w) = walk(recv, name, false) else { + refuse_and_latch(cache); + return Some(value); + }; + // This scoped key pointer is used only by the non-collecting walk + // and confirmation. The generic getter has already returned. + let bits = value.bits(); + let confirmed = match w.slot { + None => bits == crate::value::TAG_UNDEFINED, + Some(s) => bits == slot_bits(w.holder, s) && bits != crate::value::TAG_HOLE, + }; + if !confirmed { + refuse_and_latch(cache); + return Some(value); + } + if !cache.is_null() { + publish(cache, recv, &w, false); + } + Some(value) + }) } unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk, accessor: bool) { @@ -924,12 +926,13 @@ mod tests { ); }); } - let p1_addr = p1.get_raw_const_ptr::() as usize; - let p2_addr = p2.get_raw_const_ptr::() as usize; - let shape = unsafe { object_shape_stamp(p1_addr as *const ObjectHeader) }; - assert_ne!(p1_addr, p2_addr); - assert_eq!(shape, unsafe { - object_shape_stamp(p2_addr as *const ObjectHeader) + p1.with_const_ptr::(|first| { + p2.with_const_ptr::(|second| { + assert_ne!(first, second); + assert_eq!(unsafe { object_shape_stamp(first) }, unsafe { + object_shape_stamp(second) + }); + }); }); let packed = b"holder_class_key"; @@ -950,51 +953,57 @@ mod tests { CID, 0, 1, keys, recv_shape, 0, ); let recv = scope.root_raw_mut_ptr(recv); - let receiver = recv.get_raw_const_ptr::(); - assert_eq!( - unsafe { shape_proto_id(object_shape_stamp(receiver)) }, - Some(PROTO_ID_CLASS | u64::from(CID)) - ); + recv.with_const_ptr::(|receiver| { + assert_eq!( + unsafe { shape_proto_id(object_shape_stamp(receiver)) }, + Some(PROTO_ID_CLASS | u64::from(CID)) + ); - crate::object::test_seed_class_decl_prototype_object_root(CID, p1_addr); - let first = unsafe { class_accessor_walk(receiver, b"path") }.expect("first accessor"); - let cache: &'static mut PicCache = - Box::leak(Box::new([0; crate::codegen_abi::PIC_CACHE_WORDS])); - let mut slot: PicCacheSlot = cache; - let w = Walk { - holder: first.holder, - holder_shape: first.shape, - slot: Some(first.slot), - hops: [(0, 0); HOLDER_MAX_DEPTH - 1], - depth: 1, - }; - unsafe { publish(cache, receiver, &w, true) }; - assert_eq!( - unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), - Some(2.0) - ); + p1.with_const_ptr::(|ptr| { + crate::object::test_seed_class_decl_prototype_object_root(CID, ptr as usize); + }); + let first = unsafe { class_accessor_walk(receiver, b"path") }.expect("first accessor"); + let cache: &'static mut PicCache = + Box::leak(Box::new([0; crate::codegen_abi::PIC_CACHE_WORDS])); + let mut slot: PicCacheSlot = cache; + let w = Walk { + holder: first.holder, + holder_shape: first.shape, + slot: Some(first.slot), + hops: [(0, 0); HOLDER_MAX_DEPTH - 1], + depth: 1, + }; + unsafe { publish(cache, receiver, &w, true) }; + assert_eq!( + unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), + Some(2.0) + ); - let old_relinks = read_accessor_same_shape_relinks(); - crate::object::test_seed_class_decl_prototype_object_root(CID, p2_addr); - assert_eq!(unsafe { object_shape_stamp(receiver) }, recv_shape); - assert!( - unsafe { try_cached_class_accessor(receiver, &mut slot) }.is_none(), - "stale getter was served after registry replacement" - ); - let second = unsafe { class_accessor_walk(receiver, b"path") }.expect("second accessor"); - let w = Walk { - holder: second.holder, - holder_shape: second.shape, - slot: Some(second.slot), - hops: [(0, 0); HOLDER_MAX_DEPTH - 1], - depth: 1, - }; - unsafe { publish(cache, receiver, &w, true) }; - assert!(read_accessor_same_shape_relinks() > old_relinks); - assert_eq!( - unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), - Some(8.0) - ); + let old_relinks = read_accessor_same_shape_relinks(); + p2.with_const_ptr::(|ptr| { + crate::object::test_seed_class_decl_prototype_object_root(CID, ptr as usize); + }); + assert_eq!(unsafe { object_shape_stamp(receiver) }, recv_shape); + assert!( + unsafe { try_cached_class_accessor(receiver, &mut slot) }.is_none(), + "stale getter was served after registry replacement" + ); + let second = + unsafe { class_accessor_walk(receiver, b"path") }.expect("second accessor"); + let w = Walk { + holder: second.holder, + holder_shape: second.shape, + slot: Some(second.slot), + hops: [(0, 0); HOLDER_MAX_DEPTH - 1], + depth: 1, + }; + unsafe { publish(cache, receiver, &w, true) }; + assert!(read_accessor_same_shape_relinks() > old_relinks); + assert_eq!( + unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), + Some(8.0) + ); + }); } #[test] From b7eea0a28ccd7bfb80e9d4108729b0436b19c5e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 14:28:37 +0000 Subject: [PATCH 21/40] Scope class read key confirmation after generic getter --- .../method_site/read_holder/class_read.rs | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs index e6a3234a6b..82a0bae582 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs @@ -177,39 +177,37 @@ pub(super) unsafe fn prime( let handle = scope.root_raw_mut_ptr(obj as *mut ObjectHeader); let key_handle = scope.root_string_ptr(key); let (value, obj) = handle.across_mut::(|| { - crate::object::field_get_set::get_field_by_name_after_site_miss( - obj, - key_handle.get_raw_const_ptr::(), - ) + crate::object::field_get_set::get_field_by_name_after_site_miss(obj, key) }); if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return Some(value); } - let key = key_handle.get_raw_const_ptr::(); - let name = crate::string::header_str_checked(key)?.as_bytes(); - let Some(obj) = ordinary_receiver(obj as usize) else { - return Some(value); - }; - let Some(w) = walk(obj, name, true) else { - return Some(value); - }; - let bits = value.bits(); - let confirmed = match w.slot { - None => bits == crate::value::TAG_UNDEFINED, - Some(slot) => { - bits == slot_bits(w.holder, slot) - && bits != crate::value::TAG_HOLE - && bits != crate::value::TAG_UNDEFINED - && bits != crate::value::TAG_NULL - } - }; - if confirmed { - let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); - if !cache.is_null() { - publish(cache, obj, &w); + key_handle.with_const_ptr::(|key| { + let name = crate::string::header_str_checked(key)?.as_bytes(); + let Some(obj) = ordinary_receiver(obj as usize) else { + return Some(value); + }; + let Some(w) = walk(obj, name, true) else { + return Some(value); + }; + let bits = value.bits(); + let confirmed = match w.slot { + None => bits == crate::value::TAG_UNDEFINED, + Some(slot) => { + bits == slot_bits(w.holder, slot) + && bits != crate::value::TAG_HOLE + && bits != crate::value::TAG_UNDEFINED + && bits != crate::value::TAG_NULL + } + }; + if confirmed { + let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); + if !cache.is_null() { + publish(cache, obj, &w); + } } - } - Some(value) + Some(value) + }) } unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk) { From 5beae695955444579850f655645d56b470345119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 14:45:17 +0000 Subject: [PATCH 22/40] perf: answer depth-one data holder before rare read kinds --- .../src/object/method_site/read_holder.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 8920ad70e5..5ec37d31b6 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -200,6 +200,16 @@ pub(crate) unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { return multi_absent_extra_answer(c, token); } let kind = c[HOLDER_KIND]; + // A depth-1 data holder is the common inherited-read hit. Its kind is + // exactly an inline slot number; answer it before the absent, accessor, + // multi-shape and deeper-hop decoding below. + if (kind as u64) <= u32::MAX as u64 { + let holder = c[HOLDER_OBJ] as usize; + if shape_word(holder) != c[HOLDER_SHAPE] as u32 { + return None; + } + return Some(slot_bits(holder, kind as u32)); + } if kind as u64 & (HOLDER_ACCESSOR | HOLDER_MULTI_ABSENT) != 0 { if kind as u64 & HOLDER_ACCESSOR != 0 || kind as u64 & HOLDER_ABSENT_BIT == 0 { return None; From 43582ee4d917321111612a83cbec1925caa92722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 14:58:20 +0000 Subject: [PATCH 23/40] perf: keep rare holder reads out of inline class-field hit --- .../src/object/method_site/read_holder.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 5ec37d31b6..17be80bc89 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -210,6 +210,14 @@ pub(crate) unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { } return Some(slot_bits(holder, kind as u32)); } + entry_answer_other(c, kind) +} + +/// Uncommon entries keep their complete depth/absence validation off the +/// inlined depth-1 data hit, including the class-field read's caller. +#[cold] +#[inline(never)] +unsafe fn entry_answer_other(c: &PicCache, kind: i64) -> Option { if kind as u64 & (HOLDER_ACCESSOR | HOLDER_MULTI_ABSENT) != 0 { if kind as u64 & HOLDER_ACCESSOR != 0 || kind as u64 & HOLDER_ABSENT_BIT == 0 { return None; @@ -302,7 +310,7 @@ fn set_multi_absent_id(c: &mut PicCache, i: usize, id: u32) { /// # Safety /// `handle` is a pointer above the handle band; `cache_slot` null or the /// site's live read cache slot. -#[inline] +#[inline(always)] pub(crate) unsafe fn read_holder_hit( handle: *const ObjectHeader, cache_slot: *mut PicCacheSlot, From 2323621aa145a95d7c5dc783ec7be78e4a8dba80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 15:11:59 +0000 Subject: [PATCH 24/40] perf: reuse holder shape proof on class getter hit --- .../src/object/method_site/read_holder.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 17be80bc89..f6787950b4 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -497,7 +497,7 @@ pub(crate) unsafe fn try_cached_class_read( /// Collecting read-miss arm. The GC-leaf front always declines this kind. /// Every hit confirms the receiver's shape and live link, the rooted holder's -/// shape, and the accessor descriptor before invoking with the ORIGINAL receiver. +/// shape, and the current accessor pair before invoking with the ORIGINAL receiver. pub(crate) unsafe fn try_cached_class_accessor( recv: *const ObjectHeader, cache_slot: *mut PicCacheSlot, @@ -528,17 +528,11 @@ pub(crate) unsafe fn try_cached_class_accessor( if shape_word(holder) != c[HOLDER_SHAPE] as u32 { return None; } + // The holder's unchanged ShapeId carries the prime-time proof that this + // inline slot exists and is an accessor. Key/attribute changes transition + // the ShapeId; a raw-only pair replacement may not, so reread the pair + // itself on every hit before invoking. let slot = c[HOLDER_KIND] as u32; - let shape = object_shape_descriptor(holder as *const ObjectHeader)?; - let keys = shape.keys as usize as *const crate::array::ArrayHeader; - if keys.is_null() - || slot >= shape.live_inline_slot_count - || crate::object::key_attrs::keys_entry(keys, slot) - & crate::object::key_attrs::ENTRY_ACCESSOR - == 0 - { - return None; - } let acc = crate::object::accessor_pair::pair_of_value(slot_bits(holder, slot))?; if acc.raw_get == 0 && acc.raw_set == 0 { return None; From 6de5642a5c0cb89169ceadd6e5e33de2aea38ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 15:01:38 +0000 Subject: [PATCH 25/40] Memoize direct class setters at packed PutValue sites --- crates/perry-runtime/src/gc/mod.rs | 1 + crates/perry-runtime/src/proxy.rs | 1 + crates/perry-runtime/src/proxy/put_value.rs | 1 + .../src/proxy/put_value/packed_set.rs | 13 +- .../src/proxy/put_value/setter_site.rs | 472 ++++++++++++++++++ scripts/gc_runtime_root_holders.json | 4 +- 6 files changed, 488 insertions(+), 4 deletions(-) create mode 100644 crates/perry-runtime/src/proxy/put_value/setter_site.rs diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index a6e294be76..ac263f992d 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -1071,6 +1071,7 @@ pub fn gc_init() { // key and the receiver's recorded prototype, and compares them on every // use, so both are STRONG roots (`object::chain_store`). reg_scanner!(crate::object::chain_store::scan_chain_store_roots_mut); + reg_scanner!(crate::proxy::scan_setter_site_roots_mut); // An inherited method-site entry roots its direct prototype holder. reg_scanner!(crate::object::method_site::scan_method_site_roots_mut); // A read site's holder entry names the object that holds the answer (and diff --git a/crates/perry-runtime/src/proxy.rs b/crates/perry-runtime/src/proxy.rs index 7d112c0e9e..5cc3fb4ff8 100644 --- a/crates/perry-runtime/src/proxy.rs +++ b/crates/perry-runtime/src/proxy.rs @@ -32,6 +32,7 @@ pub use has_delete::{js_proxy_delete, js_proxy_has}; mod invariants; mod put_value; pub(crate) use put_value::note_packed_add_carriers; +pub(crate) use put_value::scan_setter_site_roots_mut; pub use put_value::{js_proxy_set, js_put_value_set}; pub(crate) use put_value::{ js_put_value_set_ic_miss, proxy_set_with_receiver, IC_SLOT_OVERFLOW_BIT, diff --git a/crates/perry-runtime/src/proxy/put_value.rs b/crates/perry-runtime/src/proxy/put_value.rs index 44583fa067..326cc3d00f 100644 --- a/crates/perry-runtime/src/proxy/put_value.rs +++ b/crates/perry-runtime/src/proxy/put_value.rs @@ -403,6 +403,7 @@ pub use packed_add::PackedSetSite; pub(crate) use packed_add::{ census as store_census, C_REP_CONVERGE, C_REP_MIGRATE, C_REP_VALIDITY_BUMP, }; +pub(crate) use packed_set::scan_setter_site_roots_mut; pub use packed_set::{js_put_value_set_packed_miss, PACKED_SET_EMPTY}; pub(crate) use packed_set::{packed_set_cache_resolve, PackedSetWaysSlot, PACKED_SET_CHAIN_WORD}; diff --git a/crates/perry-runtime/src/proxy/put_value/packed_set.rs b/crates/perry-runtime/src/proxy/put_value/packed_set.rs index 58071df2c3..3736a63ce6 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_set.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_set.rs @@ -61,6 +61,9 @@ use std::sync::atomic::{AtomicU64, Ordering}; use super::*; +mod setter_site; +pub(crate) use setter_site::scan_roots as scan_setter_site_roots_mut; + /// The value `@perry_ic_N_packed_set` holds before its first prime. /// /// **Must equal `PACKED_SET_EMPTY` in @@ -91,15 +94,18 @@ pub const PACKED_SET_INLINE_WAYS: usize = 4; /// (`object::chain_store`), 0 until the site primes one. Never compared by /// the emitted code, which reads only ways `0..PACKED_SET_INLINE_WAYS`. pub const PACKED_SET_CHAIN_WORD: usize = PACKED_SET_WAYS; +/// Collecting-only direct class setter memo; emitted code never reads it. +pub const PACKED_SET_SETTER_WORD: usize = PACKED_SET_WAYS + 1; /// A site's way cache: packed words in the compact word's format, then the /// chain entry word. -pub type PackedSetWays = [u64; PACKED_SET_WAYS + 1]; +pub type PackedSetWays = [u64; PACKED_SET_WAYS + 2]; /// A site cache no prime has touched: every way empty, no chain entry. pub const fn packed_set_cache_empty() -> PackedSetWays { - let mut cache = [PACKED_SET_EMPTY; PACKED_SET_WAYS + 1]; + let mut cache = [PACKED_SET_EMPTY; PACKED_SET_WAYS + 2]; cache[PACKED_SET_CHAIN_WORD] = 0; + cache[PACKED_SET_SETTER_WORD] = 0; cache } @@ -196,6 +202,9 @@ pub extern "C" fn js_put_value_set_packed_miss( return stored; } } + if let Some(stored) = unsafe { setter_site::try_set(cache_slot, target, key, value) } { + return stored; + } // The receiver's ShapeId before the store: the pre-shape a key-add memo // is keyed on. Allocation-free. let pre_shape = unsafe { crate::object::chain_store::pre_store_shape(target) }; diff --git a/crates/perry-runtime/src/proxy/put_value/setter_site.rs b/crates/perry-runtime/src/proxy/put_value/setter_site.rs new file mode 100644 index 0000000000..8a429e5aab --- /dev/null +++ b/crates/perry-runtime/src/proxy/put_value/setter_site.rs @@ -0,0 +1,472 @@ +//! One direct compiled class setter at a static-key PutValue site. +//! +//! The packed store's first eight words are own-data ways and word eight is +//! the key-add chain verdict. Word nine names this bounded, collecting-path +//! setter entry. The emitted leaf never reads it. A miss validates the live +//! class-prototype link, receiver and holder shapes, inline accessor slot and +//! raw setter before calling with the original receiver. Any uncertainty +//! falls through to ordinary `[[Set]]`. + +use super::*; +use std::sync::atomic::{AtomicU64, Ordering}; + +const SITE_TAG: u64 = 0xA2C2_0000_0000_0000; + +struct Entry { + key: usize, + holder: usize, + class_id: u32, + receiver_shape: u32, + holder_shape: u32, + slot: u32, + raw_set: usize, + validity: u64, + vtable_gen: u64, +} + +crate::perry_thread_local! { + static ENTRIES: std::cell::UnsafeCell> = + const { std::cell::UnsafeCell::new(Vec::new()) }; +} + +static HITS: AtomicU64 = AtomicU64::new(0); +static PRIMES: AtomicU64 = AtomicU64::new(0); +static ROOT_REWRITES: AtomicU64 = AtomicU64::new(0); + +fn stats_enabled() -> bool { + #[cfg(test)] + { + true + } + #[cfg(not(test))] + { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + let on = std::env::var_os("PERRY_SETTER_SITE_STATS").is_some(); + if on { + extern "C" fn report() { + eprintln!( + "[setter-site] primes={} hits={} root_rewrites={}", + PRIMES.load(Ordering::Relaxed), + HITS.load(Ordering::Relaxed), + ROOT_REWRITES.load(Ordering::Relaxed), + ); + } + unsafe { libc::atexit(report) }; + } + on + }) + } +} + +#[inline] +fn primary_only() -> bool { + crate::object::method_site::WORKER_AGENTS_EXIST.load(Ordering::SeqCst) == 0 + && crate::agent::current_agent() == crate::agent::PRIMARY_AGENT +} + +unsafe fn entry(slot: *mut PackedSetWaysSlot) -> Option<&'static mut Entry> { + if slot.is_null() { + return None; + } + let cache = crate::object::pic_slot_peek(slot); + if cache.is_null() { + return None; + } + let word = (*cache)[PACKED_SET_SETTER_WORD]; + if word & !crate::value::POINTER_MASK != SITE_TAG { + return None; + } + let ptr = (word & crate::value::POINTER_MASK) as usize as *mut Entry; + (!ptr.is_null()).then_some(&mut *ptr) +} + +unsafe fn class_link(recv: *const crate::ObjectHeader) -> Option<*const crate::ObjectHeader> { + use crate::object::shapes::{ + object_proto_id, object_shape_stamp, shape_proto_id, PROTO_ID_CLASS, PROTO_ID_MIXED, + PROTO_ID_UNIQUE, + }; + let pid = shape_proto_id(object_shape_stamp(recv))?; + if object_proto_id(recv) != pid { + return None; + } + let holder = if (PROTO_ID_MIXED..PROTO_ID_UNIQUE).contains(&pid) { + let meta = (*recv).meta; + if meta.is_null() { + return None; + } + let p = crate::JSValue::from_bits((*meta).prototype); + if !p.is_pointer() { + return None; + } + p.as_pointer::() + } else if (PROTO_ID_CLASS..PROTO_ID_MIXED).contains(&pid) { + crate::object::class_decl_prototype_object((*recv).class_id) + } else { + return None; + }; + (!holder.is_null() && holder != recv).then_some(holder) +} + +unsafe fn candidate( + recv: *const crate::ObjectHeader, + key: *const crate::StringHeader, +) -> Option { + if key.is_null() || !crate::value::addr_class::is_above_handle_band(key as usize) { + return None; + } + let key_gc = crate::value::addr_class::try_read_gc_header(key as usize)?; + if key_gc.obj_type != crate::gc::GC_TYPE_STRING + || key_gc.gc_flags & (crate::gc::GC_FLAG_FORWARDED | crate::gc::GC_FLAG_INTERNED) + != crate::gc::GC_FLAG_INTERNED + { + return None; + } + let name = crate::string::header_str_checked(key)?.as_bytes(); + if name.is_empty() || name[0] == b'#' || name[0].is_ascii_digit() { + return None; + } + let gc = crate::value::addr_class::try_read_gc_header(recv as usize)?; + if gc.obj_type != crate::gc::GC_TYPE_OBJECT + || gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || gc._reserved & crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO != 0 + || crate::object::dictionary::is_dictionary(recv) + { + return None; + } + let class_id = (*recv).class_id; + if class_id == 0 + || class_id == crate::object::NATIVE_MODULE_CLASS_ID + || crate::object::is_anon_shape_class_id(class_id) + { + return None; + } + let meta = (*recv).meta; + if !meta.is_null() + && ((*meta).elements != 0 + || (*meta).flags & crate::object::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER != 0) + { + return None; + } + let recv_shape = crate::object::shapes::object_shape_descriptor(recv)?; + if !recv_shape.object_kind.is_ordinary_layout() { + return None; + } + let recv_keys = recv_shape.keys as usize as *const crate::array::ArrayHeader; + if !recv_keys.is_null() + && crate::object::keys_find_slot_by_bytes_resolved( + recv_keys, + recv_shape.logical_key_count, + name, + ) + .is_some() + { + return None; + } + + let holder = class_link(recv)?; + let holder_gc = crate::value::addr_class::try_read_gc_header(holder as usize)?; + if holder_gc.obj_type != crate::gc::GC_TYPE_OBJECT + || holder_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || crate::object::dictionary::is_dictionary(holder) + { + return None; + } + let shape = crate::object::shapes::object_shape_descriptor(holder)?; + if !shape.object_kind.is_ordinary_layout() { + return None; + } + let keys = shape.keys as usize as *const crate::array::ArrayHeader; + if keys.is_null() { + return None; + } + let slot = + crate::object::keys_find_slot_by_bytes_resolved(keys, shape.logical_key_count, name)?; + if slot >= shape.live_inline_slot_count + || crate::object::key_attrs::keys_entry(keys, slot) + & crate::object::key_attrs::ENTRY_ACCESSOR + == 0 + { + return None; + } + let field = (holder as *const u8) + .add(std::mem::size_of::() + slot as usize * 8) + as *const u64; + let acc = crate::object::accessor_pair::pair_of_value(*field)?; + if acc.raw_set == 0 { + return None; + } + // The direct declared pair is the only admitted route. A registered + // ancestor or a closure-backed replacement keeps the generic walk. + let name_str = std::str::from_utf8(name).ok()?; + if !crate::object::class_chain_has_instance_accessor(class_id, name_str) { + return None; + } + Some(Entry { + key: key as usize, + holder: holder as usize, + class_id, + receiver_shape: crate::object::shapes::object_shape_stamp(recv), + holder_shape: crate::object::shapes::object_shape_stamp(holder), + slot, + raw_set: acc.raw_set, + validity: crate::object::proto_validity::proto_validity(), + vtable_gen: crate::object::vtable_generation(), + }) +} + +unsafe fn validated_raw_set( + e: &Entry, + recv: *const crate::ObjectHeader, + key: *const crate::StringHeader, +) -> Option { + let gc = crate::value::addr_class::try_read_gc_header(recv as usize)?; + if gc.obj_type != crate::gc::GC_TYPE_OBJECT + || gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + || gc._reserved & crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO != 0 + || crate::object::dictionary::is_dictionary(recv) + { + return None; + } + let meta = (*recv).meta; + if !meta.is_null() + && ((*meta).elements != 0 + || (*meta).flags & crate::object::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER != 0) + { + return None; + } + if e.key != key as usize + || e.class_id != (*recv).class_id + || e.receiver_shape != crate::object::shapes::object_shape_stamp(recv) + || e.validity != crate::object::proto_validity::proto_validity() + || e.vtable_gen != crate::object::vtable_generation() + || class_link(recv)? as usize != e.holder + || crate::object::shapes::object_shape_stamp(e.holder as *const crate::ObjectHeader) + != e.holder_shape + { + return None; + } + let holder_gc = crate::value::addr_class::try_read_gc_header(e.holder)?; + if holder_gc.obj_type != crate::gc::GC_TYPE_OBJECT + || holder_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + { + return None; + } + let field = (e.holder as *const u8) + .add(std::mem::size_of::() + e.slot as usize * 8) + as *const u64; + let acc = crate::object::accessor_pair::pair_of_value(*field)?; + (acc.raw_set == e.raw_set && acc.raw_set != 0).then_some(acc.raw_set) +} + +unsafe fn invoke(raw_set: usize, target: f64, value: f64) -> f64 { + let scope = crate::gc::RuntimeHandleScope::new(); + let recv_h = scope.root_nanbox_f64(target); + let value_h = scope.root_nanbox_f64(value); + let f = crate::closure::body_call::js_method_body_fn!(raw_set as *const u8; value); + let _ = f(recv_h.get_nanbox_f64(), value_h.get_nanbox_f64()); + value_h.get_nanbox_f64() +} + +/// Collecting miss only; the emitted GC-leaf store never consults this word. +pub(super) unsafe fn try_set( + slot: *mut PackedSetWaysSlot, + target: f64, + key: *const crate::StringHeader, + value: f64, +) -> Option { + if !primary_only() || slot.is_null() || key.is_null() { + return None; + } + let bits = target.to_bits(); + if bits & !crate::value::POINTER_MASK != crate::value::POINTER_TAG { + return None; + } + let addr = (bits & crate::value::POINTER_MASK) as usize; + if !crate::value::addr_class::is_above_handle_band(addr) { + return None; + } + let recv = addr as *const crate::ObjectHeader; + if let Some(e) = entry(slot) { + if let Some(raw_set) = validated_raw_set(e, recv, key) { + if stats_enabled() { + HITS.fetch_add(1, Ordering::Relaxed); + } + return Some(invoke(raw_set, target, value)); + } + } + let fresh = candidate(recv, key)?; + let raw_set = fresh.raw_set; + let cache = packed_set_cache_resolve(slot); + if !cache.is_null() { + if let Some(e) = entry(slot) { + *e = fresh; + } else { + let ptr = Box::into_raw(Box::new(fresh)); + ENTRIES.with(|cell| (*cell.get()).push(ptr)); + let addr = ptr as usize as u64; + assert_eq!(addr & !crate::value::POINTER_MASK, 0); + (*cache)[PACKED_SET_SETTER_WORD] = SITE_TAG | addr; + } + if stats_enabled() { + PRIMES.fetch_add(1, Ordering::Relaxed); + } + } + Some(invoke(raw_set, target, value)) +} + +pub(crate) fn scan_roots(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + if !primary_only() { + return; + } + ENTRIES.with(|cell| unsafe { + for &ptr in (*cell.get()).iter() { + let e = &mut *ptr; + if visitor.visit_tagged_usize_slot(&mut e.key, crate::value::STRING_TAG) { + ROOT_REWRITES.fetch_add(1, Ordering::Relaxed); + } + if visitor.visit_tagged_usize_slot(&mut e.holder, crate::value::POINTER_TAG) { + ROOT_REWRITES.fetch_add(1, Ordering::Relaxed); + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicU32, Ordering}; + + static FIRST: AtomicU32 = AtomicU32::new(0); + static SECOND: AtomicU32 = AtomicU32::new(0); + + extern "C" fn first(_recv: f64, _value: f64) -> f64 { + FIRST.fetch_add(1, Ordering::Relaxed); + 0.0 + } + extern "C" fn second(_recv: f64, _value: f64) -> f64 { + SECOND.fetch_add(1, Ordering::Relaxed); + 0.0 + } + fn fnv1a(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325u64, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) + } + + #[test] + fn direct_setter_rechecks_same_shape_relink_own_shadow_and_worker_gate() { + let _lock = crate::gc::global_side_table_test_lock(); + const CID: u32 = 0x0C3C_79B5; + let before_gate = crate::object::method_site::WORKER_AGENTS_EXIST.swap(0, Ordering::SeqCst); + FIRST.store(0, Ordering::Relaxed); + SECOND.store(0, Ordering::Relaxed); + unsafe { + crate::object::js_register_class_id(CID); + crate::object::js_register_class_setter( + CID as i64, + b"points".as_ptr(), + 6, + first as *const () as i64, + 1, + ); + } + let scope = crate::gc::RuntimeHandleScope::new(); + let p1 = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 2)); + let p2 = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 2)); + for (holder, raw_set) in [ + (&p1, first as *const () as usize), + (&p2, second as *const () as usize), + ] { + holder.with_mut_ptr::(|ptr| { + crate::object::set_builtin_accessor_pair( + ptr as usize, + "points".to_owned(), + crate::object::accessor_pair::Accessor { + raw_set, + ..Default::default() + }, + crate::object::PropertyAttrs::new(true, false, true), + ); + }); + } + let shape1 = p1.with_const_ptr::(|ptr| unsafe { + crate::object::shapes::object_shape_stamp(ptr) + }); + let shape2 = p2.with_const_ptr::(|ptr| unsafe { + crate::object::shapes::object_shape_stamp(ptr) + }); + assert_eq!(shape1, shape2); + let packed = b"own"; + let keys = crate::object::js_build_class_keys_array( + CID, + 1, + packed.as_ptr(), + packed.len() as u32, + 0, + ); + let recv_shape = crate::object::shapes::js_object_shape_id_for_class_keys( + keys as usize as u64, + 1, + CID, + 0, + ); + let recv = + scope.root_raw_mut_ptr(crate::object::js_object_alloc_class_inline_keys_stamped( + CID, 0, 1, keys, recv_shape, 0, + )); + let key_raw = crate::string::js_string_from_bytes(b"points".as_ptr(), 6); + let key = scope.root_string_ptr(crate::string::js_string_intern(key_raw, fnv1a(b"points"))); + p1.with_const_ptr::(|p| { + crate::object::test_seed_class_decl_prototype_object_root(CID, p as usize) + }); + let cache: &'static mut PackedSetWays = Box::leak(Box::new(packed_set_cache_empty())); + assert_eq!(cache[PACKED_SET_SETTER_WORD], 0); + let mut slot: PackedSetWaysSlot = cache; + let target = recv.with_const_ptr::(|p| { + crate::value::js_nanbox_pointer(p as i64) + }); + let key_ptr = key.get_raw_const_ptr::(); + assert_eq!( + unsafe { try_set(&mut slot, target, key_ptr, 5.0) }, + Some(5.0) + ); + assert_eq!( + unsafe { try_set(&mut slot, target, key_ptr, 6.0) }, + Some(6.0) + ); + assert_eq!(FIRST.load(Ordering::Relaxed), 2); + p2.with_const_ptr::(|p| { + crate::object::test_seed_class_decl_prototype_object_root(CID, p as usize) + }); + assert_eq!( + unsafe { + validated_raw_set(entry(&mut slot).unwrap(), recv.get_raw_const_ptr(), key_ptr) + }, + None + ); + assert_eq!( + unsafe { try_set(&mut slot, target, key_ptr, 7.0) }, + Some(7.0) + ); + assert_eq!(SECOND.load(Ordering::Relaxed), 1); + crate::object::method_site::WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); + assert_eq!(unsafe { try_set(&mut slot, target, key_ptr, 8.0) }, None); + crate::object::method_site::WORKER_AGENTS_EXIST.store(before_gate, Ordering::SeqCst); + recv.with_mut_ptr::(|p| unsafe { + crate::object::object_ops::define_property_force_store_value(p, key_ptr, 99.0); + }); + assert_ne!( + recv.with_const_ptr::(|p| unsafe { + crate::object::shapes::object_shape_stamp(p) + }), + recv_shape + ); + assert_eq!( + unsafe { + validated_raw_set(entry(&mut slot).unwrap(), recv.get_raw_const_ptr(), key_ptr) + }, + None + ); + } +} diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 002a3453b8..5bfddf270d 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -350,7 +350,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete \u2192 sweep-entry window of a synchronous full \u2014 where PASS1_MARKED is populated and consumed within one `run_to_completion` \u2014 is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize \u2014 INSIDE the window \u2014 the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes \u2014 in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 (charter step 5, P0): the field-representation census adds `mod census_field_repr;` to `gc/mod.rs` and, in `census.rs`, one Rust-owned accumulator fed from `visit_object` inside `take_census`, which runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED; it reads slots and the per-object layout maps (try_borrow), allocates nothing on the JS heap and alters no mark/sweep control flow. The window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 (class constructors as function objects, #11414): `gc/mod.rs` gains one `reg_scanner!` registration (`object::class_value::scan_class_value_roots_mut`, the per-agent class function-object table, visited and rewritten) \u2014 a root-scanner registration that alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 after #11659 added `verify::verify_array_hole_tails_at_collection()` to `gc/mod.rs` and `gc/policy.rs`. Every call sits at a collection or budgeted-cycle ENTRY, beside `roots::ensure_stack_maps_built()`, i.e. before mark begins and so before `census_pass1_if_armed` opens the window. The call is a read-only walk of array headers (debug builds, or release with PERRY_GC_VERIFY_ARRAY_HOLES): it neither allocates GC memory, relocates, runs JS, nor reads or writes PASS1_MARKED, and it panics rather than continuing on a violation. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Pin re-audited 2026-09-29 after PR #11646 added one call inside that window, in `cycle.rs` beside `census_take_if_armed_at_full_sweep_start`: `object::shapes::store_kind::audit_heap_at_full_sweep_start()`. It compiles to nothing unless the `shape-fact-audit` feature is on; when on it walks the arena read-only (`gc::for_each_live_object_at_sweep_start`, in `gc/verify.rs`) and reads each marked object header and its shape record. It allocates no GC object, moves nothing, runs no JS callback and holds no address past the walk, so PASS1_MARKED stays valid across it; a disagreement panics rather than continuing. Re-audited 2026-09-29 after the pinned-roots fix touched `gc/cycle.rs` and `gc/mod.rs`: the block-persistence live-block predicate now reads GC_FLAG_MARKED alone (pinned objects are marked as roots, so the set of live blocks is unchanged), and `gc/mod.rs` adds one `reg_scanner!` registration. Neither relocates, runs JS, or moves the mark-complete to sweep-entry window. Re-audited again 2026-09-29 (same fix, copying-minor follow-up): `BlockPersistCycleState`'s force-mark in `gc/cycle.rs` no longer skips an unmarked pinned header, so it marks and pushes it like any other object of a live block. That is marking inside the budgeted cycle's persistence step: nothing relocates, no JS runs, and the budgeted path still skips both census boundaries. The window is unchanged. Re-audited 2026-09-29 for this-as-a-parameter stage 3: `gc/mod.rs` registers `scan_dispatch_binding_roots_mut` in place of `scan_implicit_this_roots_mut` (the implicit-`this` cell is deleted; the same scanner body keeps `new.target`, the static-`this` override and the static private-owner stack) and rewords its comment; a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 for the JsFunctionInfo lane: `gc/census.rs`'s `side_tables()` loses its `closure_registry_census()` row (the closure-body registry is deleted); a census report row, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 for the read-site holder entry: `gc/mod.rs` adds one `reg_scanner!` registration, `read_holder::scan_read_holder_roots_mut`, which visits the holder and hop words of registered read-site caches during ROOT SCAN, before mark propagation completes. It rewrites root slots only through the visitor (as every registered scanner does), runs no JS, and nothing it does executes between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. The window is unchanged. Re-audited 2026-09-30 for Step 5 P4: census.rs only removes the typed-layout count, and gc/mod.rs removes shape-install registration/re-export and an init-time typed-intact verifier gate. The snapshot remains stored after mark propagation and taken at sweep entry; none of these edits relocates objects or invokes callbacks in that interval. Re-audited 2026-09-29 (#11549 trace cost): `gc/mod.rs` gains one `mod copying_object_scan;` declaration, the copying minor's plain-object slot walk. It is reached only from the copying drain, never from a full cycle, and changes no mark or sweep control flow. The window is unchanged. Re-audited 2026-09-30 after rebasing P4 over #11549/#11676: gc/mod.rs retains the copying_object_scan declaration and P4 removes typed-layout registration; both are module wiring outside the synchronous full mark-complete to sweep-entry window. The census boundaries and the window owner in cycle.rs are unchanged. Re-audited for A2 inherited-read-cache deletion: gc/mod.rs removes only its old root-scanner registration and adjacent comments. The method-site and read-holder scanners still run during root scan before mark completion; no collection phase or callback was added inside the mark-complete to sweep-entry window. Re-audited 2026-09-30 for the scope-context change: gc/mod.rs also removes the box root-scanner registration and exit-time box statistics. The former ran during root scan before mark completion, and the latter at process exit. Neither changes the synchronous mark-complete to sweep-entry window.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete \u2192 sweep-entry window of a synchronous full \u2014 where PASS1_MARKED is populated and consumed within one `run_to_completion` \u2014 is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize \u2014 INSIDE the window \u2014 the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes \u2014 in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 (charter step 5, P0): the field-representation census adds `mod census_field_repr;` to `gc/mod.rs` and, in `census.rs`, one Rust-owned accumulator fed from `visit_object` inside `take_census`, which runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED; it reads slots and the per-object layout maps (try_borrow), allocates nothing on the JS heap and alters no mark/sweep control flow. The window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 (class constructors as function objects, #11414): `gc/mod.rs` gains one `reg_scanner!` registration (`object::class_value::scan_class_value_roots_mut`, the per-agent class function-object table, visited and rewritten) \u2014 a root-scanner registration that alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 after #11659 added `verify::verify_array_hole_tails_at_collection()` to `gc/mod.rs` and `gc/policy.rs`. Every call sits at a collection or budgeted-cycle ENTRY, beside `roots::ensure_stack_maps_built()`, i.e. before mark begins and so before `census_pass1_if_armed` opens the window. The call is a read-only walk of array headers (debug builds, or release with PERRY_GC_VERIFY_ARRAY_HOLES): it neither allocates GC memory, relocates, runs JS, nor reads or writes PASS1_MARKED, and it panics rather than continuing on a violation. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Pin re-audited 2026-09-29 after PR #11646 added one call inside that window, in `cycle.rs` beside `census_take_if_armed_at_full_sweep_start`: `object::shapes::store_kind::audit_heap_at_full_sweep_start()`. It compiles to nothing unless the `shape-fact-audit` feature is on; when on it walks the arena read-only (`gc::for_each_live_object_at_sweep_start`, in `gc/verify.rs`) and reads each marked object header and its shape record. It allocates no GC object, moves nothing, runs no JS callback and holds no address past the walk, so PASS1_MARKED stays valid across it; a disagreement panics rather than continuing. Re-audited 2026-09-29 after the pinned-roots fix touched `gc/cycle.rs` and `gc/mod.rs`: the block-persistence live-block predicate now reads GC_FLAG_MARKED alone (pinned objects are marked as roots, so the set of live blocks is unchanged), and `gc/mod.rs` adds one `reg_scanner!` registration. Neither relocates, runs JS, or moves the mark-complete to sweep-entry window. Re-audited again 2026-09-29 (same fix, copying-minor follow-up): `BlockPersistCycleState`'s force-mark in `gc/cycle.rs` no longer skips an unmarked pinned header, so it marks and pushes it like any other object of a live block. That is marking inside the budgeted cycle's persistence step: nothing relocates, no JS runs, and the budgeted path still skips both census boundaries. The window is unchanged. Re-audited 2026-09-29 for this-as-a-parameter stage 3: `gc/mod.rs` registers `scan_dispatch_binding_roots_mut` in place of `scan_implicit_this_roots_mut` (the implicit-`this` cell is deleted; the same scanner body keeps `new.target`, the static-`this` override and the static private-owner stack) and rewords its comment; a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 for the JsFunctionInfo lane: `gc/census.rs`'s `side_tables()` loses its `closure_registry_census()` row (the closure-body registry is deleted); a census report row, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 for the read-site holder entry: `gc/mod.rs` adds one `reg_scanner!` registration, `read_holder::scan_read_holder_roots_mut`, which visits the holder and hop words of registered read-site caches during ROOT SCAN, before mark propagation completes. It rewrites root slots only through the visitor (as every registered scanner does), runs no JS, and nothing it does executes between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. The window is unchanged. Re-audited 2026-09-30 for Step 5 P4: census.rs only removes the typed-layout count, and gc/mod.rs removes shape-install registration/re-export and an init-time typed-intact verifier gate. The snapshot remains stored after mark propagation and taken at sweep entry; none of these edits relocates objects or invokes callbacks in that interval. Re-audited 2026-09-29 (#11549 trace cost): `gc/mod.rs` gains one `mod copying_object_scan;` declaration, the copying minor's plain-object slot walk. It is reached only from the copying drain, never from a full cycle, and changes no mark or sweep control flow. The window is unchanged. Re-audited 2026-09-30 after rebasing P4 over #11549/#11676: gc/mod.rs retains the copying_object_scan declaration and P4 removes typed-layout registration; both are module wiring outside the synchronous full mark-complete to sweep-entry window. The census boundaries and the window owner in cycle.rs are unchanged. Re-audited for A2 inherited-read-cache deletion: gc/mod.rs removes only its old root-scanner registration and adjacent comments. The method-site and read-holder scanners still run during root scan before mark completion; no collection phase or callback was added inside the mark-complete to sweep-entry window. Re-audited 2026-09-30 for the setter-site root scanner: gc/mod.rs adds only its reg_scanner! registration in the root-scan setup. That scanner visits per-site key and holder slots before mark propagation completes; it runs no JS and adds no relocation or callback between census pass1 and sweep entry. The synchronous-full snapshot window and both boundaries remain unchanged. Re-audited 2026-09-30 for the scope-context change: gc/mod.rs also removes the box root-scanner registration and exit-time box statistics. The former ran during root scan before mark completion, and the latter at process exit. Neither changes the synchronous mark-complete to sweep-entry window.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -367,7 +367,7 @@ "sources": { "crates/perry-runtime/src/gc/census.rs": "4a611bfe5615559642b0e6e1eaf0f25440c5e228db5302d67dba43e577e0a1b6", "crates/perry-runtime/src/gc/cycle.rs": "9b992d27700382c84758f6f4a5574115a9862234bed2e18e0c5a3cca25c25236", - "crates/perry-runtime/src/gc/mod.rs": "5314f692ce92c67a659e88709e9a8ff253ab375d388082821f961a6a3c188f03", + "crates/perry-runtime/src/gc/mod.rs": "dd9761bc38694444bf4839c5829c328828a465548d08cae4b256002972a4bd1f", "crates/perry-runtime/src/gc/policy.rs": "84a869e0aa09e932a4d2b6601186260129ba851b4482f29579047713c60fc95e", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } From 9e4c103bc6909d1cb96656de1f5fc7e4bdc35475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 15:08:08 +0000 Subject: [PATCH 26/40] Test inherited setter site across evacuation and real worker gate --- tests/fixtures/one_shape_setter_site/check.sh | 34 +++++++++++++ .../one_shape_setter_site/expected-worker.txt | 5 ++ .../one_shape_setter_site/expected.txt | 3 ++ tests/fixtures/one_shape_setter_site/main.ts | 50 +++++++++++++++++++ .../fixtures/one_shape_setter_site/worker.cjs | 18 +++++++ 5 files changed, 110 insertions(+) create mode 100755 tests/fixtures/one_shape_setter_site/check.sh create mode 100644 tests/fixtures/one_shape_setter_site/expected-worker.txt create mode 100644 tests/fixtures/one_shape_setter_site/expected.txt create mode 100644 tests/fixtures/one_shape_setter_site/main.ts create mode 100644 tests/fixtures/one_shape_setter_site/worker.cjs diff --git a/tests/fixtures/one_shape_setter_site/check.sh b/tests/fixtures/one_shape_setter_site/check.sh new file mode 100755 index 0000000000..4de729ed6f --- /dev/null +++ b/tests/fixtures/one_shape_setter_site/check.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail +binary=$(realpath "${1:?pass the compiled fixture executable}") +fixture_dir=$(cd "$(dirname "$0")" && pwd) +repo_root=$(cd "$fixture_dir/../../.." && pwd) +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT +cd "$repo_root" +PERRY_SETTER_SITE_STATS=1 PERRY_GC_FORCE_EVACUATE=1 \ + PERRY_GC_VERIFY_EVACUATION=1 PERRY_GC_DIAG=1 "$binary" \ + > "$tmp_dir/control.out" 2> "$tmp_dir/control.err" +A2_START_WORKER=1 PERRY_SETTER_SITE_STATS=1 PERRY_GC_FORCE_EVACUATE=1 \ + PERRY_GC_VERIFY_EVACUATION=1 PERRY_GC_DIAG=1 "$binary" \ + > "$tmp_dir/worker.out" 2> "$tmp_dir/worker.err" +cmp "$fixture_dir/expected.txt" "$tmp_dir/control.out" +cmp "$fixture_dir/expected-worker.txt" "$tmp_dir/worker.out" +sum_counter() { + sed -n "s/.*$2=\\([0-9][0-9]*\\).*/\\1/p" "$1" | + awk '{ sum += $1 } END { print sum + 0 }' +} +control_primes=$(sum_counter "$tmp_dir/control.err" primes) +control_hits=$(sum_counter "$tmp_dir/control.err" hits) +worker_hits=$(sum_counter "$tmp_dir/worker.err" hits) +control_rewrites=$(sum_counter "$tmp_dir/control.err" root_rewrites) +control_copied=$(sum_counter "$tmp_dir/control.err" copied_objects) +worker_copied=$(sum_counter "$tmp_dir/worker.err" copied_objects) +test "$control_primes" -gt 0 +test "$worker_hits" -gt 0 +test "$control_hits" -gt "$worker_hits" +test "$control_rewrites" -gt 0 +test "$control_copied" -gt 0 +test "$worker_copied" -gt 0 +printf 'setter primes=%s hits control=%s worker=%s; root_rewrites=%s; copied_objects control=%s worker=%s\n' \ + "$control_primes" "$control_hits" "$worker_hits" "$control_rewrites" "$control_copied" "$worker_copied" diff --git a/tests/fixtures/one_shape_setter_site/expected-worker.txt b/tests/fixtures/one_shape_setter_site/expected-worker.txt new file mode 100644 index 0000000000..d23dbd9de0 --- /dev/null +++ b/tests/fixtures/one_shape_setter_site/expected-worker.txt @@ -0,0 +1,5 @@ +before 9 1200 +after 2 2400 +after-again 12 3600 +worker 13 1000 +worker-exit 1 diff --git a/tests/fixtures/one_shape_setter_site/expected.txt b/tests/fixtures/one_shape_setter_site/expected.txt new file mode 100644 index 0000000000..d6b3d9260b --- /dev/null +++ b/tests/fixtures/one_shape_setter_site/expected.txt @@ -0,0 +1,3 @@ +before 9 1200 +after 2 2400 +after-again 12 3600 diff --git a/tests/fixtures/one_shape_setter_site/main.ts b/tests/fixtures/one_shape_setter_site/main.ts new file mode 100644 index 0000000000..30d5d8d283 --- /dev/null +++ b/tests/fixtures/one_shape_setter_site/main.ts @@ -0,0 +1,50 @@ +import { Worker } from "node:worker_threads"; + +class Meter { + value = 0; + writes = 0; + get points(): number { return this.value; } + set points(next: number) { this.value = next; this.writes++; } +} + +const meter: any = new Meter(); +function write(target: any, next: number): void { target.points = next; } +function run(start: number): void { + for (let i = start; i < start + 1200; i++) write(meter, i % 17); +} +function collect(): void { + let churn: any[] = []; + for (let i = 0; i < 20000; i++) churn.push({ i }); + (globalThis as any).gc(); + churn = []; +} +function report(label: string): void { + console.log(label, meter.points, meter.writes); +} + +run(0); +report("before"); +if (process.env.A2_START_WORKER !== "1") { + collect(); + run(1200); + report("after"); + run(2400); + report("after-again"); +} else { + process.chdir("tests/fixtures/one_shape_setter_site"); + const worker = new Worker("./worker.cjs"); + worker.on("message", (message: any) => { + if (message === "ready") { + collect(); + run(1200); + report("after"); + run(2400); + report("after-again"); + worker.postMessage("go"); + } else { + console.log("worker", message); + worker.terminate(); + } + }); + worker.on("exit", (code: number) => console.log("worker-exit", code)); +} diff --git a/tests/fixtures/one_shape_setter_site/worker.cjs b/tests/fixtures/one_shape_setter_site/worker.cjs new file mode 100644 index 0000000000..65e4656f93 --- /dev/null +++ b/tests/fixtures/one_shape_setter_site/worker.cjs @@ -0,0 +1,18 @@ +const { parentPort } = require("node:worker_threads"); +class Meter { + constructor() { this.value = 0; this.writes = 0; } + get points() { return this.value; } + set points(next) { this.value = next; this.writes++; } +} +const meter = new Meter(); +function write(target, next) { target.points = next; } +parentPort.on("message", (message) => { + if (message !== "go") return; + let churn = []; + for (let i = 0; i < 20000; i++) churn.push({ i }); + global.gc(); + churn = []; + for (let i = 0; i < 1000; i++) write(meter, i % 17); + parentPort.postMessage(`${meter.points} ${meter.writes}`); +}); +parentPort.postMessage("ready"); From 93afad36b43320793634ffb095b0e729d9ccc728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 15:12:00 +0000 Subject: [PATCH 27/40] Restrict setter memo to store-admitted receiver shapes --- crates/perry-runtime/src/proxy/put_value/setter_site.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/proxy/put_value/setter_site.rs b/crates/perry-runtime/src/proxy/put_value/setter_site.rs index 8a429e5aab..0d5b4d0fbd 100644 --- a/crates/perry-runtime/src/proxy/put_value/setter_site.rs +++ b/crates/perry-runtime/src/proxy/put_value/setter_site.rs @@ -149,7 +149,11 @@ unsafe fn candidate( return None; } let recv_shape = crate::object::shapes::object_shape_descriptor(recv)?; - if !recv_shape.object_kind.is_ordinary_layout() { + if !matches!( + recv_shape.object_kind, + crate::object::shapes::ShapeObjectKind::Ordinary + | crate::object::shapes::ShapeObjectKind::OrdinaryNumericProof + ) { return None; } let recv_keys = recv_shape.keys as usize as *const crate::array::ArrayHeader; From 4b5d947bc323f4a40bb68e6b513e94aea92a69bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 15:24:08 +0000 Subject: [PATCH 28/40] Give bundled setter worker a distinct class name --- tests/fixtures/one_shape_setter_site/worker.cjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/one_shape_setter_site/worker.cjs b/tests/fixtures/one_shape_setter_site/worker.cjs index 65e4656f93..a1860df327 100644 --- a/tests/fixtures/one_shape_setter_site/worker.cjs +++ b/tests/fixtures/one_shape_setter_site/worker.cjs @@ -1,10 +1,10 @@ const { parentPort } = require("node:worker_threads"); -class Meter { +class WorkerMeter { constructor() { this.value = 0; this.writes = 0; } get points() { return this.value; } set points(next) { this.value = next; this.writes++; } } -const meter = new Meter(); +const meter = new WorkerMeter(); function write(target, next) { target.points = next; } parentPort.on("message", (message) => { if (message !== "go") return; From 84da0a76969597a80b5271f4ef1dd4db5eeca7d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 15:33:55 +0000 Subject: [PATCH 29/40] Probe direct setter before chain-store miss route --- crates/perry-runtime/src/proxy/put_value/packed_set.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/perry-runtime/src/proxy/put_value/packed_set.rs b/crates/perry-runtime/src/proxy/put_value/packed_set.rs index 3736a63ce6..eeb5e9e75c 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_set.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_set.rs @@ -183,6 +183,12 @@ pub extern "C" fn js_put_value_set_packed_miss( } } + // P4 checked inherited setters before key interning and the clear-chain + // add memo. A direct setter cannot add a receiver key, and this collecting + // route validates its own live link and descriptor before invocation. + if let Some(stored) = unsafe { setter_site::try_set(cache_slot, target, key, value) } { + return stored; + } // Inherited-access lane: a key-adding store whose chain this site has // already proved clear takes the transition append (`object::chain_store`). // Allocation-free on a decline. @@ -202,9 +208,6 @@ pub extern "C" fn js_put_value_set_packed_miss( return stored; } } - if let Some(stored) = unsafe { setter_site::try_set(cache_slot, target, key, value) } { - return stored; - } // The receiver's ShapeId before the store: the pre-shape a key-add memo // is keyed on. Allocation-free. let pre_shape = unsafe { crate::object::chain_store::pre_store_shape(target) }; From 1429cf3aa5ec6ddac3bc06ec4a63c0d0bd8d90ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 15:46:43 +0000 Subject: [PATCH 30/40] Guard class accessor sites with registry generation --- .../src/object/class_gc_roots.rs | 2 + .../src/object/class_registry/gc_roots.rs | 3 ++ .../src/object/method_site/read_holder.rs | 45 ++++++++++++++++--- crates/perry-runtime/src/proxy/metadata.rs | 1 + 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/crates/perry-runtime/src/object/class_gc_roots.rs b/crates/perry-runtime/src/object/class_gc_roots.rs index 495f926b43..51f0176f15 100644 --- a/crates/perry-runtime/src/object/class_gc_roots.rs +++ b/crates/perry-runtime/src/object/class_gc_roots.rs @@ -99,6 +99,7 @@ pub(crate) fn test_seed_decl_class_prototype_root(class_id: u32, proto_ptr: usiz .get_or_insert_with(Default::default) .insert(class_id, proto_ptr); }); + super::class_lookup_surface_gen_bump(); } #[cfg(test)] @@ -165,4 +166,5 @@ pub(crate) fn test_clear_class_inheritance_roots(proto_cid: u32, closure_cid: u3 m.remove(&closure_cid); } }); + super::class_lookup_surface_gen_bump(); } diff --git a/crates/perry-runtime/src/object/class_registry/gc_roots.rs b/crates/perry-runtime/src/object/class_registry/gc_roots.rs index 0033089974..b18e0a9e1c 100644 --- a/crates/perry-runtime/src/object/class_registry/gc_roots.rs +++ b/crates/perry-runtime/src/object/class_registry/gc_roots.rs @@ -614,6 +614,9 @@ pub(crate) fn test_clear_class_side_table_roots() { *guard = None; } }); + // Test-only map reset has the same invalidation contract as production + // registry stores: a still-live accessor site must decline its old link. + super::class_lookup_surface_gen_bump(); CLASS_PARENT_CLOSURES.with(|table| { if let Ok(mut guard) = table.write() { *guard = None; diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index f6787950b4..e0a779fccf 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -81,7 +81,8 @@ pub const HOLDER_SHAPE: usize = crate::codegen_abi::PIC_HOLDER_SHAPE_WORD; pub const HOLDER_KIND: usize = crate::codegen_abi::PIC_HOLDER_KIND_WORD; /// First of three intermediate hop addresses (depth 2..=4). pub const HOLDER_HOPS: usize = HOLDER_KIND + 1; -/// Low 32 bits: the first hop's ShapeId. High 32 bits: the second hop's. +/// Data/absence: first and second hop ShapeIds. Class accessor: the class +/// lookup-surface generation at prime time (no hop pointer uses this word). pub const HOLDER_HOP_SHAPES: usize = HOLDER_HOPS + 3; /// The site's holder state: [`STATE_REGISTERED`], [`STATE_LATCHED`] and the /// count of re-primes for a different receiver shape. @@ -398,8 +399,8 @@ pub(super) unsafe fn admitted_proto_id(obj: *const ObjectHeader) -> Option } /// A MIXED identity records an explicit serial link. A bare CLASS identity -/// does not pin its registry-resolved prototype, so the accessor-only hit must -/// compare the LIVE declared-prototype pointer with the cached holder. +/// does not pin its registry-resolved prototype, so priming resolves the live +/// declared-prototype pointer and the hit checks the registry's generation. unsafe fn class_link(recv: *const ObjectHeader) -> Option<*const ObjectHeader> { let pid = shape_proto_id(object_shape_stamp(recv))?; if object_proto_id(recv) != pid { @@ -415,6 +416,27 @@ unsafe fn class_link(recv: *const ObjectHeader) -> Option<*const ObjectHeader> { (!holder.is_null() && holder != recv).then_some(holder) } +/// The accessor's prime-time holder is still the direct prototype. Explicit +/// MIXED links are checked by pointer. Every writer that can replace a bare +/// CLASS registry link bumps the lookup-surface generation; GC rewrites both +/// this site's rooted holder and the registry root without changing it. +#[inline] +unsafe fn accessor_link_still_current(recv: *const ObjectHeader, stamp: u32, c: &PicCache) -> bool { + let Some(pid) = shape_proto_id(stamp) else { + return false; + }; + if object_proto_id(recv) != pid || c[HOLDER_OBJ] as usize == recv as usize { + return false; + } + if (PROTO_ID_CLASS..PROTO_ID_MIXED).contains(&pid) { + c[HOLDER_HOP_SHAPES] as u64 == crate::object::class_lookup_surface_generation() + } else if (PROTO_ID_MIXED..PROTO_ID_UNIQUE).contains(&pid) { + next_prototype(recv) as usize == c[HOLDER_OBJ] as usize + } else { + false + } +} + struct ClassAccessor { holder: usize, shape: u32, @@ -520,7 +542,7 @@ pub(crate) unsafe fn try_cached_class_accessor( let stamp = object_shape_stamp(recv); if stamp == 0 || c[HOLDER_RECV] != (u64::from(stamp) | PIC_ID_TOKEN_BIT) as i64 - || class_link(recv)? as usize != c[HOLDER_OBJ] as usize + || !accessor_link_still_current(recv, stamp, c) { return None; } @@ -843,7 +865,11 @@ unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk, acc for i in 0..HOLDER_MAX_DEPTH - 1 { c[HOLDER_HOPS + i] = w.hops[i].0 as i64; } - c[HOLDER_HOP_SHAPES] = (u64::from(w.hops[0].1) | u64::from(w.hops[1].1) << 32) as i64; + c[HOLDER_HOP_SHAPES] = if accessor { + crate::object::class_lookup_surface_generation() as i64 + } else { + (u64::from(w.hops[0].1) | u64::from(w.hops[1].1) << 32) as i64 + }; if c[HOLDER_STATE] & STATE_REGISTERED == 0 { c[HOLDER_STATE] |= STATE_REGISTERED; if let Ok(mut sites) = HOLDER_SITES.lock() { @@ -986,6 +1012,11 @@ mod tests { depth: 1, }; unsafe { publish(cache, receiver, &w, true) }; + let first_generation = cache[HOLDER_HOP_SHAPES]; + assert_eq!( + first_generation as u64, + crate::object::class_lookup_surface_generation() + ); assert_eq!( unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), Some(2.0) @@ -995,6 +1026,10 @@ mod tests { p2.with_const_ptr::(|ptr| { crate::object::test_seed_class_decl_prototype_object_root(CID, ptr as usize); }); + assert_ne!( + cache[HOLDER_HOP_SHAPES] as u64, + crate::object::class_lookup_surface_generation() + ); assert_eq!(unsafe { object_shape_stamp(receiver) }, recv_shape); assert!( unsafe { try_cached_class_accessor(receiver, &mut slot) }.is_none(), diff --git a/crates/perry-runtime/src/proxy/metadata.rs b/crates/perry-runtime/src/proxy/metadata.rs index 9dcfd43f29..da5ffcd1ef 100644 --- a/crates/perry-runtime/src/proxy/metadata.rs +++ b/crates/perry-runtime/src/proxy/metadata.rs @@ -331,6 +331,7 @@ mod tests { .get_or_insert_with(Default::default) .insert(cid, fake_proto_ptr); }); + crate::object::class_lookup_surface_gen_bump(); let target = f64::from_bits(POINTER_TAG | (fake_proto_ptr as u64 & POINTER_MASK)); assert_eq!( normalize_target_bits(target), From d6d18cc4357188b36f3cb8fb144581e4fe8630a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 16:13:29 +0000 Subject: [PATCH 31/40] Decode only raw class accessor entries on read hits --- .../perry-runtime/src/object/accessor_pair.rs | 24 ++++++++++++ .../src/object/accessor_pair_tests.rs | 39 +++++++++++++++++++ .../src/object/method_site/read_holder.rs | 20 +++++----- 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/crates/perry-runtime/src/object/accessor_pair.rs b/crates/perry-runtime/src/object/accessor_pair.rs index 3adbda6358..eee85a0dce 100644 --- a/crates/perry-runtime/src/object/accessor_pair.rs +++ b/crates/perry-runtime/src/object/accessor_pair.rs @@ -191,6 +191,30 @@ pub(crate) unsafe fn pair_of_value(value: u64) -> Option { }) } +/// The compiled INSTANCE getter at an already-proved accessor slot, or +/// `Some(0)` for a setter-only pair (whose read is `undefined`). A class +/// accessor site's hit needs neither closure nor static-entry decoding. +/// The pair's tag, GC kind and length are still checked on every hit because +/// the slot's value may be replaced without a holder ShapeId transition. +/// +/// # Safety +/// `value` is the slot value of a key proved to carry `ENTRY_ACCESSOR`. +#[inline] +pub(crate) unsafe fn raw_instance_getter_of_value(value: u64) -> Option { + if value & TAG_MASK != POINTER_TAG { + return None; + } + let pair = (value & POINTER_MASK) as *const ArrayHeader; + let header = crate::value::addr_class::try_read_gc_header(pair as usize)?; + if header.obj_type != crate::gc::GC_TYPE_ARRAY || (*pair).length as usize != PAIR_LEN { + return None; + } + let w = crate::array::array_elements_ptr(pair); + let raw_get = raw_of(*w.add(PAIR_RAW_GET)); + let raw_set = raw_of(*w.add(PAIR_RAW_SET)); + (raw_get != 0 || raw_set != 0).then_some(raw_get) +} + /// The accessor stored in `obj`'s slot for key position `pos`. /// /// # Safety diff --git a/crates/perry-runtime/src/object/accessor_pair_tests.rs b/crates/perry-runtime/src/object/accessor_pair_tests.rs index ab5cd191c4..37cb5fbb26 100644 --- a/crates/perry-runtime/src/object/accessor_pair_tests.rs +++ b/crates/perry-runtime/src/object/accessor_pair_tests.rs @@ -41,6 +41,45 @@ fn a_pair_round_trips_both_forms() { } } +#[test] +fn raw_instance_getter_probe_preserves_setter_only_and_static_refusal() { + let _lock = crate::gc::global_side_table_test_lock(); + unsafe { + for (acc, answer) in [ + ( + Accessor { + raw_get: 0x5555_1234_5678, + ..Default::default() + }, + Some(0x5555_1234_5678), + ), + ( + Accessor { + raw_set: 0x5555_8765_4320, + ..Default::default() + }, + Some(0), + ), + ( + Accessor { + static_get: 0x5555_1234_5678, + ..Default::default() + }, + None, + ), + (Accessor::default(), None), + ] { + let pair = pair_new(acc); + let value = crate::value::js_nanbox_pointer(pair as i64).to_bits(); + assert_eq!(raw_instance_getter_of_value(value), answer); + } + assert_eq!( + raw_instance_getter_of_value(crate::value::TAG_UNDEFINED), + None + ); + } +} + /// An ordinary object's accessor lives in its key's slot, not in the /// owner-keyed table, and reads back through the descriptor API. Sabotage: /// storing the pair in the table instead leaves the slot `undefined` and the diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index e0a779fccf..eaf3b7cc8d 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -5,20 +5,22 @@ //! is an ordinary object, and its [[Prototype]] identity. Only a serial //! identity or `PROTO_ID_DEFAULT` (the realm's `Object.prototype`) pins ONE //! object for the GC-leaf data/absent path. A collecting accessor entry may -//! also use a declared class identity because it rechecks the live registry -//! prototype pointer on every hit. +//! also use a declared class identity: a registry generation check proves +//! its rooted holder is still the registered prototype on every hit. //! * The holder's ShapeId `SH` vouches that `k` is an own inline data slot of //! the holder `H` — or, for an ABSENT entry, that the terminal object lacks //! `k` and has a null [[Prototype]]. //! * For a holder deeper than the direct prototype, each intermediate hop's //! ShapeId vouches that the hop lacks `k` and still links to the next hop. //! -//! Every fact is compared on use, so there is no invalidation and no global -//! word: a key add, delete, descriptor change or `setPrototypeOf` on any object +//! Data/absence facts are compared on use without a global invalidation word: +//! a key add, delete, descriptor change or `setPrototypeOf` on any object //! the entry names moves that object's ShapeId, and a value store to the //! holder's slot is seen because the hit LOADS the slot. A delete is a shape //! transition (#10826), so a holder whose ShapeId matches still has the slot: -//! the hit needs no `TAG_HOLE` test, as the emitted MRU hit needs none. +//! the hit needs no `TAG_HOLE` test, as the emitted MRU hit needs none. The +//! collecting class-accessor route additionally checks the class registry's +//! lookup-surface generation for a bare declared-prototype link. //! //! The entry lives in the read site's own cache (`PicCache` words //! [`HOLDER_RECV`]..=[`HOLDER_REGISTERED`]). The holder and the hops are @@ -555,13 +557,11 @@ pub(crate) unsafe fn try_cached_class_accessor( // the ShapeId; a raw-only pair replacement may not, so reread the pair // itself on every hit before invoking. let slot = c[HOLDER_KIND] as u32; - let acc = crate::object::accessor_pair::pair_of_value(slot_bits(holder, slot))?; - if acc.raw_get == 0 && acc.raw_set == 0 { - return None; - } + let raw_get = + crate::object::accessor_pair::raw_instance_getter_of_value(slot_bits(holder, slot))?; HITS_ACCESSOR.fetch_add(1, Ordering::Relaxed); super::stats_report_enabled(); - Some(invoke_class_getter(recv, acc.raw_get)) + Some(invoke_class_getter(recv, raw_get)) } unsafe fn walk(recv: *const ObjectHeader, name: &[u8], class_first: bool) -> Option { From b7105ce1f786808f4906a4202f6005387d5b5506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 16:08:25 +0000 Subject: [PATCH 32/40] Use validity epoch for direct class setter link --- .../src/proxy/put_value/setter_site.rs | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/perry-runtime/src/proxy/put_value/setter_site.rs b/crates/perry-runtime/src/proxy/put_value/setter_site.rs index 0d5b4d0fbd..e3210873a7 100644 --- a/crates/perry-runtime/src/proxy/put_value/setter_site.rs +++ b/crates/perry-runtime/src/proxy/put_value/setter_site.rs @@ -17,6 +17,7 @@ struct Entry { holder: usize, class_id: u32, receiver_shape: u32, + bare_class_link: bool, holder_shape: u32, slot: u32, raw_set: usize, @@ -168,6 +169,9 @@ unsafe fn candidate( return None; } + let bare_class_link = (crate::object::shapes::PROTO_ID_CLASS + ..crate::object::shapes::PROTO_ID_MIXED) + .contains(&recv_shape.proto_id); let holder = class_link(recv)?; let holder_gc = crate::value::addr_class::try_read_gc_header(holder as usize)?; if holder_gc.obj_type != crate::gc::GC_TYPE_OBJECT @@ -211,6 +215,7 @@ unsafe fn candidate( holder: holder as usize, class_id, receiver_shape: crate::object::shapes::object_shape_stamp(recv), + bare_class_link, holder_shape: crate::object::shapes::object_shape_stamp(holder), slot, raw_set: acc.raw_set, @@ -244,12 +249,27 @@ unsafe fn validated_raw_set( || e.receiver_shape != crate::object::shapes::object_shape_stamp(recv) || e.validity != crate::object::proto_validity::proto_validity() || e.vtable_gen != crate::object::vtable_generation() - || class_link(recv)? as usize != e.holder || crate::object::shapes::object_shape_stamp(e.holder as *const crate::ObjectHeader) != e.holder_shape { return None; } + if e.bare_class_link { + // ShapeId proves the class-link mode. Every declared-prototype root + // replacement (including generic-origin redirects) bumps the validity + // word checked above. GC moves both the registry root and this rooted + // holder entry together. A per-instance prototype change must either + // restamp the ShapeId or leave an explicit meta link, rejected here. + if gc._reserved & crate::gc::OBJ_FLAG_NULL_PROTO != 0 + || (!meta.is_null() && (*meta).prototype != 0) + || e.holder == recv as usize + { + return None; + } + } else if class_link(recv)? as usize != e.holder { + // MIXED receivers keep the full live per-object link comparison. + return None; + } let holder_gc = crate::value::addr_class::try_read_gc_header(e.holder)?; if holder_gc.obj_type != crate::gc::GC_TYPE_OBJECT || holder_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 @@ -439,6 +459,7 @@ mod tests { unsafe { try_set(&mut slot, target, key_ptr, 6.0) }, Some(6.0) ); + assert!(unsafe { entry(&mut slot).unwrap().bare_class_link }); assert_eq!(FIRST.load(Ordering::Relaxed), 2); p2.with_const_ptr::(|p| { crate::object::test_seed_class_decl_prototype_object_root(CID, p as usize) @@ -457,6 +478,21 @@ mod tests { crate::object::method_site::WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); assert_eq!(unsafe { try_set(&mut slot, target, key_ptr, 8.0) }, None); crate::object::method_site::WORKER_AGENTS_EXIST.store(before_gate, Ordering::SeqCst); + let p1_value = p1.with_const_ptr::(|p| { + crate::value::js_nanbox_pointer(p as i64) + }); + crate::object::object_ops::js_object_set_prototype_of(target, p1_value); + let key_ptr = key.get_raw_const_ptr::(); + let explicit_shape = recv.with_const_ptr::(|p| unsafe { + crate::object::shapes::object_shape_stamp(p) + }); + assert_ne!(explicit_shape, recv_shape); + assert_eq!( + unsafe { + validated_raw_set(entry(&mut slot).unwrap(), recv.get_raw_const_ptr(), key_ptr) + }, + None + ); recv.with_mut_ptr::(|p| unsafe { crate::object::object_ops::define_property_force_store_value(p, key_ptr, 99.0); }); @@ -464,7 +500,7 @@ mod tests { recv.with_const_ptr::(|p| unsafe { crate::object::shapes::object_shape_stamp(p) }), - recv_shape + explicit_shape ); assert_eq!( unsafe { From e4833240b176dee1c90a15ab2a05fb618cc5c8a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 17:22:09 +0000 Subject: [PATCH 33/40] test: align A2 runtime gates with worker and value-store invariants --- changelog.d/inherited-read-one-shape.md | 2 +- .../src/object/method_site/read_holder.rs | 10 ++++++++++ .../perry-runtime/src/object/proto_validity_tests.rs | 9 ++++----- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/changelog.d/inherited-read-one-shape.md b/changelog.d/inherited-read-one-shape.md index a53b9119e0..c6508f45b8 100644 --- a/changelog.d/inherited-read-one-shape.md +++ b/changelog.d/inherited-read-one-shape.md @@ -1 +1 @@ -**Objects:** inherited reads and method calls use receiver and holder shape facts; the old inherited-read cache and its GC roots are removed. Method sites take ordinary dispatch after worker startup. +**Objects:** Inherited reads and method calls now use receiver and holder shape facts; the old inherited-read cache and its GC roots are removed. Class getter and setter sites cache direct accessors with live shape and prototype checks. Worker startup gates holder-backed sites so each agent uses ordinary dispatch safely. diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index eaf3b7cc8d..cf463a7f64 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -944,6 +944,16 @@ mod tests { #[test] fn class_accessor_rechecks_same_shape_holder_link() { let _lock = crate::gc::global_side_table_test_lock(); + // Earlier runtime tests may have started a worker. Reset the sticky + // process gate only for this isolated site test, then restore it even + // when an assertion fails. + struct RestoreWorkerGate(u8); + impl Drop for RestoreWorkerGate { + fn drop(&mut self) { + WORKER_AGENTS_EXIST.store(self.0, Ordering::SeqCst); + } + } + let _gate = RestoreWorkerGate(WORKER_AGENTS_EXIST.swap(0, Ordering::SeqCst)); const CID: u32 = 0x0C3C_79A3; let scope = crate::gc::RuntimeHandleScope::new(); let p1 = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 2)); diff --git a/crates/perry-runtime/src/object/proto_validity_tests.rs b/crates/perry-runtime/src/object/proto_validity_tests.rs index 37240d143a..6c8775dfee 100644 --- a/crates/perry-runtime/src/object/proto_validity_tests.rs +++ b/crates/perry-runtime/src/object/proto_validity_tests.rs @@ -102,7 +102,7 @@ fn a_structural_mutation_of_an_unmarked_object_does_not_bump_validity() { } #[test] -fn a_plain_value_store_on_a_marked_object_bumps_validity() { +fn a_plain_value_store_on_a_marked_object_keeps_validity() { let _scope = Scope::new(); unsafe { let proto = crate::object::js_object_alloc(0, 4); @@ -111,12 +111,11 @@ fn a_plain_value_store_on_a_marked_object_bumps_validity() { let before = proto_validity(); set(proto, "pv_store_a", 99.0); - assert_ne!( + assert_eq!( proto_validity(), before, - "owner decision D3(b): an inherited method-site entry memoizes the \ - VALUE (the method closure), so replacing a value on a marked \ - prototype must invalidate it" + "a plain overwrite leaves the chain-store verdict intact; \ + inherited method sites reload the holder slot on every hit" ); // The same store on an object nobody inherits from stays free. From a0db1b48f4b5843ef7f847e43fa7a4f817dbf18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 17:32:47 +0000 Subject: [PATCH 34/40] test: preserve sticky worker gate across A2 unit tests --- .../src/object/method_site/read_holder.rs | 89 +++++++++++++------ .../method_site/read_holder/class_read.rs | 24 +++-- .../src/proxy/put_value/setter_site.rs | 29 ++++-- 3 files changed, 96 insertions(+), 46 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index cf463a7f64..731d8b6f7b 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -193,6 +193,13 @@ pub(crate) unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return None; } + entry_answer_after_worker_gate(c, token) +} + +/// The same site-word validation for primary-agent unit tests whose process +/// may already have started a worker. The public entry remains gate guarded. +#[inline(always)] +unsafe fn entry_answer_after_worker_gate(c: &PicCache, token: i64) -> Option { if token == 0 { return None; } @@ -526,7 +533,20 @@ pub(crate) unsafe fn try_cached_class_accessor( recv: *const ObjectHeader, cache_slot: *mut PicCacheSlot, ) -> Option { - if cache_slot.is_null() || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { + if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { + return None; + } + try_cached_class_accessor_after_worker_gate(recv, cache_slot) +} + +/// The primary-agent accessor validation, also used by the unit test when +/// another test has already set the sticky process-wide worker gate. +#[inline(always)] +unsafe fn try_cached_class_accessor_after_worker_gate( + recv: *const ObjectHeader, + cache_slot: *mut PicCacheSlot, +) -> Option { + if cache_slot.is_null() { return None; } let cache = crate::object::field_get_set::pic_slot_peek::(cache_slot); @@ -944,16 +964,6 @@ mod tests { #[test] fn class_accessor_rechecks_same_shape_holder_link() { let _lock = crate::gc::global_side_table_test_lock(); - // Earlier runtime tests may have started a worker. Reset the sticky - // process gate only for this isolated site test, then restore it even - // when an assertion fails. - struct RestoreWorkerGate(u8); - impl Drop for RestoreWorkerGate { - fn drop(&mut self) { - WORKER_AGENTS_EXIST.store(self.0, Ordering::SeqCst); - } - } - let _gate = RestoreWorkerGate(WORKER_AGENTS_EXIST.swap(0, Ordering::SeqCst)); const CID: u32 = 0x0C3C_79A3; let scope = crate::gc::RuntimeHandleScope::new(); let p1 = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 2)); @@ -1028,7 +1038,8 @@ mod tests { crate::object::class_lookup_surface_generation() ); assert_eq!( - unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), + unsafe { try_cached_class_accessor_after_worker_gate(receiver, &mut slot) } + .map(|v| v.as_number()), Some(2.0) ); @@ -1042,7 +1053,8 @@ mod tests { ); assert_eq!(unsafe { object_shape_stamp(receiver) }, recv_shape); assert!( - unsafe { try_cached_class_accessor(receiver, &mut slot) }.is_none(), + unsafe { try_cached_class_accessor_after_worker_gate(receiver, &mut slot) } + .is_none(), "stale getter was served after registry replacement" ); let second = @@ -1057,7 +1069,8 @@ mod tests { unsafe { publish(cache, receiver, &w, true) }; assert!(read_accessor_same_shape_relinks() > old_relinks); assert_eq!( - unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), + unsafe { try_cached_class_accessor_after_worker_gate(receiver, &mut slot) } + .map(|v| v.as_number()), Some(8.0) ); }); @@ -1066,7 +1079,6 @@ mod tests { #[test] fn ten_receiver_shapes_share_one_confirmed_absent_terminal() { let _lock = crate::gc::global_side_table_test_lock(); - let gate = WORKER_AGENTS_EXIST.swap(0, Ordering::SeqCst); let base = crate::object::shapes::SHAPE_ID_BASE; let holder = Box::new(ObjectHeader { class_id: 0, @@ -1100,21 +1112,28 @@ mod tests { } assert_ne!(cache[HOLDER_KIND] as u64 & HOLDER_MULTI_ABSENT, 0); assert_eq!( - unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base)) as i64) }, + unsafe { + entry_answer_after_worker_gate(&cache, (PIC_ID_TOKEN_BIT | u64::from(base)) as i64) + }, None, "the oldest of eleven shapes must leave a ten-shape site" ); for i in 1..11 { let token = (PIC_ID_TOKEN_BIT | u64::from(base + i)) as i64; assert_eq!( - unsafe { entry_answer(&cache, token) }, + unsafe { entry_answer_after_worker_gate(&cache, token) }, Some(crate::value::TAG_UNDEFINED) ); } // A new shape after an own-key shadow has no entry, while a terminal // mutation invalidates every receiver shape in the shared entry. assert_eq!( - unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64) }, + unsafe { + entry_answer_after_worker_gate( + &cache, + (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64, + ) + }, None ); let mut moved = Box::new(ObjectHeader { @@ -1124,12 +1143,22 @@ mod tests { }); cache[HOLDER_OBJ] = (&mut *moved as *mut ObjectHeader) as i64; assert_eq!( - unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64) }, + unsafe { + entry_answer_after_worker_gate( + &cache, + (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64, + ) + }, Some(crate::value::TAG_UNDEFINED) ); moved.parent_class_id = base + 102; assert_eq!( - unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64) }, + unsafe { + entry_answer_after_worker_gate( + &cache, + (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64, + ) + }, None ); // A different terminal never inherits the old entry's receiver set. @@ -1142,19 +1171,23 @@ mod tests { unsafe { publish(&mut cache, &recv, &distinct, false) }; assert_eq!(cache[HOLDER_KIND], HOLDER_ABSENT_DEPTH1); assert_eq!( - unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64) }, + unsafe { + entry_answer_after_worker_gate( + &cache, + (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64, + ) + }, None ); assert_eq!( - unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64) }, + unsafe { + entry_answer_after_worker_gate( + &cache, + (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64, + ) + }, Some(crate::value::TAG_UNDEFINED) ); - WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); - assert_eq!( - unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64) }, - None - ); - WORKER_AGENTS_EXIST.store(gate, Ordering::SeqCst); } /// A class instance has a valid, stamped ShapeId, but its prototype is diff --git a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs index 82a0bae582..56f21447d2 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs @@ -137,10 +137,20 @@ pub(super) unsafe fn try_hit( recv: *const ObjectHeader, cache_slot: *mut PicCacheSlot, ) -> Option { - if cache_slot.is_null() - || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 - || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT - { + if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { + return None; + } + try_hit_after_worker_gate(recv, cache_slot) +} + +/// Primary-agent site validation used by the unit test after another test +/// has already started a worker; the public hit stays gate guarded. +#[inline(always)] +unsafe fn try_hit_after_worker_gate( + recv: *const ObjectHeader, + cache_slot: *mut PicCacheSlot, +) -> Option { + if cache_slot.is_null() || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT { return None; } let cache = crate::object::field_get_set::pic_slot_peek::(cache_slot); @@ -382,14 +392,10 @@ mod tests { cache[SITE_WORD] = (SITE_TAG | record as usize as u64) as i64; cache[HOLDER_STATE] = STATE_CLASS_SITE; let mut slot = &mut cache as *mut PicCache; - let gate = WORKER_AGENTS_EXIST.swap(0, Ordering::SeqCst); assert_eq!( - unsafe { try_hit(recv, &mut slot) }.map(|v| v.bits()), + unsafe { try_hit_after_worker_gate(recv, &mut slot) }.map(|v| v.bits()), Some(crate::value::TAG_UNDEFINED) ); - WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); - assert!(unsafe { try_hit(recv, &mut slot) }.is_none()); - WORKER_AGENTS_EXIST.store(gate, Ordering::SeqCst); unsafe { drop(Box::from_raw(record)) }; } } diff --git a/crates/perry-runtime/src/proxy/put_value/setter_site.rs b/crates/perry-runtime/src/proxy/put_value/setter_site.rs index e3210873a7..a443b8e425 100644 --- a/crates/perry-runtime/src/proxy/put_value/setter_site.rs +++ b/crates/perry-runtime/src/proxy/put_value/setter_site.rs @@ -299,7 +299,22 @@ pub(super) unsafe fn try_set( key: *const crate::StringHeader, value: f64, ) -> Option { - if !primary_only() || slot.is_null() || key.is_null() { + if !primary_only() { + return None; + } + try_set_after_worker_gate(slot, target, key, value) +} + +/// The primary-agent setter validation, also exercised by the unit test +/// without resetting the process-wide sticky worker gate. +#[inline(always)] +unsafe fn try_set_after_worker_gate( + slot: *mut PackedSetWaysSlot, + target: f64, + key: *const crate::StringHeader, + value: f64, +) -> Option { + if slot.is_null() || key.is_null() { return None; } let bits = target.to_bits(); @@ -379,10 +394,9 @@ mod tests { } #[test] - fn direct_setter_rechecks_same_shape_relink_own_shadow_and_worker_gate() { + fn direct_setter_rechecks_same_shape_relink_and_own_shadow() { let _lock = crate::gc::global_side_table_test_lock(); const CID: u32 = 0x0C3C_79B5; - let before_gate = crate::object::method_site::WORKER_AGENTS_EXIST.swap(0, Ordering::SeqCst); FIRST.store(0, Ordering::Relaxed); SECOND.store(0, Ordering::Relaxed); unsafe { @@ -452,11 +466,11 @@ mod tests { }); let key_ptr = key.get_raw_const_ptr::(); assert_eq!( - unsafe { try_set(&mut slot, target, key_ptr, 5.0) }, + unsafe { try_set_after_worker_gate(&mut slot, target, key_ptr, 5.0) }, Some(5.0) ); assert_eq!( - unsafe { try_set(&mut slot, target, key_ptr, 6.0) }, + unsafe { try_set_after_worker_gate(&mut slot, target, key_ptr, 6.0) }, Some(6.0) ); assert!(unsafe { entry(&mut slot).unwrap().bare_class_link }); @@ -471,13 +485,10 @@ mod tests { None ); assert_eq!( - unsafe { try_set(&mut slot, target, key_ptr, 7.0) }, + unsafe { try_set_after_worker_gate(&mut slot, target, key_ptr, 7.0) }, Some(7.0) ); assert_eq!(SECOND.load(Ordering::Relaxed), 1); - crate::object::method_site::WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); - assert_eq!(unsafe { try_set(&mut slot, target, key_ptr, 8.0) }, None); - crate::object::method_site::WORKER_AGENTS_EXIST.store(before_gate, Ordering::SeqCst); let p1_value = p1.with_const_ptr::(|p| { crate::value::js_nanbox_pointer(p as i64) }); From 026cb9a125ba2f187e30097fc0d461772152484e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 17:41:26 +0000 Subject: [PATCH 35/40] test: isolate A2 worker-gate units in fresh processes --- .../perry-runtime/src/object/method_site.rs | 29 +++++++ .../src/object/method_site/read_holder.rs | 87 ++++++------------- .../method_site/read_holder/class_read.rs | 27 +++--- .../src/proxy/put_value/setter_site.rs | 32 +++---- 4 files changed, 79 insertions(+), 96 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site.rs b/crates/perry-runtime/src/object/method_site.rs index f07dfe0adf..7036551aad 100644 --- a/crates/perry-runtime/src/object/method_site.rs +++ b/crates/perry-runtime/src/object/method_site.rs @@ -197,6 +197,35 @@ static METHOD_SITES: std::sync::Mutex> = std::sync::Mutex::new(Vec::n pub(crate) static WORKER_AGENTS_EXIST: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); +/// Run a gate-sensitive unit in a fresh test process. Worker startup is +/// process-wide and sticky: clearing it in a parallel libtest process can +/// re-enable a worker's access to primary-heap holder pointers. +#[cfg(test)] +pub(crate) fn run_with_fresh_worker_gate(filter: &str) -> bool { + const MARKER: &str = "PERRY_A2_FRESH_WORKER_GATE_TEST"; + if std::env::var_os(MARKER).as_deref() == Some(std::ffi::OsStr::new(filter)) { + assert_eq!( + WORKER_AGENTS_EXIST.load(Ordering::SeqCst), + 0, + "the filtered child must begin before worker startup" + ); + return true; + } + let output = std::process::Command::new(std::env::current_exe().expect("test executable")) + .arg("--test-threads=1") + .arg(filter) + .env(MARKER, filter) + .output() + .expect("run filtered test in a fresh process"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success() && stdout.contains("running 1 test") && stdout.contains("1 passed"), + "isolated test {filter} failed or matched no test:\n{stdout}\n{stderr}", + ); + false +} + /// Called by `agent::enter_worker_agent` before the worker runs any code. pub fn note_worker_agent() { // Publish the gate under the same lock as `publish`: every in-flight diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 731d8b6f7b..4952184a3d 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -193,13 +193,6 @@ pub(crate) unsafe fn entry_answer(c: &PicCache, token: i64) -> Option { if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return None; } - entry_answer_after_worker_gate(c, token) -} - -/// The same site-word validation for primary-agent unit tests whose process -/// may already have started a worker. The public entry remains gate guarded. -#[inline(always)] -unsafe fn entry_answer_after_worker_gate(c: &PicCache, token: i64) -> Option { if token == 0 { return None; } @@ -533,20 +526,7 @@ pub(crate) unsafe fn try_cached_class_accessor( recv: *const ObjectHeader, cache_slot: *mut PicCacheSlot, ) -> Option { - if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { - return None; - } - try_cached_class_accessor_after_worker_gate(recv, cache_slot) -} - -/// The primary-agent accessor validation, also used by the unit test when -/// another test has already set the sticky process-wide worker gate. -#[inline(always)] -unsafe fn try_cached_class_accessor_after_worker_gate( - recv: *const ObjectHeader, - cache_slot: *mut PicCacheSlot, -) -> Option { - if cache_slot.is_null() { + if cache_slot.is_null() || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return None; } let cache = crate::object::field_get_set::pic_slot_peek::(cache_slot); @@ -963,6 +943,11 @@ mod tests { /// link, rather than trust receiver and holder shapes alone. #[test] fn class_accessor_rechecks_same_shape_holder_link() { + if !crate::object::method_site::run_with_fresh_worker_gate( + "class_accessor_rechecks_same_shape_holder_link", + ) { + return; + } let _lock = crate::gc::global_side_table_test_lock(); const CID: u32 = 0x0C3C_79A3; let scope = crate::gc::RuntimeHandleScope::new(); @@ -1038,8 +1023,7 @@ mod tests { crate::object::class_lookup_surface_generation() ); assert_eq!( - unsafe { try_cached_class_accessor_after_worker_gate(receiver, &mut slot) } - .map(|v| v.as_number()), + unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), Some(2.0) ); @@ -1053,8 +1037,7 @@ mod tests { ); assert_eq!(unsafe { object_shape_stamp(receiver) }, recv_shape); assert!( - unsafe { try_cached_class_accessor_after_worker_gate(receiver, &mut slot) } - .is_none(), + unsafe { try_cached_class_accessor(receiver, &mut slot) }.is_none(), "stale getter was served after registry replacement" ); let second = @@ -1069,8 +1052,7 @@ mod tests { unsafe { publish(cache, receiver, &w, true) }; assert!(read_accessor_same_shape_relinks() > old_relinks); assert_eq!( - unsafe { try_cached_class_accessor_after_worker_gate(receiver, &mut slot) } - .map(|v| v.as_number()), + unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), Some(8.0) ); }); @@ -1078,6 +1060,11 @@ mod tests { #[test] fn ten_receiver_shapes_share_one_confirmed_absent_terminal() { + if !crate::object::method_site::run_with_fresh_worker_gate( + "ten_receiver_shapes_share_one_confirmed_absent_terminal", + ) { + return; + } let _lock = crate::gc::global_side_table_test_lock(); let base = crate::object::shapes::SHAPE_ID_BASE; let holder = Box::new(ObjectHeader { @@ -1112,28 +1099,21 @@ mod tests { } assert_ne!(cache[HOLDER_KIND] as u64 & HOLDER_MULTI_ABSENT, 0); assert_eq!( - unsafe { - entry_answer_after_worker_gate(&cache, (PIC_ID_TOKEN_BIT | u64::from(base)) as i64) - }, + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base)) as i64) }, None, "the oldest of eleven shapes must leave a ten-shape site" ); for i in 1..11 { let token = (PIC_ID_TOKEN_BIT | u64::from(base + i)) as i64; assert_eq!( - unsafe { entry_answer_after_worker_gate(&cache, token) }, + unsafe { entry_answer(&cache, token) }, Some(crate::value::TAG_UNDEFINED) ); } // A new shape after an own-key shadow has no entry, while a terminal // mutation invalidates every receiver shape in the shared entry. assert_eq!( - unsafe { - entry_answer_after_worker_gate( - &cache, - (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64, - ) - }, + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64) }, None ); let mut moved = Box::new(ObjectHeader { @@ -1143,22 +1123,12 @@ mod tests { }); cache[HOLDER_OBJ] = (&mut *moved as *mut ObjectHeader) as i64; assert_eq!( - unsafe { - entry_answer_after_worker_gate( - &cache, - (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64, - ) - }, + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64) }, Some(crate::value::TAG_UNDEFINED) ); moved.parent_class_id = base + 102; assert_eq!( - unsafe { - entry_answer_after_worker_gate( - &cache, - (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64, - ) - }, + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64) }, None ); // A different terminal never inherits the old entry's receiver set. @@ -1171,23 +1141,18 @@ mod tests { unsafe { publish(&mut cache, &recv, &distinct, false) }; assert_eq!(cache[HOLDER_KIND], HOLDER_ABSENT_DEPTH1); assert_eq!( - unsafe { - entry_answer_after_worker_gate( - &cache, - (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64, - ) - }, + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 10)) as i64) }, None ); assert_eq!( - unsafe { - entry_answer_after_worker_gate( - &cache, - (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64, - ) - }, + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64) }, Some(crate::value::TAG_UNDEFINED) ); + WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); + assert_eq!( + unsafe { entry_answer(&cache, (PIC_ID_TOKEN_BIT | u64::from(base + 11)) as i64) }, + None + ); } /// A class instance has a valid, stamped ShapeId, but its prototype is diff --git a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs index 56f21447d2..e7c66c0dc2 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs @@ -137,20 +137,10 @@ pub(super) unsafe fn try_hit( recv: *const ObjectHeader, cache_slot: *mut PicCacheSlot, ) -> Option { - if WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { - return None; - } - try_hit_after_worker_gate(recv, cache_slot) -} - -/// Primary-agent site validation used by the unit test after another test -/// has already started a worker; the public hit stays gate guarded. -#[inline(always)] -unsafe fn try_hit_after_worker_gate( - recv: *const ObjectHeader, - cache_slot: *mut PicCacheSlot, -) -> Option { - if cache_slot.is_null() || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT { + if cache_slot.is_null() + || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 + || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT + { return None; } let cache = crate::object::field_get_set::pic_slot_peek::(cache_slot); @@ -309,6 +299,11 @@ mod tests { #[test] fn bare_class_link_replacement_with_same_holder_shape_declines() { + if !crate::object::method_site::run_with_fresh_worker_gate( + "bare_class_link_replacement_with_same_holder_shape_declines", + ) { + return; + } let _lock = crate::gc::global_side_table_test_lock(); const CID: u32 = 0x0C3C_79A3; const PROTO_CID: u32 = 0x0C3C_79A4; @@ -393,9 +388,11 @@ mod tests { cache[HOLDER_STATE] = STATE_CLASS_SITE; let mut slot = &mut cache as *mut PicCache; assert_eq!( - unsafe { try_hit_after_worker_gate(recv, &mut slot) }.map(|v| v.bits()), + unsafe { try_hit(recv, &mut slot) }.map(|v| v.bits()), Some(crate::value::TAG_UNDEFINED) ); + WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); + assert!(unsafe { try_hit(recv, &mut slot) }.is_none()); unsafe { drop(Box::from_raw(record)) }; } } diff --git a/crates/perry-runtime/src/proxy/put_value/setter_site.rs b/crates/perry-runtime/src/proxy/put_value/setter_site.rs index a443b8e425..f58a20210f 100644 --- a/crates/perry-runtime/src/proxy/put_value/setter_site.rs +++ b/crates/perry-runtime/src/proxy/put_value/setter_site.rs @@ -299,22 +299,7 @@ pub(super) unsafe fn try_set( key: *const crate::StringHeader, value: f64, ) -> Option { - if !primary_only() { - return None; - } - try_set_after_worker_gate(slot, target, key, value) -} - -/// The primary-agent setter validation, also exercised by the unit test -/// without resetting the process-wide sticky worker gate. -#[inline(always)] -unsafe fn try_set_after_worker_gate( - slot: *mut PackedSetWaysSlot, - target: f64, - key: *const crate::StringHeader, - value: f64, -) -> Option { - if slot.is_null() || key.is_null() { + if !primary_only() || slot.is_null() || key.is_null() { return None; } let bits = target.to_bits(); @@ -394,7 +379,12 @@ mod tests { } #[test] - fn direct_setter_rechecks_same_shape_relink_and_own_shadow() { + fn direct_setter_rechecks_same_shape_relink_own_shadow_and_worker_gate() { + if !crate::object::method_site::run_with_fresh_worker_gate( + "direct_setter_rechecks_same_shape_relink_own_shadow_and_worker_gate", + ) { + return; + } let _lock = crate::gc::global_side_table_test_lock(); const CID: u32 = 0x0C3C_79B5; FIRST.store(0, Ordering::Relaxed); @@ -466,11 +456,11 @@ mod tests { }); let key_ptr = key.get_raw_const_ptr::(); assert_eq!( - unsafe { try_set_after_worker_gate(&mut slot, target, key_ptr, 5.0) }, + unsafe { try_set(&mut slot, target, key_ptr, 5.0) }, Some(5.0) ); assert_eq!( - unsafe { try_set_after_worker_gate(&mut slot, target, key_ptr, 6.0) }, + unsafe { try_set(&mut slot, target, key_ptr, 6.0) }, Some(6.0) ); assert!(unsafe { entry(&mut slot).unwrap().bare_class_link }); @@ -485,10 +475,12 @@ mod tests { None ); assert_eq!( - unsafe { try_set_after_worker_gate(&mut slot, target, key_ptr, 7.0) }, + unsafe { try_set(&mut slot, target, key_ptr, 7.0) }, Some(7.0) ); assert_eq!(SECOND.load(Ordering::Relaxed), 1); + crate::object::method_site::WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); + assert_eq!(unsafe { try_set(&mut slot, target, key_ptr, 8.0) }, None); let p1_value = p1.with_const_ptr::(|p| { crate::value::js_nanbox_pointer(p as i64) }); From ecfd2a95a99ee333076ca79fa4f651f01f472a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 17:54:45 +0000 Subject: [PATCH 36/40] ci: reconcile A2 root inventory with scope-context main --- scripts/gc_runtime_root_holders.json | 77 ++++------------------------ 1 file changed, 9 insertions(+), 68 deletions(-) diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 5bfddf270d..69c9065866 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -184,13 +184,6 @@ "verdict": "not_a_gc_pointer", "why": "Monotonic counter of live async-resource handles. Holds no address at all; the resource objects live in RESOURCES, which scan_async_hooks_roots_mut visits." }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOX_YOUNG_ROOTS", - "verdict": "covered_elsewhere", - "why": "#9976: the minor remembered set for box roots — a YoungLog of box addresses whose payload may matter to a minor. Every address in it is also in the box REGISTRY, which the module's own doc calls the authoritative full/major root set and which `scan_box_roots_mut` walks. The log is an accelerator over that set, not an independent holder: an address dropped from it is still reached through the registry.", - "scanner": "box::scan_box_roots_mut (crates/perry-runtime/src/box.rs), registered by reg_scanner! in crates/perry-runtime/src/gc/mod.rs" - }, { "file": "crates/perry-runtime/src/buffer/header.rs", "name": "BUFFER_ADDR_RANGE", @@ -350,7 +343,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete \u2192 sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs \u2014 it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase \u2014 after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged \u2014 `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` \u2014 and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` \u2192 `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only \u2014 no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound \u2014 the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses \u2014 no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects \u2014 and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module \u2014 all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete \u2192 sweep-entry window of a synchronous full \u2014 where PASS1_MARKED is populated and consumed within one `run_to_completion` \u2014 is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize \u2014 INSIDE the window \u2014 the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes \u2014 in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 (charter step 5, P0): the field-representation census adds `mod census_field_repr;` to `gc/mod.rs` and, in `census.rs`, one Rust-owned accumulator fed from `visit_object` inside `take_census`, which runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED; it reads slots and the per-object layout maps (try_borrow), allocates nothing on the JS heap and alters no mark/sweep control flow. The window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 (class constructors as function objects, #11414): `gc/mod.rs` gains one `reg_scanner!` registration (`object::class_value::scan_class_value_roots_mut`, the per-agent class function-object table, visited and rewritten) \u2014 a root-scanner registration that alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 after #11659 added `verify::verify_array_hole_tails_at_collection()` to `gc/mod.rs` and `gc/policy.rs`. Every call sits at a collection or budgeted-cycle ENTRY, beside `roots::ensure_stack_maps_built()`, i.e. before mark begins and so before `census_pass1_if_armed` opens the window. The call is a read-only walk of array headers (debug builds, or release with PERRY_GC_VERIFY_ARRAY_HOLES): it neither allocates GC memory, relocates, runs JS, nor reads or writes PASS1_MARKED, and it panics rather than continuing on a violation. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Pin re-audited 2026-09-29 after PR #11646 added one call inside that window, in `cycle.rs` beside `census_take_if_armed_at_full_sweep_start`: `object::shapes::store_kind::audit_heap_at_full_sweep_start()`. It compiles to nothing unless the `shape-fact-audit` feature is on; when on it walks the arena read-only (`gc::for_each_live_object_at_sweep_start`, in `gc/verify.rs`) and reads each marked object header and its shape record. It allocates no GC object, moves nothing, runs no JS callback and holds no address past the walk, so PASS1_MARKED stays valid across it; a disagreement panics rather than continuing. Re-audited 2026-09-29 after the pinned-roots fix touched `gc/cycle.rs` and `gc/mod.rs`: the block-persistence live-block predicate now reads GC_FLAG_MARKED alone (pinned objects are marked as roots, so the set of live blocks is unchanged), and `gc/mod.rs` adds one `reg_scanner!` registration. Neither relocates, runs JS, or moves the mark-complete to sweep-entry window. Re-audited again 2026-09-29 (same fix, copying-minor follow-up): `BlockPersistCycleState`'s force-mark in `gc/cycle.rs` no longer skips an unmarked pinned header, so it marks and pushes it like any other object of a live block. That is marking inside the budgeted cycle's persistence step: nothing relocates, no JS runs, and the budgeted path still skips both census boundaries. The window is unchanged. Re-audited 2026-09-29 for this-as-a-parameter stage 3: `gc/mod.rs` registers `scan_dispatch_binding_roots_mut` in place of `scan_implicit_this_roots_mut` (the implicit-`this` cell is deleted; the same scanner body keeps `new.target`, the static-`this` override and the static private-owner stack) and rewords its comment; a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 for the JsFunctionInfo lane: `gc/census.rs`'s `side_tables()` loses its `closure_registry_census()` row (the closure-body registry is deleted); a census report row, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 for the read-site holder entry: `gc/mod.rs` adds one `reg_scanner!` registration, `read_holder::scan_read_holder_roots_mut`, which visits the holder and hop words of registered read-site caches during ROOT SCAN, before mark propagation completes. It rewrites root slots only through the visitor (as every registered scanner does), runs no JS, and nothing it does executes between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. The window is unchanged. Re-audited 2026-09-30 for Step 5 P4: census.rs only removes the typed-layout count, and gc/mod.rs removes shape-install registration/re-export and an init-time typed-intact verifier gate. The snapshot remains stored after mark propagation and taken at sweep entry; none of these edits relocates objects or invokes callbacks in that interval. Re-audited 2026-09-29 (#11549 trace cost): `gc/mod.rs` gains one `mod copying_object_scan;` declaration, the copying minor's plain-object slot walk. It is reached only from the copying drain, never from a full cycle, and changes no mark or sweep control flow. The window is unchanged. Re-audited 2026-09-30 after rebasing P4 over #11549/#11676: gc/mod.rs retains the copying_object_scan declaration and P4 removes typed-layout registration; both are module wiring outside the synchronous full mark-complete to sweep-entry window. The census boundaries and the window owner in cycle.rs are unchanged. Re-audited for A2 inherited-read-cache deletion: gc/mod.rs removes only its old root-scanner registration and adjacent comments. The method-site and read-holder scanners still run during root scan before mark completion; no collection phase or callback was added inside the mark-complete to sweep-entry window. Re-audited 2026-09-30 for the setter-site root scanner: gc/mod.rs adds only its reg_scanner! registration in the root-scan setup. That scanner visits per-site key and holder slots before mark propagation completes; it runs no JS and adds no relocation or callback between census pass1 and sweep entry. The synchronous-full snapshot window and both boundaries remain unchanged. Re-audited 2026-09-30 for the scope-context change: gc/mod.rs also removes the box root-scanner registration and exit-time box statistics. The former ran during root scan before mark completion, and the latter at process exit. Neither changes the synchronous mark-complete to sweep-entry window.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-11 for the startup memory profile: gc/mod.rs only retains the pre-main allocator-policy constructor in js_gc_init. The constructor applies process allocation options, without invoking GC or JS. No census boundary, collector phase, or mark-complete to sweep-entry control flow changed. Re-audited 2026-09-13 for #10179: census.rs only adds a native regex cache metadata row and its unit assertion; snapshot consumption and the full-cycle window are unchanged. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window. Re-audited 2026-09-14 for the tiny-parse nursery-cap boundary, which touched `gc/policy.rs`. It adds `tiny_parse_generational_collection_due`, a pure predicate (the existing `tiny_parse_pressure_due` OR the existing `young_scavenge_cap_due` read), and calls it instead of `tiny_parse_pressure_due` from `gc_bump_malloc_trigger_inner` and `gc_collect_pending_suppressed_parse_slow` (generational branch only) and from `gc_schedule_parse_boundary_collection_if_pressure`. All three are JSON.parse mutator-side boundaries, none reachable from `step_mark_propagation` or `step_sweep`; the predicate reads counters and allocates nothing. Neither census boundary nor the synchronous mark-complete to sweep-entry interval changed. Re-audited 2026-09-13 for #10182's full-collection throughput follow-up, which touched `gc/cycle.rs` in one hunk, INSIDE the window: the `RememberedSetRebuild` subphase of a synchronous full now first asks `verify::full_remembered_rebuild_provably_empty` and, when it holds, installs `OldToYoungRememberedRebuildState::provably_empty()` (an empty sticky set, no walk) instead of the require-marked rebuild. The predicate reads `arena_block_snapshots()` (one `Vec` through the global allocator), the census's per-block reached/pre-marked facts and the malloc registry's length; the constructor bumps a `Cell` counter and prints one line under `PERRY_GC_DIAG`. None of it allocates a GC object, relocates anything, collects, or runs a JS callback, and both census boundaries stay where they were. Re-audited 2026-09-14 for #10182's pacing-full work, which touched `gc/cycle.rs`, `gc/mod.rs` and `gc/policy.rs`. `gc/cycle.rs`: `GcCycleState::new_full` no longer calls `materialize_all_promoted_page_runs`; that call ran in the constructor, before the census and far before `census_pass1_if_armed`, and removing it adds nothing to the window. `gc/mod.rs`: one `mod promoted_cohort;` declaration. `gc/policy.rs`: (a) `credit_promoted_bytes_to_old_baseline` also credits a `Cell` cohort counter (it runs after a copying minor completes); (b) `finish_full_old_reclaim_baseline` also records the verified old live bytes and resets that counter (Publish, after `step_sweep` consumed the snapshot); (c) `gc_safepoint_moving_minor` arms and disarms the promotion-census record around its nursery minor and calls `run_promoted_cohort_full_if_due`, which starts a synchronous full through the same `gc_collect_full_mark_sweep_with_trigger` entry and reads byte counters before and after it. All of it runs before a cycle starts or after it completes; none of it runs between mark completion and sweep entry, allocates a GC object, relocates anything, or calls into JS. The census the promoted-cohort full may adopt from the promotion walk is built in `BuildValidPointerSet`, before either boundary. Both boundaries are unchanged. Re-audited 2026-09-14 for the #10182 dead-stack scrub in `gc/cycle.rs`: `step_build_valid_pointer_set` now calls `scrub_dead_stack_below`, which zeroes a local array in its own frame (dead stack below the caller), right after the census finishes — in `BuildValidPointerSet`, before the root scan and far before `census_pass1_if_armed`. It writes no heap memory, allocates nothing, relocates nothing and calls no JS; both boundaries are unchanged. Re-audited 2026-09-14 for #10241 (cohort survival), which touched `gc/cycle.rs` and `gc/policy.rs`. `gc/cycle.rs`: one call, `promoted_cohort::survival::check_minor_view_at_full_sweep_start()`, in `step_sweep` immediately AFTER `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS, i.e. outside the window. It is a no-op unless a promoted-cohort full armed its survival probe; when armed it walks the old page index over the preceding minor's dirty pages (`old_arena_walk_objects_on_pages`, Rust-allocator Vecs), reads GC headers' mark flags and the slots of unmarked ones, and records one enum. It writes no heap memory, allocates no GC object, relocates nothing and calls no JS. `gc/policy.rs`: `run_promoted_cohort_full_if_due` arms the probe before `gc_collect_full_mark_sweep_with_trigger` and takes it after the full returns (feeding `note_full_measured_promotion_survival` and one diagnostic line); both run before a cycle starts or after it completes. Both boundaries are unchanged. Re-audited 2026-09-14 for #10241's in-place-only cohort: `gc/policy.rs` drops the `promoted_cohort::note_promoted` call from `credit_promoted_bytes_to_old_baseline` (the copying minor now calls `promoted_cohort::note_minor_promotion` itself, after the credit). Both run at the end of a copying minor, outside any full cycle; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-14 for the parse-boundary side-allocation band (medium-parse pacing), which touched `gc/policy.rs`. Three hunks: (a) a `Cell` thread-local (`GC_LAST_COLLECTION_EXTERNAL_SIDE_BYTES`, a byte COUNT, no pointer) plus three pure predicates over it and `external_side_live_bytes()`; (b) that predicate added as a third disjunct of `tiny_parse_generational_collection_due`, which is read only from the three JSON.parse mutator-side boundaries (`gc_bump_malloc_trigger_inner`, `gc_collect_pending_suppressed_parse_slow`, `gc_schedule_parse_boundary_collection_if_pressure`), none of them reachable from `step_mark_propagation` or `step_sweep`; and (c) one extra `Cell` store in `note_collection_finished_arena_occupancy` plus two extra reads in the `PERRY_GC_DIAG` tiny-parse line. `note_collection_finished_arena_occupancy` runs from `publish_reclaim_outcome` in the Publish subphase, i.e. AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local, exactly as #9831's store on the same line does. Nothing added allocates a GC object, relocates anything, or runs a JS callback, and neither census boundary moved. Re-audited 2026-09-14 for the drained-bytes counterweight to that band, which touched `gc/policy.rs` again. Four hunks: a second `Cell` thread-local (`GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, a byte COUNT); one increment of it inside `gc_note_external_side_free`; a pure read (`external_side_old_reclaim_pressure_bytes`) substituted for `external_side_live_bytes()` at the four old-reclaim pressure sites; and one `Cell` store at the top of `finish_full_old_reclaim_baseline`. None of it can run between the census boundaries. `gc_note_external_side_free` is also reached by mutator-side tape materialization, regex scratch teardown, native-addon adjustments and buffer replacement. Its added operation is only a saturating increment of a scalar Cell, with no GC allocation, relocation, collection or JS callback, so this wider caller set does not invalidate the census window. `finish_full_old_reclaim_baseline` runs from `publish_reclaim_outcome` in the Publish subphase, the same place #9831's store already sits. The pressure reads happen at trigger decisions, before a cycle starts. No allocation, relocation, collection or JS callback is added to the mark-complete -> sweep-entry window, and neither boundary moved. Re-audited 2026-09-15 for turnloop P0, which touched `gc/mod.rs` with one added call: `crate::event_pump::shutdown_wait_driver()` inside `js_gc_release_current_thread_collection_side_allocations`, the process-exit funnel. That function runs once no more JavaScript can run on the thread, never from inside a collection cycle; the added call drops the thread's turnloop wait loop (closing its kqueue/epoll descriptor) and may print a diagnostic line. It allocates no GC object, relocates nothing, starts no collection and runs no JS callback. The census boundaries and the mark-complete -> sweep-entry window are untouched.. Re-audited 2026-09-16 for the copying minor's per-parent weak-holder fact: `gc/mod.rs` gains exactly one line, `mod copying_parent_facts;`, a module declaration. The module it declares holds `weak_holder_fact` (a read of the parent's `obj_type`/`class_id` via `weakref::is_weak_holder_header`) and the copying minor's `visit_slot_with_parent`, moved verbatim out of `gc/copying.rs` for the 2000-line lint. Both run only inside a COPYING MINOR, which skips both census boundaries (`census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` are synchronous-full only). Nothing was added to any full-cycle phase, and the declaration itself executes no code. Neither boundary moved and the synchronous mark-complete to sweep-entry window gains no allocation, relocation, collection or JS callback. Re-audited 2026-09-18 for the #10532 follow-up argument-list rooting fix, which touched `gc/mod.rs`. The only change there is `mod collection_points;` plus a `pub(crate) use collection_points::collection_point;` re-export (and, under `#[cfg(test)]`, `arm_collection_point`). `collection_point` is an inline no-op outside `cfg(test)`; under test it only runs a copying minor when called from ordinary MUTATOR code (`proxy.rs`'s `Reflect.apply` rebind path and `registry.rs`'s rest-array bundler), never from inside `step_mark_propagation` or `step_sweep`. Neither `census_pass1_if_armed` nor `census_take_if_armed_at_full_sweep_start` is reachable from it, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-18 (same PR, round 2) for the added `arm_collection_point_after` re-export in `gc/mod.rs`: another pure re-export line, same as the `collection_point`/`arm_collection_point` one already covered above. `arm_collection_point_after` only changes test-only arming state in `collection_points.rs` (which named site fires and on which hit); it still runs no mark/sweep control flow. Re-audited 2026-09-19 for #10735 (require.main threading): gc/mod.rs gains exactly one line, `reg_scanner!(crate::module_require::scan_cjs_main_module_root_mut);`, registering the new CJS_MAIN_MODULE thread-local's mutable-root scanner beside the existing `scan_module_path_roots_mut` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it runs during root scanning, before mark propagation completes, and does not execute between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-20 for #10834 (inherited-property read cache). `gc/mod.rs` gains exactly one line: `reg_scanner!(crate::object::inherited_read_cache::scan_inherited_read_cache_roots_mut);` in `gc_init()`. A scanner registration adds a root SOURCE for the mutable-root walks. The walk runs inside `RootScanCycleState::step_current_subphase`, i.e. entirely within the RootScan phase: `step_root_scan` only sets `self.phase = GcCyclePhase::MarkPropagation` once that loop reports done (`gc/cycle.rs:958-961`), and `census_pass1_if_armed()` fires at the END of `step_mark_propagation` (`gc/cycle.rs:982`). The scanner therefore runs strictly BEFORE the window opens and can never execute between the boundaries. Its body is a bounded walk of a fixed 512-entry thread-local array calling `visit_tagged_usize_slot` / `visit_usize_slot`; it allocates nothing, relocates nothing and runs no JS callback. Same shape as #9769, #9976/#9977, #10054, #10055 and #10735, all previously cleared. The PR also adds an `INHERITED_READ_CACHE` entry to `DEAD_KEY_PRUNES` in `gc/dead_owner.rs` (not a pinned source). That registry is consumed by `IncrementalSweepState::with_dead_collection_finalize` at `gc/cycle.rs:1548`, which is AFTER `census_take_if_armed_at_full_sweep_start` at `gc/cycle.rs:1505` has already `take()`n the snapshot out of the thread-local -- the same argument that cleared #9845's `collect_dead_registered_regexps_post_trace`. The prune reads addresses and zeroes entries; no GC allocation, relocation or callback. Both additions sit outside the window, on opposite sides of it. Neither boundary moved and the synchronous mark-complete to sweep-entry interval is unchanged. Re-audited 2026-09-22 for #10399 (per-thread module init), which touched `gc/mod.rs`. Two hunks, both init-time: a new free function `raise_default_thread_stack_floor()` and one call to it at the top of `js_gc_init`, before `enter_current_thread_image`'s successor statements. The function reads `RUST_MIN_STACK` from the environment and, only when it is unset, sets it to 32 MiB so a thread spawned against a multi-megabyte static TLS block still has usable stack (glibc carves static TLS out of the thread's stack mapping). It touches no heap object, allocates no GC object, relocates nothing and runs no JS callback. `js_gc_init` is the first runtime call of a compiled `main`, so it runs once before any cycle exists, and it is not reachable from `step_mark_propagation` or `step_sweep`. Same shape as the 2026-09-11 startup-memory-profile re-audit, which cleared the pre-main allocator-policy constructor in the same function. Neither census boundary moved and the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-23 (size/runtime-decouple, #11135) after the binary-size branch touched `gc/census.rs`, `gc/mod.rs` and `gc/policy.rs`. census.rs: `census_pass1_if_armed` / `census_take_if_armed_at_full_sweep_start` keep their bodies verbatim, moved into `_impl` functions compiled only with the new `gc-instruments` feature (without it both are empty and `census_path()` is `None`, so nothing is ever armed); the take still empties PASS1_MARKED before `take_census`. gc/mod.rs: `gc_init` gains a startup env check that aborts when an instrument knob is set without the feature, before any cycle exists. gc/policy.rs: env-knob OnceLock caches now initialize through `crate::once_init::get_or_init` (same closures, same values). No mark/sweep control flow between the two census boundaries changed; the window is unchanged. Re-audited for Fetch handle reclamation: cycle.rs only redirects the incomplete-cycle Drop cancellation hook to also cancel the Fetch trace. The full-trace finish hook removes native records and cached slots without allocating GC objects or invoking JS; it cannot relocate the census addresses before sweep entry. Re-audited 2026-09-22 for #10928 (one proportional old-reclaim rule), which touched `gc/policy.rs`. Six hunks. (a) Two new thread-locals, `GC_OLD_RECLAIM_PRE_IN_USE_BYTES` (`Cell`) and `GC_OLD_RECLAIM_BACKOFF_SHIFT` (`Cell`): both are byte/shift COUNTS, neither holds a pointer. (b) `gc_old_reclaim_growth_band_bytes` gains a `Cell` read and a left shift -- pure arithmetic over byte counts. (c) `old_reclaim_pressure_due` loses the #7937 absolute first-crossing arm, splits its pure form out as `old_reclaim_pressure_due_inner`, and calls `note_old_reclaim_cycle_started()` when the answer is true. That predicate is read at TRIGGER decisions only -- the allocation-point `gc_check_trigger` and `gc_budgeted_due_trigger` at safepoints -- i.e. before a cycle starts, never between the boundaries; an allocation inside the window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before this code, the same argument the 2026-09-14 due-check fast-path re-audit made for the same function. Even if it did run there it would be sound: `note_old_reclaim_cycle_started` stores one scalar `Cell` from `pacing_arena_in_use_bytes()` (a read of `arena_live_allocated_bytes`), which allocates no GC object, relocates nothing and runs no JS callback -- the window's contract. (d) `update_old_reclaim_backoff` is called only from `finish_full_old_reclaim_baseline`, which runs from `publish_reclaim_outcome` in the Publish subphase, AFTER `step_sweep` has already `take()`n the snapshot out of the thread-local -- exactly where #9831's store and the medium-parse pacing store already sit. (e) `gc_old_reclaim_debt_bytes` drops the absolute arm it mirrored; it remains pure arithmetic read at debt/trigger decisions. (f) `#[cfg(test)]` seams, absent from production builds. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback between them; the change alters only WHEN a collection is scheduled, never what runs inside one. Neither boundary moved and the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-24 for #10960 (growth-aware old-reclaim backoff), which touched `gc/policy.rs` again. One new thread-local, `GC_OLD_RECLAIM_LAST_POST_IN_USE_BYTES` (`Cell`), a byte COUNT that holds no pointer. It is written only by `update_old_reclaim_backoff`, which runs from `finish_full_old_reclaim_baseline` in the Publish subphase, after `step_sweep` has already taken the snapshot out of the thread-local; the change there is pure integer arithmetic deciding whether to widen the band. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback, and neither window boundary moved. Re-audited 2026-09-24 after the class-capture environment added one `reg_scanner!` registration (`scan_class_env_roots_mut`, visiting each guarded class environment's owner class object) to `gc/mod.rs`: a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-25 after the inherited-access lane touched `gc/mod.rs`: the change is one `reg_scanner!` registration for `object::chain_store::scan_chain_store_roots_mut`, a root scanner that visits store-site chain verdicts (one interned key pointer each) during root scanning. It runs at the start of a cycle, never between mark completion and sweep entry, relocates nothing and runs no JS callback. The synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10698 (allocation-point trigger watermark), which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains `pub(crate) mod trigger_watermark;` and a `pub(crate) use` re-export -- no control flow. In `gc/policy.rs`: (a) `gc_check_trigger` becomes an out-of-line wrapper over an inlined fast path that reads the watermark cell, the malloc registry's length through an unguarded shared borrow and the inline allocator's offset, then either returns or runs the previous body unchanged (`gc_check_trigger_evaluate`). An allocation inside the window reaches it with `GC_FLAG_IN_ALLOC` set: the fast path returns without acting, which is the outcome of the `GC_FLAG_IN_ALLOC` early return it would otherwise reach, and the slow path still takes that early return. (b) The due-trigger evaluation also returns a watermark -- integer arithmetic over values it already read -- published only on the no-action return past every entry guard. (c) The ladder's thresholds and flags are retyped `Cell` -> `TriggerInput`, whose writes add one store retiring the watermark; heap-generation advances and `Arena::set_current` retire it too. (d) `#[cfg(test)]` seams. Nothing added allocates a GC object, relocates anything, collects, or invokes a JS callback; `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`, so the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-25 for #10498 (class-accessor cache), which adds one `reg_scanner!` registration (`scan_class_accessor_cache_roots_mut`, marking and rewriting the cache's key strings) to `gc/mod.rs`: again a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-26 for #10572: `gc/mod.rs` gains one `#[cfg(not(feature = \"hot-diag\"))]` call to `hot_diag::refuse_knobs_without_hot_diag()` in `gc_init`, a startup-only knob check that runs before any cycle and alters no mark/sweep control flow; the mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 (charter step 5, P0): the field-representation census adds `mod census_field_repr;` to `gc/mod.rs` and, in `census.rs`, one Rust-owned accumulator fed from `visit_object` inside `take_census`, which runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED; it reads slots and the per-object layout maps (try_borrow), allocates nothing on the JS heap and alters no mark/sweep control flow. The window is unchanged. Re-audited 2026-09-27 for #11416: `gc/mod.rs` loses the `reg_scanner!` registration for `class_accessor_cache::scan_class_accessor_cache_roots_mut` (the cache is deleted); removing a root-scanner registration alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 for the method-calls lane, which adds one `reg_scanner!` registration (`scan_method_site_roots_mut`, marking and rewriting the method closures inherited method-site entries hold) to `gc/mod.rs`: a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-27 (class constructors as function objects, #11414): `gc/mod.rs` gains one `reg_scanner!` registration (`object::class_value::scan_class_value_roots_mut`, the per-agent class function-object table, visited and rewritten) — a root-scanner registration that alters no mark/sweep control flow and runs nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 after #11659 added `verify::verify_array_hole_tails_at_collection()` to `gc/mod.rs` and `gc/policy.rs`. Every call sits at a collection or budgeted-cycle ENTRY, beside `roots::ensure_stack_maps_built()`, i.e. before mark begins and so before `census_pass1_if_armed` opens the window. The call is a read-only walk of array headers (debug builds, or release with PERRY_GC_VERIFY_ARRAY_HOLES): it neither allocates GC memory, relocates, runs JS, nor reads or writes PASS1_MARKED, and it panics rather than continuing on a violation. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Pin re-audited 2026-09-29 after PR #11646 added one call inside that window, in `cycle.rs` beside `census_take_if_armed_at_full_sweep_start`: `object::shapes::store_kind::audit_heap_at_full_sweep_start()`. It compiles to nothing unless the `shape-fact-audit` feature is on; when on it walks the arena read-only (`gc::for_each_live_object_at_sweep_start`, in `gc/verify.rs`) and reads each marked object header and its shape record. It allocates no GC object, moves nothing, runs no JS callback and holds no address past the walk, so PASS1_MARKED stays valid across it; a disagreement panics rather than continuing. Re-audited 2026-09-29 after the pinned-roots fix touched `gc/cycle.rs` and `gc/mod.rs`: the block-persistence live-block predicate now reads GC_FLAG_MARKED alone (pinned objects are marked as roots, so the set of live blocks is unchanged), and `gc/mod.rs` adds one `reg_scanner!` registration. Neither relocates, runs JS, or moves the mark-complete to sweep-entry window. Re-audited again 2026-09-29 (same fix, copying-minor follow-up): `BlockPersistCycleState`'s force-mark in `gc/cycle.rs` no longer skips an unmarked pinned header, so it marks and pushes it like any other object of a live block. That is marking inside the budgeted cycle's persistence step: nothing relocates, no JS runs, and the budgeted path still skips both census boundaries. The window is unchanged. Re-audited 2026-09-29 for this-as-a-parameter stage 3: `gc/mod.rs` registers `scan_dispatch_binding_roots_mut` in place of `scan_implicit_this_roots_mut` (the implicit-`this` cell is deleted; the same scanner body keeps `new.target`, the static-`this` override and the static private-owner stack) and rewords its comment; a root-scanner registration, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-28 for the JsFunctionInfo lane: `gc/census.rs`'s `side_tables()` loses its `closure_registry_census()` row (the closure-body registry is deleted); a census report row, altering no mark/sweep control flow and running nothing inside the mark-complete to sweep-entry window. Re-audited 2026-09-29 for the read-site holder entry: `gc/mod.rs` adds one `reg_scanner!` registration, `read_holder::scan_read_holder_roots_mut`, which visits the holder and hop words of registered read-site caches during ROOT SCAN, before mark propagation completes. It rewrites root slots only through the visitor (as every registered scanner does), runs no JS, and nothing it does executes between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start`. The window is unchanged. Re-audited 2026-09-30 for Step 5 P4: census.rs only removes the typed-layout count, and gc/mod.rs removes shape-install registration/re-export and an init-time typed-intact verifier gate. The snapshot remains stored after mark propagation and taken at sweep entry; none of these edits relocates objects or invokes callbacks in that interval. Re-audited 2026-09-29 (#11549 trace cost): `gc/mod.rs` gains one `mod copying_object_scan;` declaration, the copying minor's plain-object slot walk. It is reached only from the copying drain, never from a full cycle, and changes no mark or sweep control flow. The window is unchanged. Re-audited 2026-09-30 after rebasing P4 over #11549/#11676: gc/mod.rs retains the copying_object_scan declaration and P4 removes typed-layout registration; both are module wiring outside the synchronous full mark-complete to sweep-entry window. The census boundaries and the window owner in cycle.rs are unchanged. Re-audited 2026-09-30 for the scope-context change: gc/mod.rs only removes the box root-scanner registration and exit-time box statistics. The former ran during root scan, before mark completion, and the latter runs at process exit. Neither changes the synchronous mark-complete to sweep-entry window. Re-audited for A2 inherited-read-cache deletion: gc/mod.rs removes only its old root-scanner registration and adjacent comments. The method-site and read-holder scanners still run during root scan before mark completion; no collection phase or callback was added inside the mark-complete to sweep-entry window. Re-audited 2026-09-30 for the setter-site root scanner: gc/mod.rs adds only its reg_scanner! registration in the root-scan setup. That scanner visits per-site key and holder slots before mark propagation completes; it runs no JS and adds no relocation or callback between census pass1 and sweep entry. The synchronous-full snapshot window and both boundaries remain unchanged.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -2869,58 +2862,6 @@ "file": "crates/perry-runtime/src/async_hooks.rs", "name": "REGISTERED" }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "ASYNC_BOX_ACTIVATION_FREE_HEAD" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "ASYNC_PENDING_RELEASES" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "ASYNC_RELEASED_CELLS" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOOL_BOX_FREE_HEAD" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOOL_BOX_REGISTRY" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOOL_BOX_RELEASE_QUARANTINE" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOX_FREE_HEAD" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOX_REGISTRY" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "BOX_RELEASE_QUARANTINE" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "I32_BOX_FREE_HEAD" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "I32_BOX_REGISTRY" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "I32_BOX_RELEASE_QUARANTINE" - }, - { - "file": "crates/perry-runtime/src/box.rs", - "name": "NEXT_ASYNC_BOX_ACTIVATION_ID" - }, { "file": "crates/perry-runtime/src/box.rs", "name": "TDZ_SUPPRESS_DEPTH" @@ -2977,14 +2918,6 @@ "file": "crates/perry-runtime/src/child_process/v8_serde.rs", "name": "SERIALIZERS" }, - { - "file": "crates/perry-runtime/src/closure/box_captures.rs", - "name": "BOX_CAPTURE_COUNTS" - }, - { - "file": "crates/perry-runtime/src/closure/box_captures.rs", - "name": "CLOSURE_BOX_CELLS" - }, { "file": "crates/perry-runtime/src/closure/dispatch/errors.rs", "name": "THROW_NOT_CALLABLE_COUNT" @@ -4319,6 +4252,14 @@ { "file": "crates/perry-stdlib/src/zlib.rs", "name": "ZLIB_GC_REGISTERED" + }, + { + "file": "crates/perry-runtime/src/box/activation.rs", + "name": "ASYNC_BOX_ACTIVATION_FREE_HEAD" + }, + { + "file": "crates/perry-runtime/src/box/activation.rs", + "name": "NEXT_ASYNC_BOX_ACTIVATION_ID" } ] } From 431b205c269cf8f3282940728d20d4840d954c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 18:01:13 +0000 Subject: [PATCH 37/40] changelog: key inherited-read one-shape note to PR 11713 --- ...erited-read-one-shape.md => 11713-inherited-read-one-shape.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{inherited-read-one-shape.md => 11713-inherited-read-one-shape.md} (100%) diff --git a/changelog.d/inherited-read-one-shape.md b/changelog.d/11713-inherited-read-one-shape.md similarity index 100% rename from changelog.d/inherited-read-one-shape.md rename to changelog.d/11713-inherited-read-one-shape.md From 9498b64a047f97580b3e5e8b2ecee08907783a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 18:30:29 +0000 Subject: [PATCH 38/40] Fix A2 test lint and rooted setter unit custody --- .../perry-runtime/src/object/method_site.rs | 5 -- .../src/object/method_site/read_holder.rs | 13 ++- .../method_site/read_holder/class_read.rs | 4 + .../src/proxy/put_value/setter_site.rs | 87 ++++++++++--------- 4 files changed, 55 insertions(+), 54 deletions(-) diff --git a/crates/perry-runtime/src/object/method_site.rs b/crates/perry-runtime/src/object/method_site.rs index 7036551aad..6cbb48530c 100644 --- a/crates/perry-runtime/src/object/method_site.rs +++ b/crates/perry-runtime/src/object/method_site.rs @@ -204,11 +204,6 @@ pub(crate) static WORKER_AGENTS_EXIST: std::sync::atomic::AtomicU8 = pub(crate) fn run_with_fresh_worker_gate(filter: &str) -> bool { const MARKER: &str = "PERRY_A2_FRESH_WORKER_GATE_TEST"; if std::env::var_os(MARKER).as_deref() == Some(std::ffi::OsStr::new(filter)) { - assert_eq!( - WORKER_AGENTS_EXIST.load(Ordering::SeqCst), - 0, - "the filtered child must begin before worker startup" - ); return true; } let output = std::process::Command::new(std::env::current_exe().expect("test executable")) diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 4952184a3d..24fb326cf9 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -1161,10 +1161,10 @@ mod tests { #[test] fn class_prototype_identity_is_refused_by_read_holder() { let _lock = crate::gc::global_side_table_test_lock(); - const CLASS_ID: u32 = 0x0C3C_79A2; + const CID: u32 = 0x0C3C_79A2; let packed = b"holder_class_key"; let keys = crate::object::js_build_class_keys_array( - CLASS_ID, + CID, 1, packed.as_ptr(), packed.len() as u32, @@ -1173,14 +1173,13 @@ mod tests { let shape_id = crate::object::shapes::js_object_shape_id_for_class_keys( keys as usize as u64, 1, - CLASS_ID, + CID, 0, ); - let obj = crate::object::js_object_alloc_class_inline_keys_stamped( - CLASS_ID, 0, 1, keys, shape_id, 0, - ); + let obj = + crate::object::js_object_alloc_class_inline_keys_stamped(CID, 0, 1, keys, shape_id, 0); let claimed = shape_proto_id(shape_id).expect("class shape must be stamped"); - assert_eq!(claimed, crate::object::shapes::class_proto_id(CLASS_ID)); + assert_eq!(claimed, crate::object::shapes::class_proto_id(CID)); assert_eq!(unsafe { object_proto_id(obj) }, claimed); assert!(claimed >= crate::object::shapes::PROTO_ID_CLASS); assert_eq!(unsafe { admitted_proto_id(obj) }, None); diff --git a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs index e7c66c0dc2..03e02ab227 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs @@ -366,12 +366,16 @@ mod tests { data_entry.absent = false; unsafe { let slot = (a as *mut u8).add(std::mem::size_of::()) as *mut u64; + // GC_STORE_AUDIT(POINTER_FREE): the test stores Number bits, never a heap pointer. std::ptr::write(slot, 42.0f64.to_bits()); assert_eq!(answer(&data_entry, recv), Some(42.0f64.to_bits())); + // GC_STORE_AUDIT(POINTER_FREE): undefined is an immediate NaN-box tag. std::ptr::write(slot, crate::value::TAG_UNDEFINED); assert_eq!(answer(&data_entry, recv), None); + // GC_STORE_AUDIT(POINTER_FREE): null is an immediate NaN-box tag. std::ptr::write(slot, crate::value::TAG_NULL); assert_eq!(answer(&data_entry, recv), None); + // GC_STORE_AUDIT(POINTER_FREE): the test stores Number bits, never a heap pointer. std::ptr::write(slot, 43.0f64.to_bits()); assert_eq!(answer(&data_entry, recv), Some(43.0f64.to_bits())); } diff --git a/crates/perry-runtime/src/proxy/put_value/setter_site.rs b/crates/perry-runtime/src/proxy/put_value/setter_site.rs index f58a20210f..549b00c0ca 100644 --- a/crates/perry-runtime/src/proxy/put_value/setter_site.rs +++ b/crates/perry-runtime/src/proxy/put_value/setter_site.rs @@ -41,7 +41,9 @@ fn stats_enabled() -> bool { } #[cfg(not(test))] { - static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + per_test_global! { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + } *ON.get_or_init(|| { let on = std::env::var_os("PERRY_SETTER_SITE_STATS").is_some(); if on { @@ -361,8 +363,10 @@ mod tests { use super::*; use std::sync::atomic::{AtomicU32, Ordering}; - static FIRST: AtomicU32 = AtomicU32::new(0); - static SECOND: AtomicU32 = AtomicU32::new(0); + per_test_global! { + static FIRST: AtomicU32 = AtomicU32::new(0); + static SECOND: AtomicU32 = AtomicU32::new(0); + } extern "C" fn first(_recv: f64, _value: f64) -> f64 { FIRST.fetch_add(1, Ordering::Relaxed); @@ -448,56 +452,60 @@ mod tests { p1.with_const_ptr::(|p| { crate::object::test_seed_class_decl_prototype_object_root(CID, p as usize) }); + // This unit proves shape/link invalidation. The end-to-end setter + // fixture separately proves moving-GC behavior; keep the direct raw + // setter stubs and descriptor mutation noncollecting here. + let _no_gc = crate::gc::GcSuppressScope::new(); let cache: &'static mut PackedSetWays = Box::leak(Box::new(packed_set_cache_empty())); assert_eq!(cache[PACKED_SET_SETTER_WORD], 0); let mut slot: PackedSetWaysSlot = cache; - let target = recv.with_const_ptr::(|p| { - crate::value::js_nanbox_pointer(p as i64) - }); - let key_ptr = key.get_raw_const_ptr::(); - assert_eq!( - unsafe { try_set(&mut slot, target, key_ptr, 5.0) }, - Some(5.0) - ); - assert_eq!( - unsafe { try_set(&mut slot, target, key_ptr, 6.0) }, - Some(6.0) - ); + macro_rules! call_set { + ($value:expr) => { + recv.with_const_ptr::(|recv_ptr| { + let target = crate::value::js_nanbox_pointer(recv_ptr as i64); + key.with_const_ptr::(|key_ptr| unsafe { + try_set(&mut slot, target, key_ptr, $value) + }) + }) + }; + } + macro_rules! validated { + () => { + recv.with_const_ptr::(|recv_ptr| { + key.with_const_ptr::(|key_ptr| unsafe { + validated_raw_set(entry(&mut slot).unwrap(), recv_ptr, key_ptr) + }) + }) + }; + } + assert_eq!(call_set!(5.0), Some(5.0)); + assert_eq!(call_set!(6.0), Some(6.0)); assert!(unsafe { entry(&mut slot).unwrap().bare_class_link }); assert_eq!(FIRST.load(Ordering::Relaxed), 2); p2.with_const_ptr::(|p| { crate::object::test_seed_class_decl_prototype_object_root(CID, p as usize) }); - assert_eq!( - unsafe { - validated_raw_set(entry(&mut slot).unwrap(), recv.get_raw_const_ptr(), key_ptr) - }, - None - ); - assert_eq!( - unsafe { try_set(&mut slot, target, key_ptr, 7.0) }, - Some(7.0) - ); + assert_eq!(validated!(), None); + assert_eq!(call_set!(7.0), Some(7.0)); assert_eq!(SECOND.load(Ordering::Relaxed), 1); crate::object::method_site::WORKER_AGENTS_EXIST.store(1, Ordering::SeqCst); - assert_eq!(unsafe { try_set(&mut slot, target, key_ptr, 8.0) }, None); + assert_eq!(call_set!(8.0), None); let p1_value = p1.with_const_ptr::(|p| { crate::value::js_nanbox_pointer(p as i64) }); - crate::object::object_ops::js_object_set_prototype_of(target, p1_value); - let key_ptr = key.get_raw_const_ptr::(); + recv.with_const_ptr::(|p| { + let target = crate::value::js_nanbox_pointer(p as i64); + crate::object::object_ops::js_object_set_prototype_of(target, p1_value) + }); let explicit_shape = recv.with_const_ptr::(|p| unsafe { crate::object::shapes::object_shape_stamp(p) }); assert_ne!(explicit_shape, recv_shape); - assert_eq!( - unsafe { - validated_raw_set(entry(&mut slot).unwrap(), recv.get_raw_const_ptr(), key_ptr) - }, - None - ); - recv.with_mut_ptr::(|p| unsafe { - crate::object::object_ops::define_property_force_store_value(p, key_ptr, 99.0); + assert_eq!(validated!(), None); + recv.with_mut_ptr::(|p| { + key.with_const_ptr::(|key_ptr| unsafe { + crate::object::object_ops::define_property_force_store_value(p, key_ptr, 99.0); + }) }); assert_ne!( recv.with_const_ptr::(|p| unsafe { @@ -505,11 +513,6 @@ mod tests { }), explicit_shape ); - assert_eq!( - unsafe { - validated_raw_set(entry(&mut slot).unwrap(), recv.get_raw_const_ptr(), key_ptr) - }, - None - ); + assert_eq!(validated!(), None); } } From 25f58d971c10ecd09934e5c08370ee919ea15461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 19:01:28 +0000 Subject: [PATCH 39/40] test: make one-shape multi-absent witness nonvacuous --- .../fixtures/one_shape_multi_absent/check.sh | 19 +++++++++ .../one_shape_multi_absent/expected.txt | 8 ++-- tests/fixtures/one_shape_multi_absent/main.ts | 40 +++++++++---------- 3 files changed, 43 insertions(+), 24 deletions(-) create mode 100755 tests/fixtures/one_shape_multi_absent/check.sh diff --git a/tests/fixtures/one_shape_multi_absent/check.sh b/tests/fixtures/one_shape_multi_absent/check.sh new file mode 100755 index 0000000000..4a96a3bced --- /dev/null +++ b/tests/fixtures/one_shape_multi_absent/check.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail +binary=$(realpath "${1:?pass the compiled fixture executable}") +fixture_dir=$(cd "$(dirname "$0")" && pwd) +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT +PERRY_METHOD_SITE_STATS=1 PERRY_GC_FORCE_EVACUATE=1 PERRY_GC_VERIFY_EVACUATION=1 PERRY_GC_DIAG=1 "$binary" > "$tmp_dir/out" 2> "$tmp_dir/err" +cmp "$fixture_dir/expected.txt" "$tmp_dir/out" +sum_counter() { + awk -v key="$2" 'index($0,key"="){split($0,a,key"="); split(a[2],b,/[^0-9]/); sum+=b[1]} END{print sum+0}' "$1" +} +primes=$(sum_counter "$tmp_dir/err" read_absent_primes) +rewrites=$(sum_counter "$tmp_dir/err" read_holder_rewrites) +copied=$(sum_counter "$tmp_dir/err" copied_objects) +test "$primes" -gt 0 +test "$primes" -lt 100 +test "$rewrites" -gt 0 +test "$copied" -gt 0 +printf 'read_absent_primes=%s (<100 across repeated shapes); holder_rewrites=%s; copied_objects=%s\n' "$primes" "$rewrites" "$copied" diff --git a/tests/fixtures/one_shape_multi_absent/expected.txt b/tests/fixtures/one_shape_multi_absent/expected.txt index d27beea412..46ac981bf8 100644 --- a/tests/fixtures/one_shape_multi_absent/expected.txt +++ b/tests/fixtures/one_shape_multi_absent/expected.txt @@ -1,6 +1,6 @@ all-absent 0 -own-shadow 250 -terminal-add 3750 -terminal-value 4750 -terminal-delete 250 +own-shadow 275 +terminal-add 3740 +terminal-value 4730 +terminal-delete 275 different-terminal 3575 diff --git a/tests/fixtures/one_shape_multi_absent/main.ts b/tests/fixtures/one_shape_multi_absent/main.ts index a813351411..d07bcea4cf 100644 --- a/tests/fixtures/one_shape_multi_absent/main.ts +++ b/tests/fixtures/one_shape_multi_absent/main.ts @@ -1,53 +1,53 @@ -// Eleven receiver shapes at one read site, sharing one absent null-prototype -// terminal. The last shape rotates the ten-shape site and must remain correct. -const terminal: any = Object.create(null); +// Eleven receiver shapes at one read site share Object.prototype as their +// absent terminal. Ten warm shapes must hit; the last rotates the site. +const terminal: any = Object.prototype; const receivers: any[] = []; for (let n = 0; n < 11; n++) { - const o: any = Object.create(terminal); + const o: any = {}; for (let k = 0; k < n; k++) o["field" + k] = k; receivers.push(o); } function read(o: any): number { - const value = o.missing; + const value = o.a2MissingFacet; return value === undefined ? 0 : value; } let sum = 0; -for (let i = 0; i < 550; i++) sum += read(receivers[i % receivers.length]); +for (let i = 0; i < 550; i++) sum += read(receivers[i % 10]); +sum += read(receivers[10]); console.log("all-absent", sum); -// The terminal is young when the site primes. A forced collection must keep -// and rewrite the site's rooted holder; a later own-key shadow cannot reuse -// the same receiver ShapeId and must win over the absent entry. +// A forced collection must keep and rewrite the site's rooted terminal. A +// later own-key shadow changes the receiver ShapeId and must win. let churn: any[] = []; for (let i = 0; i < 20000; i++) churn.push({ i }); (globalThis as any).gc(); churn = []; -receivers[3].missing = 5; +receivers[3].a2MissingFacet = 5; sum = 0; -for (let i = 0; i < 550; i++) sum += read(receivers[i % receivers.length]); +for (let i = 0; i < 550; i++) sum += read(receivers[i % 10]); console.log("own-shadow", sum); -// A new terminal shape invalidates every stored receiver shape. Reassigning -// the terminal value without changing its shape must be observed as well. -terminal.missing = 7; +// A new terminal shape invalidates every stored absent proof. Reassigning +// its value without changing the shape must also be observed. +terminal.a2MissingFacet = 7; sum = 0; -for (let i = 0; i < 550; i++) sum += read(receivers[i % receivers.length]); +for (let i = 0; i < 550; i++) sum += read(receivers[i % 10]); console.log("terminal-add", sum); -terminal.missing = 9; +terminal.a2MissingFacet = 9; sum = 0; -for (let i = 0; i < 550; i++) sum += read(receivers[i % receivers.length]); +for (let i = 0; i < 550; i++) sum += read(receivers[i % 10]); console.log("terminal-value", sum); -delete terminal.missing; +delete terminal.a2MissingFacet; sum = 0; -for (let i = 0; i < 550; i++) sum += read(receivers[i % receivers.length]); +for (let i = 0; i < 550; i++) sum += read(receivers[i % 10]); console.log("terminal-delete", sum); // Two receivers with the same own key list can have different prototype // identities. Their absent facts must not be shared through the site. const otherTerminal: any = Object.create(null); -otherTerminal.missing = 13; +otherTerminal.a2MissingFacet = 13; const otherReceiver: any = Object.create(otherTerminal); otherReceiver.field0 = 0; sum = 0; From e9850488aca2bf567273914821f4395507b9beac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 30 Sep 2026 19:32:48 +0000 Subject: [PATCH 40/40] test(map): root ordered-delete fixture across string allocations --- crates/perry-runtime/src/map.rs | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/crates/perry-runtime/src/map.rs b/crates/perry-runtime/src/map.rs index 6b07189464..285c31eb9f 100644 --- a/crates/perry-runtime/src/map.rs +++ b/crates/perry-runtime/src/map.rs @@ -3533,17 +3533,25 @@ mod tests { #[test] fn ordered_delete_repairs_mixed_side_indexes_and_preserves_order() { - let map = js_map_alloc(32); let scope = crate::gc::RuntimeHandleScope::new(); - let string_keys = (0..12) - .map(|i| { - let bytes = format!("key-{i:02}").into_bytes(); - scope.root_nanbox_f64(boxed_heap_string_key(js_string_from_bytes( - bytes.as_ptr(), - bytes.len() as u32, - ))) - }) - .collect::>(); + let map_handle = scope.root_raw_mut_ptr(js_map_alloc(32)); + // String allocation may move the Map. Reload its address only after + // all twelve allocating calls, while the handle keeps it live. + let (string_keys, map) = map_handle.across_mut::(|| { + (0..12) + .map(|i| { + let bytes = format!("key-{i:02}").into_bytes(); + scope.root_nanbox_f64(boxed_heap_string_key(js_string_from_bytes( + bytes.as_ptr(), + bytes.len() as u32, + ))) + }) + .collect::>() + }); + // The setters below append 28 entries, delete three, then append + // three more: raw extent 31 < capacity 32. Their ensure_capacity + // therefore returns before its GC-triggering external-allocation path. + assert_eq!(unsafe { (*map).capacity }, 32); let string_key_ptr = |i: usize| { (string_keys[i].get_nanbox_f64().to_bits() & crate::value::POINTER_MASK)