From 056abd7a751819d7f42bb3e07a4a1a50c80db558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 23 Sep 2026 18:13:42 +0000 Subject: [PATCH 01/16] perf(runtime): one string per property-key text, so a key confirm is a pointer compare A canonical key list stored whichever string its first grower passed, and a read site holds its module's pooled literal: two objects with the same bytes. Every key match against a shape's list therefore fell through to a byte compare, including the megamorphic read's confirm of its slot guess. Pool literals of at most 64 bytes are now minted as ATOMS at module init (js_string_pool_atom): the one string object for that text in the agent, shared by every module's pool. The intern cache's miss paths hand out the atom for its text, and canonical lists write the atom of every key they store (Appended::atomized on extend_slot's write paths and canonicalize's copy). The trie still validates edges by bytes, so which object a list holds never changes which node a probe reaches. The atom table is per agent, bounded by program text, strong (every atom is also a registered pool handle's value) and rewritten on move by the intern table root scanner. A pointer match proves equal text; a mismatch proves nothing (a list written before its atom existed), so every consumer keeps its byte fallback. The megamorphic shape answer now scans for identity before it compares any bytes. --- .../perry-codegen/src/codegen/string_pool.rs | 33 ++- crates/perry-codegen/src/runtime_decls/mod.rs | 4 + .../src/gc/tests/canonical_keys_holders.rs | 136 +++++++++ .../src/object/canonical_keys.rs | 47 ++++ .../object/field_get_set/ic_miss/ic_slow.rs | 166 +++++++++++ crates/perry-runtime/src/object/shapes.rs | 30 +- crates/perry-runtime/src/string/intern.rs | 257 ++++++++++++++++++ crates/perry-runtime/src/string/mod.rs | 4 + 8 files changed, 661 insertions(+), 16 deletions(-) diff --git a/crates/perry-codegen/src/codegen/string_pool.rs b/crates/perry-codegen/src/codegen/string_pool.rs index 079c4ce0ec..5c82c68741 100644 --- a/crates/perry-codegen/src/codegen/string_pool.rs +++ b/crates/perry-codegen/src/codegen/string_pool.rs @@ -96,6 +96,13 @@ impl<'a> InitChunker<'a> { } } +/// Pool literals at most this long are minted as ATOMS. **Must equal +/// `INTERN_MAX_BYTE_LEN` in `perry-runtime/src/string/intern.rs`**, the +/// longest key the runtime interns; `js_string_pool_atom` falls back to a +/// plain allocation past it, so a mismatch costs only the atom, never +/// correctness. +pub(crate) const POOL_ATOM_MAX_BYTE_LEN: usize = 64; + /// Emit the string pool into the module: byte-array constants, handle /// globals, and the `__perry_init_strings_` function that /// allocates + NaN-boxes + GC-roots each handle exactly once at startup. @@ -446,12 +453,30 @@ pub(super) fn emit_string_pool( let bytes_ref = format!("@{}", entry.bytes_global); let handle_ref = format!("@{}", entry.handle_global); let len_str = entry.byte_len.to_string(); - let from_bytes_fn = if entry.is_wtf8 { - "js_string_from_wtf8_bytes" + // A literal short enough to be a property key becomes its text's ATOM + // (`js_string_pool_atom`): one string object per key text for the + // whole agent, shared by every module's pool, every canonical shape + // key list and every intern hit — so a read site's key and the + // receiver's shape key compare by pointer (S3b). Longer literals, and + // WTF-8 ones (lone surrogates: never an identifier key), keep the + // plain allocation. + let atomize = + !entry.is_wtf8 && entry.byte_len > 0 && entry.byte_len <= POOL_ATOM_MAX_BYTE_LEN; + let handle = if atomize { + let hash = crate::nanbox::i64_literal(entry.dispatch_hash); + blk.call( + I64, + "js_string_pool_atom", + &[(PTR, &bytes_ref), (I32, &len_str), (I64, &hash), (I32, "0")], + ) } else { - "js_string_from_bytes" + let from_bytes_fn = if entry.is_wtf8 { + "js_string_from_wtf8_bytes" + } else { + "js_string_from_bytes" + }; + blk.call(I64, from_bytes_fn, &[(PTR, &bytes_ref), (I32, &len_str)]) }; - let handle = blk.call(I64, from_bytes_fn, &[(PTR, &bytes_ref), (I32, &len_str)]); let nanboxed = blk.call(DOUBLE, "js_nanbox_string", &[(I64, &handle)]); // Plain store, no remembered-set write barrier: the handle slot is // registered as a permanent global root on the very next line (always diff --git a/crates/perry-codegen/src/runtime_decls/mod.rs b/crates/perry-codegen/src/runtime_decls/mod.rs index 72d544d6fc..9d9de7474a 100644 --- a/crates/perry-codegen/src/runtime_decls/mod.rs +++ b/crates/perry-codegen/src/runtime_decls/mod.rs @@ -159,6 +159,10 @@ pub fn declare_phase1(module: &mut LlModule) { // Strings (enough to produce string literals for later phases). module.declare_function("js_string_from_bytes", I64, &[PTR, I32]); module.declare_function("js_string_from_wtf8_bytes", I64, &[PTR, I32]); + // S3b: a pooled literal short enough to be a key is minted as the atom of + // its text (`string/intern.rs::js_string_pool_atom`): bytes, len, FNV-1a + // hash, is_wtf8. + module.declare_function("js_string_pool_atom", I64, &[PTR, I32, I64, I32]); // Type checks. module.declare_function("js_is_truthy", I32, &[DOUBLE]); diff --git a/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs b/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs index e1a3e51b23..c70c672600 100644 --- a/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs +++ b/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs @@ -657,3 +657,139 @@ fn a_dead_weak_keys_entry_is_dropped_before_its_storage_is_reused() { .remove(&DEAD_MEMO_CLASS_ID); canonical_keys::reset_for_test(); } + +// ------------------------------------------------------- S3b: key atoms + +fn atom_hash(text: &[u8]) -> u64 { + crate::object::key_bytes_hash(text.as_ptr(), text.len()) +} + +fn atom_bits(atom: usize) -> u64 { + crate::value::js_nanbox_string(atom as i64).to_bits() +} + +/// S3b: a key text has ONE string object in an agent — its atom — and every +/// canonical list written after the atom exists holds it. The atom table holds +/// its strings strongly and REWRITES them on a move (the intern-table root +/// scanner), so after a moving minor the table, the list, a fresh intern and a +/// fresh pool mint all name the atom at its NEW address, and none names the +/// address it moved away from. +#[test] +fn an_atom_and_the_lists_holding_it_follow_a_moving_minor() { + let _guard = CopyingNurseryTestGuard::new(0); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + register_object_model_scanners(); + gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut); + canonical_keys::reset_for_test(); + let scope = RuntimeHandleScope::new(); + let text = b"atom_mv_kind"; + let hash = atom_hash(text); + unsafe { + let atom = + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize; + assert!( + crate::arena::pointer_in_nursery(atom), + "premise: the atom is young, so a minor can move it" + ); + // A receiver grows the key through a DIFFERENT string with the text. + let copy = nursery_key("atom_mv_kind"); + assert_ne!(copy as usize, atom, "premise: two string objects, one text"); + let o = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0)); + set(o, nursery_key("atom_mv_a"), 1.0); + set(o, copy, 2.0); + let list = keys_of(o); + assert_eq!( + crate::array::js_array_get(list, 1).bits(), + atom_bits(atom), + "INVARIANT: the written list holds the atom, not the grower's copy" + ); + + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert!( + trace.copying_nursery.copied_objects > 0, + "premise: the minor copied" + ); + let moved = crate::string::atom_lookup(text, hash).expect("the atom survives") as usize; + assert_ne!(moved, atom, "premise: the minor moved the atom"); + let header = crate::value::addr_class::try_read_tracked_gc_header(moved) + .expect("the moved atom is a tracked cell"); + assert_eq!( + (*header.as_ptr()).gc_flags & GC_FLAG_FORWARDED, + 0, + "INVARIANT: the table names the live copy, not a forwarding header" + ); + assert_eq!( + crate::array::js_array_get(keys_of(o), 1).bits(), + atom_bits(moved), + "INVARIANT: the list follows its atom through the move" + ); + // After the move: interning another copy, and minting again, both + // return the moved atom — never a third string. + assert_eq!( + crate::string::js_string_intern(nursery_key("atom_mv_kind"), hash) as usize, + moved, + "a copy interns to the moved atom" + ); + assert_eq!( + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize, + moved, + "a second mint finds the moved atom" + ); + // A list written after the move holds the moved atom. + let o2 = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0)); + set(o2, nursery_key("atom_mv_b"), 1.0); + set(o2, nursery_key("atom_mv_kind"), 2.0); + assert_eq!( + crate::array::js_array_get(keys_of(o2), 1).bits(), + atom_bits(moved), + "INVARIANT: a list written after the move holds the moved atom" + ); + } +} + +/// S3b, a collection DURING the write: the canonical backing allocation that +/// publishes a list holding an atom is itself the collection point, and moves +/// the atom. The published list must hold the atom's LIVE address. +#[test] +fn a_list_written_while_its_atom_moves_holds_the_live_atom() { + let _guard = CopyingNurseryTestGuard::new(0); + let _pacing = crate::gc::policy::force_alloc_point_minor_pacing(); + let _scan = NoConservativeScan::new(); + let trigger = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + register_object_model_scanners(); + gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut); + canonical_keys::reset_for_test(); + let text = b"atom_during_kind"; + let hash = atom_hash(text); + unsafe { + let atom = + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize; + assert!( + crate::arena::pointer_in_nursery(atom), + "premise: the atom is young" + ); + let scope = RuntimeHandleScope::new(); + let copy = scope.root_string_ptr(nursery_key("atom_during_kind")); + // The empty list plus this key is a fresh backing: its allocation is + // the collection point. + arm_collection_on_next_block(&trigger); + let list = copy.with_const_ptr(|k: *const crate::StringHeader| { + canonical_keys::extend_key( + &SharedLayout::shape_cache_entry(), + canonical_keys::CanonicalKeys::EMPTY, + k, + ) + }); + let live = crate::string::atom_lookup(text, hash).expect("the atom survives") as usize; + assert_ne!( + live, atom, + "premise: the backing allocation did not move the atom, so this run proved \ + nothing. Check the trigger arming." + ); + assert_eq!( + crate::array::js_array_get(list.as_ptr(), 0).bits(), + atom_bits(live), + "INVARIANT: the list holds the atom's live address" + ); + } +} diff --git a/crates/perry-runtime/src/object/canonical_keys.rs b/crates/perry-runtime/src/object/canonical_keys.rs index c3f8a6c164..00704f8880 100644 --- a/crates/perry-runtime/src/object/canonical_keys.rs +++ b/crates/perry-runtime/src/object/canonical_keys.rs @@ -676,6 +676,37 @@ impl Appended { } } + /// The same slot, spelled with its text's ATOM when one exists — the one + /// string a read site's pooled key also is (`string::intern::AtomTable`). + /// + /// Applied to every key this module WRITES into a list, and nowhere else: + /// the trie validates edges by bytes, so which string object a list holds + /// never changes which node a probe reaches, only whether a later key + /// compare against the list can stop at pointer equality. Heap strings + /// only — an SSO slot stays an SSO slot, so `is_pointer` (and with it the + /// backing's all-pointer layout) is unchanged by the substitution. + /// + /// `h` is this slot's `edge_hash`, which for a string IS the FNV-1a hash of + /// its bytes — the atom table's hash. + /// + /// # Safety + /// The operand is live. + unsafe fn atomized(self, h: u64) -> Self { + match self { + Appended::Key(key) if !key.is_null() => match crate::string::atom_for_key(key, h) { + Some(atom) => Appended::Key(atom), + None => self, + }, + Appended::Slot(v) if v.is_string() => { + match crate::string::atom_for_key(v.as_string_ptr(), h) { + Some(atom) => Appended::Slot(JSValue::string_ptr(atom as *mut StringHeader)), + None => self, + } + } + _ => self, + } + } + /// Is the appended slot a heap string POINTER? An SSO short string and a /// tombstone are not, and either one costs the child its all-pointer /// layout — see `Node::all_ptr`. @@ -789,6 +820,8 @@ pub(crate) unsafe fn extend_slot( if let Some(hit) = probe(parent, parent_len, appended, entry, h) { return hit; } + // A new list is about to be WRITTEN: it holds the atom of this key's text. + let appended = appended.atomized(appended.slot_hash()); // Whether the child's slots are all heap string pointers is the parent's // answer AND this slot's, so it is read before the allocation and never @@ -1180,6 +1213,20 @@ pub(crate) unsafe fn canonicalize( // GC_STORE_AUDIT(INIT): fresh is unpublished; publish length only after // all elements are initialized, with no intervening GC allocation. std::ptr::copy_nonoverlapping(slots, dst, len as usize); + // Every heap key of the new list is its text's atom where one exists (see + // `Appended::atomized`). Heap string to heap string, so `all_ptr` holds. + for i in 0..len as usize { + let slot = Appended::Slot(JSValue::from_bits((*dst.add(i)).to_bits())); + if let Appended::Slot(v) = slot { + if v.is_string() { + if let Appended::Slot(atom) = slot.atomized(slot.slot_hash()) { + // GC_STORE_AUDIT(INIT): `fresh` is unpublished; its length + // is set on the next line, after the last slot is written. + *dst.add(i) = f64::from_bits(atom.bits()); + } + } + } + } if with_attrs { let attrs = crate::object::key_attrs::keys_attrs(fresh); crate::object::key_attrs::copy_entries(keys, 0, attrs, len); diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 50e821cb2d..09f0fc5b8a 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -525,6 +525,172 @@ mod tests { } } + /// `megamorphic_receivers` with a chosen text for the read key (slot 2), + /// each receiver growing that key through its OWN freshly allocated string + /// — never an atom — so which string a shape's list ends up holding is + /// decided by the list writer, not by the test. + fn megamorphic_receivers_keyed<'s>( + scope: &'s crate::gc::RuntimeHandleScope, + n: usize, + read_key: &[u8], + extra_prefix: &str, + ) -> Vec> { + let mut out = Vec::new(); + for i in 0..n { + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + for (k, v) in [ + (&b"pos"[..], 1.0), + (&b"end"[..], 2.0), + (read_key, 100.0 + i as f64), + ] { + let key = scope.root_string_ptr(key_of(k)); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + let extra = format!("{extra_prefix}{i}"); + let key = scope.root_string_ptr(key_of(extra.as_bytes())); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, 7.0)) + }); + out.push(obj); + } + out + } + + fn atom_of(text: &[u8]) -> *mut crate::StringHeader { + let hash = crate::object::key_bytes_hash(text.as_ptr(), text.len()); + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) + } + + fn answered() -> u64 { + crate::object::shapes::SHAPE_ANSWERED_READS.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Read `key` at one latched site over `objs` until it latches, then once + /// more; every read must return `expect(i)`. Returns the reads the + /// receivers' SHAPES answered on the latched pass. + fn latched_pass( + objs: &[crate::gc::RuntimeHandle<'_>], + key: &crate::gc::RuntimeHandle<'_>, + expect: impl Fn(usize) -> u64, + ) -> u64 { + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let read = |o: &crate::gc::RuntimeHandle<'_>, slot: &mut PicCacheSlot| { + o.with_mut_ptr(|p: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(p), k, slot, &packed)) + }) + }; + for round in 0..4 { + for (i, o) in objs.iter().enumerate() { + assert_eq!( + read(o, &mut slot).to_bits(), + expect(i), + "round {round} receiver {i}" + ); + } + } + assert!( + cache[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] < 0, + "premise: the site latched megamorphic" + ); + let before = answered(); + for (i, o) in objs.iter().enumerate() { + assert_eq!( + read(o, &mut slot).to_bits(), + expect(i), + "latched read, receiver {i}" + ); + } + answered() - before + } + + /// S3b: one key text reaching the runtime as TWO string objects — each + /// receiver grows it through its own fresh copy — is still ONE string in + /// every shape's key list: the text's atom, the same object a read site's + /// pooled key is. So the site matches by pointer; and a site holding yet + /// another copy (not the atom) still gets the right answer by bytes. + #[test] + fn a_key_text_in_two_string_objects_is_one_atom_in_every_shape() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_two_objects_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let copy = scope.root_string_ptr(key_of(text)); + let addr = |h: &crate::gc::RuntimeHandle<'_>| { + h.with_const_ptr(|p: *const crate::StringHeader| p as usize) + }; + assert_ne!( + addr(&atom), + addr(©), + "premise: two string objects, one text" + ); + assert!( + unsafe { crate::string::is_atom_for_test(atom.with_const_ptr(|p| p)) }, + "premise: the pool mint made an atom" + ); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_two_x"); + let atom_bits = crate::value::js_nanbox_string(addr(&atom) as i64).to_bits(); + for (i, o) in objs.iter().enumerate() { + let stored = o.with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::object_keys(p).get(2).bits() + }); + assert_eq!( + stored, atom_bits, + "INVARIANT: receiver {i}'s shape holds the atom, not the copy it grew with" + ); + } + // A computed key with the text interns TO the atom. + let interned = copy.with_const_ptr(|p: *const crate::StringHeader| { + crate::string::js_string_intern( + p, + crate::object::key_bytes_hash(text.as_ptr(), text.len()), + ) as usize + }); + assert_eq!(interned, addr(&atom), "a copy interns to the atom"); + // The site holds the atom: every latched read is the receiver's own + // value, answered by its shape. + let by_atom = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + by_atom >= 47, + "the shape answers a pointer-equal key: {by_atom}" + ); + // A site holding a different string object with the same text (not + // the atom): the answer is the same, found by bytes. + let fresh = scope.root_string_ptr(key_of(text)); + let by_bytes = latched_pass(&objs, &fresh, |i| (100.0 + i as f64).to_bits()); + assert!( + by_bytes >= 47, + "a non-atom key text still matches by bytes: {by_bytes}" + ); + } + + /// A list written BEFORE its key's atom existed holds another string. A + /// pointer mismatch is not an answer: the shape still answers by bytes. + #[test] + fn a_list_written_before_the_atom_existed_still_answers() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_pre_atom_kind"; + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_pre_x"); + // The lists hold the string the grow path interned; a collision + // evicts it from the cache before the atom is minted, so the atom is + // a different object. + crate::string::test_evict_interned(text); + let atom = scope.root_string_ptr(atom_of(text)); + let atom_bits = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let stored = objs[0].with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::object_keys(p).get(2).bits() + }); + assert_ne!(stored, atom_bits, "premise: the list predates the atom"); + let n = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!(n >= 47, "the shape answers by bytes: {n}"); + } + /// A plain own data read that has never primed: the entry must fall all the /// way through to the miss handler, answer the field, and leave the site /// primed exactly as the old `js_object_get_field_ic_miss_packed` edge did. diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 2a62a4a1e9..4f87d4d02e 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -257,11 +257,12 @@ impl ShapeRecordRef { /// SHAPE's key count, never the backing's length (#10969: one backing per /// growth chain). /// - /// A stored key matches the site's key by identity, or else by (byte - /// length, bytes): a canonical list holds the string its first grower - /// passed, which is usually NOT the read site's pooled literal. The - /// site's slot guess is tried first. Allocation-free, never calls user - /// code. + /// Key compares go identity first: canonical lists hold their text's ATOM + /// (`string::intern::AtomTable`), which is also what a read site's pooled + /// key is, so the site's guess, and then any position, matches by one + /// pointer compare. A byte pass remains for a list written before its + /// atom existed (and for SSO slots) — a pointer MISmatch proves nothing. + /// Allocation-free, never calls user code. #[inline] pub(crate) unsafe fn inline_slot_of_key( self, @@ -284,11 +285,16 @@ impl ShapeRecordRef { let bound = len .min(r.logical_key_count as usize) .min(r.live_inline_slot_count as usize); + let heap_bits = crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits(); // The site's slot guess first: the receiver's shape confirms it. - if hint < bound && stored_key_matches(key, (*slots.add(hint)).to_bits()) { + if hint < bound && (*slots.add(hint)).to_bits() == heap_bits { return Some(hint); } - (0..bound).find(|&i| i != hint && stored_key_matches(key, (*slots.add(i)).to_bits())) + // Identity over the whole list before any byte is compared. + if let Some(i) = (0..bound).find(|&i| (*slots.add(i)).to_bits() == heap_bits) { + return Some(i); + } + (0..bound).find(|&i| stored_key_matches(key, (*slots.add(i)).to_bits())) } /// The SPILL position at which this shape stores `key` as an own DATA @@ -333,11 +339,11 @@ impl ShapeRecordRef { } } -/// Does the key-list entry `bits` name `key`? A canonical list holds heap -/// strings of its own (NOT the site's pooled key — measured: every stored key -/// of a literal-born shape is a distinct heap string) or SSO immediates, so a -/// stored key matches by identity, by SSO identity, or by (byte length, -/// bytes). +/// Does the key-list entry `bits` name `key`? A canonical list holds its +/// text's ATOM where one exists (the site's pooled key), but must not be +/// assumed to: a list written before its atom existed holds another heap +/// string, and a slot may be an SSO immediate. So a stored key matches by +/// identity, by SSO identity, or by (byte length, bytes). #[inline] unsafe fn stored_key_matches(key: *const crate::StringHeader, bits: u64) -> bool { if bits == crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits() { diff --git a/crates/perry-runtime/src/string/intern.rs b/crates/perry-runtime/src/string/intern.rs index da9d2ef971..5083c2e22c 100644 --- a/crates/perry-runtime/src/string/intern.rs +++ b/crates/perry-runtime/src/string/intern.rs @@ -36,6 +36,229 @@ crate::perry_thread_local! { std::cell::UnsafeCell::new(crate::zeroed_cache::new_zeroed_cache(INTERN_TABLE_SIZE)); } +/// Property-key ATOMS: exactly one string object per key text among the +/// program's pooled literals. +/// +/// The direct-mapped table above is a CACHE — a collision evicts — so two +/// strings with one text can both be "interned" at different moments, and a +/// key compare can never conclude anything from pointer equality alone. That +/// is why every runtime key match, and the megamorphic read's confirm against +/// the receiver's key list, paid a byte compare: a canonical key list held the +/// string its first grower happened to pass, and a read site holds its +/// module's pooled literal, and the two were different objects with the same +/// bytes. +/// +/// An atom is the one string for its text in this agent, for the agent's +/// lifetime. Atoms are minted only from the compiled program's string pools +/// (`js_string_pool_atom`, at module init), so the table is bounded by the +/// program TEXT, never by runtime data, and it can hold its strings strongly +/// without a death prune: every atom is also the value of a registered pool +/// handle, so the table keeps nothing alive that was not already live. The +/// collector rewrites the entries on move through the intern-table root +/// scanner (`scan_intern_table_roots_mut`), exactly like the cache's. +/// +/// Two funnels consult it: the intern cache's miss paths (so a computed key +/// with an atom's text interns TO the atom), and canonical key lists when they +/// write a key (`canonical_keys::Appended::atomized`), so a read site's pooled +/// key and the receiver's shape key are one pointer. +/// +/// The table never decides an answer. A pointer match proves equal text; a +/// pointer MISmatch proves nothing (a list written before its atom existed +/// holds another string), so every consumer still falls back to bytes on a +/// mismatch. +pub(crate) struct AtomTable { + /// Open addressing, linear probe, power-of-two capacity; `string_ptr == 0` + /// is an empty slot. Entries are never removed. + slots: Vec, + len: usize, +} + +impl AtomTable { + const fn new() -> Self { + AtomTable { + slots: Vec::new(), + len: 0, + } + } + + /// The atom for `bytes` (whose FNV-1a hash is `hash`), if any. + /// + /// # Safety + /// Every tabled pointer is a live string (the collector keeps them so). + unsafe fn lookup(&self, bytes: &[u8], hash: u64) -> Option<*const StringHeader> { + self.lookup_from(0, bytes, hash) + } + + /// [`Self::lookup`] for a string that may itself BE the atom (`from`, or + /// 0): a tabled pointer equal to it answers without comparing bytes, which + /// is the common case for a key a list already holds. + /// + /// # Safety + /// As [`Self::lookup`]. + unsafe fn lookup_from( + &self, + from: usize, + bytes: &[u8], + hash: u64, + ) -> Option<*const StringHeader> { + if self.slots.is_empty() { + return None; + } + let mask = self.slots.len() - 1; + let mut i = (hash as usize) & mask; + loop { + let entry = &self.slots[i]; + if entry.string_ptr == 0 { + return None; + } + if entry.hash == hash { + let existing = entry.string_ptr as *const StringHeader; + if entry.string_ptr == from { + return Some(existing); + } + if (*existing).byte_len as usize == bytes.len() + && std::slice::from_raw_parts(string_data(existing), bytes.len()) == bytes + { + return Some(existing); + } + } + i = (i + 1) & mask; + } + } + + /// Table `atom` under `hash`. The caller has proved no atom exists for + /// its text. Rust-heap only: never allocates on the GC heap. + fn insert(&mut self, hash: u64, atom: *const StringHeader) { + if (self.len + 1) * 4 > self.slots.len() * 3 { + let cap = (self.slots.len() * 2).max(256); + let old = std::mem::replace( + &mut self.slots, + vec![ + InternEntry { + hash: 0, + string_ptr: 0, + }; + cap + ], + ); + for entry in old.into_iter().filter(|e| e.string_ptr != 0) { + self.place(entry); + } + } + self.place(InternEntry { + hash, + string_ptr: atom as usize, + }); + self.len += 1; + } + + fn place(&mut self, entry: InternEntry) { + let mask = self.slots.len() - 1; + let mut i = (entry.hash as usize) & mask; + while self.slots[i].string_ptr != 0 { + i = (i + 1) & mask; + } + self.slots[i] = entry; + } +} + +crate::perry_thread_local! { + /// Per agent, like the cache: an atom is a string in THIS agent's heap. + pub(crate) static ATOMS: std::cell::UnsafeCell = + std::cell::UnsafeCell::new(AtomTable::new()); +} + +/// The atom for `bytes`, without allocating. `None` when no atom exists — or +/// when the agent's table is already torn down. +#[inline] +pub(crate) fn atom_lookup(bytes: &[u8], hash: u64) -> Option<*const StringHeader> { + ATOMS + .try_with(|t| unsafe { (*t.get()).lookup(bytes, hash) }) + .ok() + .flatten() +} + +/// The atom with `key`'s text, if one exists. `hash` is the FNV-1a hash of +/// `key`'s bytes (`key_bytes_hash`). Allocation-free, GC-free. +/// +/// # Safety +/// `key` is a live heap `StringHeader`. +#[inline] +pub(crate) unsafe fn atom_for_key( + key: *const StringHeader, + hash: u64, +) -> Option<*const StringHeader> { + if key.is_null() || (*key).byte_len > INTERN_MAX_BYTE_LEN { + return None; + } + let bytes = std::slice::from_raw_parts(string_data(key), (*key).byte_len as usize); + ATOMS + .try_with(|t| (*t.get()).lookup_from(key as usize, bytes, hash)) + .ok() + .flatten() +} + +/// Mint (or find) the atom for a pooled literal: the one string object this +/// agent uses for that text from now on. Called from `__perry_init_strings_*` +/// in place of `js_string_from_bytes` for pool entries that can be property +/// keys (at most `INTERN_MAX_BYTE_LEN` bytes). `hash` is the pool's +/// precomputed FNV-1a hash of the bytes — the same function as every other +/// key hash here. +/// +/// Adopts the intern cache's string when it already holds this text, so the +/// keys runtime code interned before this module initialised keep matching. +/// Longer literals are not keys worth an atom and take the plain allocation. +#[no_mangle] +pub extern "C" fn js_string_pool_atom( + bytes: *const u8, + len: u32, + hash: u64, + is_wtf8: i32, +) -> *mut StringHeader { + if len == 0 || len > INTERN_MAX_BYTE_LEN || bytes.is_null() { + return if is_wtf8 != 0 { + js_string_from_wtf8_bytes(bytes, len) + } else { + js_string_from_bytes(bytes, len) + }; + } + let input = unsafe { std::slice::from_raw_parts(bytes, len as usize) }; + if let Some(atom) = atom_lookup(input, hash) { + return atom as *mut StringHeader; + } + // Finds the cache's string for this text or allocates one, marking it + // interned and immutable (`refcount = 0`). Nothing is held across the + // allocation: `bytes` is read-only data in the compiled image. + let atom = intern_dispatch_bytes(0, bytes, len as usize, 0, is_wtf8 != 0); + if atom.is_null() { + return js_string_from_bytes(bytes, len); + } + let _ = ATOMS.try_with(|t| unsafe { (*t.get()).insert(hash, atom) }); + atom as *mut StringHeader +} + +/// Test hook: evict `bytes` from the intern CACHE (a collision would), so the +/// next atom mint for that text allocates a new string instead of adopting +/// the cached one. +#[cfg(test)] +pub(crate) fn test_evict_interned(bytes: &[u8]) { + let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); + with_intern_table(|table| unsafe { + (*table)[(hash as usize) & INTERN_TABLE_MASK] = InternEntry { + hash: 0, + string_ptr: 0, + }; + }); +} + +/// Test view: is `p` the atom of its text? +#[cfg(test)] +pub(crate) unsafe fn is_atom_for_test(p: *const StringHeader) -> bool { + let bytes = std::slice::from_raw_parts(string_data(p), (*p).byte_len as usize); + let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); + atom_lookup(bytes, hash) == Some(p) +} + #[inline] pub(crate) fn with_intern_table( f: impl FnOnce(*mut [InternEntry; INTERN_TABLE_SIZE]) -> R, @@ -77,6 +300,20 @@ pub extern "C" fn js_string_intern(key: *const StringHeader, hash: u64) -> *cons return existing; } + // An atom owns this text: the cache slot takes the ATOM, so a computed + // key interns to the same string a read site and a shape key list hold + // (see `AtomTable`), never to a second copy. + let bytes = std::slice::from_raw_parts(string_data(key), byte_len as usize); + if let Some(atom) = atom_lookup(bytes, hash) { + with_intern_table(|table| { + (*table)[slot] = InternEntry { + hash, + string_ptr: atom as usize, + }; + }); + return atom; + } + // Miss or collision — insert (evict on collision) with_intern_table(|table| { (*table)[slot] = InternEntry { @@ -149,6 +386,16 @@ pub(crate) fn intern_dispatch_bytes( if let Some(existing) = hit { return existing; } + // An atom owns this text (see `AtomTable`): hand it out rather than a copy. + if let Some(atom) = atom_lookup(input, hash) { + with_intern_table(|table| unsafe { + (*table)[slot] = InternEntry { + hash, + string_ptr: atom as usize, + }; + }); + return atom; + } let key = if is_wtf8 { js_string_from_wtf8_bytes(bytes, byte_len as u32) @@ -267,6 +514,16 @@ pub fn scan_intern_table_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<' visitor.visit_tagged_usize_slot(&mut entry.string_ptr, crate::value::STRING_TAG); } }); + // The atoms: strong like the cache (every atom is also a registered pool + // handle's value, so this keeps nothing alive that was not), rewritten on + // move. A key's hash is its content's, so a move never rehashes. + let _ = ATOMS.try_with(|t| unsafe { + for entry in (*t.get()).slots.iter_mut() { + if entry.string_ptr != 0 { + visitor.visit_tagged_usize_slot(&mut entry.string_ptr, crate::value::STRING_TAG); + } + } + }); } /// #11507: the table is zero-allocated rather than filled, so a thread's first diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index 40de0c9758..d49320bf79 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -231,6 +231,10 @@ pub use html::{ js_string_fontcolor, js_string_fontsize, js_string_italics, js_string_link, js_string_small, js_string_strike, js_string_sub, js_string_sup, }; +pub(crate) use intern::atom_for_key; +pub use intern::js_string_pool_atom; +#[cfg(test)] +pub(crate) use intern::{atom_lookup, is_atom_for_test, test_evict_interned}; pub use intern::{js_string_intern, scan_intern_table_roots, scan_intern_table_roots_mut}; pub use io::{js_string_error, js_string_print, js_string_warn}; pub(crate) use iter_object::dispatch_string_iterator_method_builtin; From ee4b31a696ee6db42c598caeec2d74db291e8fcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 26 Sep 2026 08:43:31 +0000 Subject: [PATCH 02/16] perf(runtime): confirm a megamorphic site's slot guess against the receiver's key list first A site latched megamorphic sends every read that misses its compact word to js_object_get_field_ic_slow, which answered it from the receiver's shape only after decoding the word, classifying the receiver and scanning the key list. The site may hold one thing: a slot guess (the compact word's high half, the slot the receiver's shape answered last), which the receiver's own shape confirms or refutes. The slow entry now asks that first, and only at a latched site, so a site that can still be primed is primed as before: the receiver's ShapeId names its record; the record's POSITION BOUND says logical key position `guess` is inline slot `guess`; the key at that position must be this key (one pointer compare, S3b atoms); then the receiver's slot is the answer. Anything else continues down the unchanged path. Nothing is emitted at the site, so code size is unchanged. Whether a shape can answer by position is a FACT OF THE RECORD, stored in bit 15 of flags_and_kind (RECORD_POSITIONAL, in the pairwise-disjointness assert): an Ordinary, generation-0, hole-free shape with a keys array and no ACCESSOR key in its attribute summary. It is written by refresh_positional wherever an input can change (construction, with_summary, slab insert, the in-place stable-tombstone update), read with one load on the megamorphic path, and debug builds assert it against its definition on every read. The bound is then min(key count, live inline slots). A test walks every minted record of the agent and fails if the bit and its definition disagree (sabotage: dropping the slab-insert refresh fails it, 8 of 68 records). The in-place updaters only accept a private-epoch record (nonzero generation, never positional), so their refreshes cannot flip the bit today; a second test drives both updaters to zero holes and asserts that premise, so it is where those refreshes start to matter if it ever changes. Logical position i is read past the keys array's front offset (array_elements_ptr), so a shifted keys array is answered correctly. The confirm reads the record through a thread-local mirror of the ordinary page directory (pointer and length, republished whenever the slab's `pages` change, cleared before the slab is dropped): one thread-pointer-relative load and two directory loads, no runtime-state resolution. The step runs in the slow entry's frameless head; the rest of the entry moved out of line. The mirror has a per_thread verdict in thread_exit_address_globals.json. --- .../object/field_get_set/ic_miss/ic_slow.rs | 483 ++++++++++++++++++ crates/perry-runtime/src/object/shapes.rs | 132 ++++- .../src/object/shapes_slot_list.rs | 2 + .../perry-runtime/src/object/shapes_store.rs | 147 +++++- .../src/object/tombstone_tests.rs | 62 +++ scripts/thread_exit_address_globals.json | 8 + test-files/test_gap_megamorphic_slot_guess.ts | 101 ++++ 7 files changed, 915 insertions(+), 20 deletions(-) create mode 100644 test-files/test_gap_megamorphic_slot_guess.ts diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 09f0fc5b8a..f8b7be174e 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -237,6 +237,68 @@ pub extern "C-unwind" fn js_object_get_field_ic_slow( key: *const crate::StringHeader, cache_slot: *mut PicCacheSlot, packed: *const AtomicU64, +) -> f64 { + // --- 0. a MEGAMORPHIC site's slot guess, confirmed by the receiver ------ + // + // A site whose way state is latched negative sees more shapes than any + // per-site entry can name, and every read that misses its compact word + // lands here. The site may still hold one thing: a slot GUESS (the compact + // word's high half — the slot the receiver's shape answered last, step 2b), + // which the RECEIVER'S own shape confirms or refutes + // (`shapes::confirm_slot_guess`: the position bound says key position + // `guess` is inline slot `guess`, and the key there IS this key, one + // pointer compare). Everything the guess cannot answer continues in + // `ic_slow_body` unchanged. Asked first, and only at a latched site, so a + // site that can still be primed is primed exactly as before. `length` is + // excluded as in step 2b: an Array-subclass receiver serves it from its + // elements store. Kept in this frameless entry, with the body out of line, + // so the confirmed read pays no prologue for the arms below. + if let Some(value) = unsafe { megamorphic_slot_guess(obj_handle, key, cache_slot, packed) } { + return value; + } + ic_slow_body(obj_handle, key, cache_slot, packed) +} + +/// Step 0 of [`js_object_get_field_ic_slow`]: the latched site's slot guess. +/// +/// # Safety +/// The entry's contract: a POINTER receiver handle, this site's cache slot +/// and packed word. +#[inline(always)] +unsafe fn megamorphic_slot_guess( + obj_handle: i64, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, +) -> Option { + let obj = obj_handle as usize as *const ObjectHeader; + if packed.is_null() + || key.is_null() + || !crate::value::addr_class::is_above_handle_band(obj as usize) + { + return None; + } + let cache = crate::object::pic_slot_peek(cache_slot); + if cache.is_null() + || (*cache)[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] >= 0 + || key_is_length(key) + { + return None; + } + let guess = ((*packed).load(Ordering::Relaxed) >> 32) as usize; + let value = crate::object::shapes::confirm_slot_guess(obj, key, guess)?; + #[cfg(test)] + crate::object::shapes::SHAPE_ANSWERED_READS.fetch_add(1, Ordering::Relaxed); + Some(value) +} + +/// Everything after step 0 of [`js_object_get_field_ic_slow`]. +#[inline(never)] +fn ic_slow_body( + obj_handle: i64, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, ) -> f64 { let obj = obj_handle as usize as *const ObjectHeader; let addr = obj as usize; @@ -691,6 +753,427 @@ mod tests { assert!(n >= 47, "the shape answers by bytes: {n}"); } + /// Dictionary receivers keep their ShapeId across layout changes, so the + /// shape can never answer for one by position: a latched site reading + /// half-dictionary receivers answers every read correctly, and not one + /// dictionary read is counted as shape-answered. + #[test] + fn a_dictionary_receiver_is_never_answered_by_position() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_dict_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_dict_x"); + let ordinary = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + ordinary >= 47, + "premise: ordinary receivers are shape-answered ({ordinary})" + ); + let mut dict = 0; + for o in objs.iter().step_by(2) { + if o.with_mut_ptr(|p: *mut ObjectHeader| unsafe { + crate::object::dictionary::latch_object_to_dictionary(p) + }) { + dict += 1; + let bound = o.with_const_ptr(|p: *const ObjectHeader| { + crate::object::shapes::test_position_bound_of(p) + }); + assert_eq!(bound, Some(0), "a dictionary shape has no position bound"); + } + } + assert_eq!( + dict, 24, + "premise: every other receiver latched to dictionary" + ); + let n = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + n <= 24, + "INVARIANT: at most the 24 ordinary receivers are shape-answered, got {n}" + ); + } + + /// Tombstones: a receiver whose list carries a HOLE (a tombstone delete) + /// cannot be answered by position — its shape reports no position bound — + /// and every read still returns the receiver's own value; the deleted key + /// reads undefined. + #[test] + fn a_receiver_with_a_tombstoned_key_is_never_answered_by_position() { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + crate::object::delete_rest::test_set_tombstone_deletes(None); + } + } + crate::object::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = Restore; + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_tomb_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_tomb_x"); + let _ = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + let end = scope.root_string_ptr(key_of(b"end")); + let mut tombstoned = 0; + for (i, o) in objs.iter().enumerate().step_by(3) { + // The first delete of a shared list compacts (it takes ownership); + // the second tombstones in place (`tombstone_tests.rs`). + let extra = format!("s3b_tomb_x{i}"); + let extra = scope.root_string_ptr(key_of(extra.as_bytes())); + for k in [&extra, &end] { + o.with_mut_ptr(|p: *mut ObjectHeader| { + k.with_const_ptr(|kp| crate::object::js_object_delete_field(p, kp)) + }); + } + let (holes, bound) = o.with_const_ptr(|p: *const ObjectHeader| unsafe { + ( + crate::object::shapes::object_shape_hole_count(p), + crate::object::shapes::test_position_bound_of(p), + ) + }); + if holes > 0 { + tombstoned += 1; + assert_eq!( + bound, + Some(0), + "INVARIANT: a tombstoned shape has no position bound" + ); + } + } + assert_eq!( + tombstoned, 16, + "premise: every third receiver carries a tombstone" + ); + let _ = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + let _ = latched_pass(&objs, &end, |i| { + if i % 3 == 0 { + crate::value::TAG_UNDEFINED + } else { + 2.0f64.to_bits() + } + }); + } + + /// The slot-guess confirm for shape `shape_id`, as the megamorphic entry + /// asks it (`shapes::slot_guess_confirmed`): `Some(guess)` when the shape + /// says key position `guess` is inline slot `guess` and holds exactly the + /// key `site_key_bits` names, `None` for a decline. + unsafe fn guess_walk(shape_id: u32, guess: u64, site_key_bits: u64) -> Option { + let key = (site_key_bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader; + crate::object::shapes::slot_guess_confirmed(shape_id, key, guess as usize) + .then_some(guess as usize) + } + + fn stamp_of(o: &crate::gc::RuntimeHandle<'_>) -> u32 { + o.with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::shapes::object_shape_stamp(p) + }) + } + + /// For every ordinary receiver the slot-guess confirm reaches the + /// receiver's own record, confirms the site's guess against the + /// receiver's own key, and names the slot holding the receiver's own + /// value. A guess whose position holds a DIFFERENT key, a guess past the + /// shape's bound, an id outside the ShapeId range and an id whose page + /// was never allocated all decline. + #[test] + fn the_slot_guess_is_confirmed_only_against_the_receivers_own_key() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3c_walk_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let site = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3c_walk_x"); + for (i, o) in objs.iter().enumerate() { + let id = stamp_of(o); + let slot = unsafe { guess_walk(id, 2, site) }; + assert_eq!(slot, Some(2), "receiver {i}: the right guess is confirmed"); + let value = o.with_const_ptr(|p: *const ObjectHeader| unsafe { + *((p as *const u8).add(std::mem::size_of::() + 2 * 8) as *const f64) + }); + assert_eq!( + value, + 100.0 + i as f64, + "receiver {i}: the confirmed slot is its own" + ); + for wrong in [0u64, 1, 3] { + assert_eq!( + unsafe { guess_walk(id, wrong, site) }, + None, + "receiver {i}: guess {wrong} holds another key and must decline" + ); + } + assert_eq!(unsafe { guess_walk(id, 4, site) }, None, "past the bound"); + assert_eq!(unsafe { guess_walk(id, u64::from(u32::MAX), site) }, None); + } + assert_eq!( + unsafe { guess_walk(5, 2, site) }, + None, + "a class id is no ShapeId" + ); + assert_eq!( + unsafe { guess_walk(0xBFFF_FFFF, 2, site) }, + None, + "a page never minted" + ); + } + + /// The slot-guess confirm on dictionary and tombstoned receivers: declines, + /// whatever the guess. + #[test] + fn the_slot_guess_never_matches_a_dictionary_or_tombstoned_receiver() { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + crate::object::delete_rest::test_set_tombstone_deletes(None); + } + } + crate::object::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = Restore; + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3c_walk2_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let site = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let objs = megamorphic_receivers_keyed(&scope, 8, text, "s3c_walk2_x"); + let dict = &objs[0]; + assert!( + dict.with_mut_ptr(|p: *mut ObjectHeader| unsafe { + crate::object::dictionary::latch_object_to_dictionary(p) + }), + "premise: the receiver latched to dictionary" + ); + for guess in 0..4 { + assert_eq!( + unsafe { guess_walk(stamp_of(dict), guess, site) }, + None, + "dictionary" + ); + } + let tomb = &objs[1]; + let end = scope.root_string_ptr(key_of(b"end")); + let extra = scope.root_string_ptr(key_of(b"s3c_walk2_x1")); + for k in [&extra, &end] { + tomb.with_mut_ptr(|p: *mut ObjectHeader| { + k.with_const_ptr(|kp| crate::object::js_object_delete_field(p, kp)) + }); + } + let holes = tomb.with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::shapes::object_shape_hole_count(p) + }); + assert!(holes > 0, "premise: the receiver carries a tombstone"); + for guess in 0..4 { + assert_eq!( + unsafe { guess_walk(stamp_of(tomb), guess, site) }, + None, + "tombstoned" + ); + } + } + + fn boxed(p: *mut ObjectHeader) -> f64 { + f64::from_bits(0x7FFD_0000_0000_0000 | (p as u64 & 0x0000_FFFF_FFFF_FFFF)) + } + + /// A [[Prototype]] change is a shape transition (the prototype is part of + /// shape identity) that moves no own key and no own slot: receivers built + /// by `new F()` / `setPrototypeOf` before their fields are assigned — every + /// tsc AST node — get a NEW shape with the SAME own-key layout as a + /// prototype-less twin, and the megamorphic read answers them from that + /// shape, by the key-list scan and by the slot-guess confirm. + #[test] + fn a_prototype_change_is_a_new_shape_with_the_same_layout_and_is_answered() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3_proto_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let site = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let proto = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 2)); + let build = |with_proto: bool, i: usize| { + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + if with_proto { + let (o, p) = ( + obj.with_mut_ptr(|o: *mut ObjectHeader| o), + proto.with_mut_ptr(|p: *mut ObjectHeader| p), + ); + crate::object::object_ops::js_object_set_prototype_of(boxed(o), boxed(p)); + } + let extra = format!("s3_proto_x{i}"); + for (k, v) in [ + (&b"pos"[..], 1.0), + (&b"end"[..], 2.0), + (&text[..], 100.0 + i as f64), + (extra.as_bytes(), 7.0), + ] { + let key = scope.root_string_ptr(key_of(k)); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + obj + }; + let objs: Vec<_> = (0..48).map(|i| build(true, i)).collect(); + let twin = build(false, 0); + let (shape, twin_shape) = (stamp_of(&objs[0]), stamp_of(&twin)); + assert_ne!( + shape, twin_shape, + "premise: the prototype is part of shape identity" + ); + let d = |id| crate::object::shapes::shape_descriptor_by_id(id).expect("live shape"); + let (d, t) = (d(shape), d(twin_shape)); + assert_eq!( + ( + d.keys, + d.logical_key_count, + d.live_inline_slot_count, + d.semantic_generation + ), + ( + t.keys, + t.logical_key_count, + t.live_inline_slot_count, + t.semantic_generation + ), + "the prototype change moved no own key and bumped no generation" + ); + for (i, o) in objs.iter().enumerate() { + assert_eq!( + unsafe { guess_walk(stamp_of(o), 2, site) }, + Some(2), + "INVARIANT: the guess confirm answers prototype-linked receiver {i}" + ); + } + let n = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + n >= 47, + "INVARIANT: prototype-linked receivers are shape-answered: {n}" + ); + } + + /// A shape over a SHIFTED keys array (a front offset): the slot-guess + /// confirm reads LOGICAL key position `i`, past the front offset, so the + /// guess for the key now at logical 0 is confirmed at 0, and the physical + /// position it used to occupy (1) is not. + #[test] + fn a_shape_over_a_shifted_keys_array_is_answered_by_logical_position() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let arr = scope.root_raw_mut_ptr(crate::array::js_array_alloc(4)); + let mut keys = Vec::new(); + for name in [&b"sh_a"[..], b"sh_b", b"sh_c"] { + let k = scope.root_string_ptr(key_of(name)); + let bits = k.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + arr.with_mut_ptr(|a| crate::array::js_array_push(a, crate::JSValue::from_bits(bits))); + keys.push(k); + } + let a = arr.with_mut_ptr(|a: *mut crate::array::ArrayHeader| a); + let bound = + |id: u32| crate::object::shapes::test_position_bound_of_id(id).expect("a live shape"); + let flat = crate::object::shapes::shape_descriptor_ensure(a, 3, 3).expect("mints"); + assert_eq!(bound(flat), 3, "premise: an unshifted keys array"); + crate::array::js_array_shift_f64(a); + assert_ne!( + unsafe { crate::array::array_front_offset(a) }, + 0, + "premise: shift left a front offset" + ); + let shifted = crate::object::shapes::shape_descriptor_ensure(a, 2, 2).expect("mints"); + assert_eq!(bound(shifted), 2, "premise: the shifted shape has two keys"); + let sh_b = keys[1].with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + assert_eq!( + unsafe { guess_walk(shifted, 0, sh_b) }, + Some(0), + "INVARIANT: the confirm reads the logical position" + ); + assert_eq!( + unsafe { guess_walk(shifted, 1, sh_b) }, + None, + "INVARIANT: the physical position is not a key position" + ); + } + + /// The positional bit is a stored FACT of the shape record, read on every + /// latched miss; it must equal its definition for every record the agent + /// has minted. Mints ordinary, dictionary, tombstoned, accessor and + /// class-keyed shapes, then walks the whole slab. + #[test] + fn every_minted_records_positional_bit_matches_its_facts() { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + crate::object::delete_rest::test_set_tombstone_deletes(None); + } + } + crate::object::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = Restore; + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let objs = megamorphic_receivers_keyed(&scope, 24, b"pos_fact_kind", "pos_fact_x"); + // Tombstones: the in-place stable-tombstone update rewrites the hole + // count (two deletes: the first compacts, the second tombstones). + let end = scope.root_string_ptr(key_of(b"end")); + for (i, o) in objs.iter().enumerate().step_by(3) { + let extra = format!("pos_fact_x{i}"); + let extra = scope.root_string_ptr(key_of(extra.as_bytes())); + for k in [&extra, &end] { + o.with_mut_ptr(|p: *mut ObjectHeader| { + k.with_const_ptr(|kp| crate::object::js_object_delete_field(p, kp)) + }); + } + } + // Dictionaries. + for o in objs.iter().skip(1).step_by(3) { + o.with_mut_ptr(|p: *mut ObjectHeader| unsafe { + crate::object::dictionary::latch_object_to_dictionary(p) + }); + } + // Keep the tombstoned receivers growing: re-adds take the cached + // stable-tombstone update. + let again = scope.root_string_ptr(key_of(b"pos_fact_again")); + for o in objs.iter().step_by(3) { + o.with_mut_ptr(|p| { + again.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(p, kp, 3.0)) + }); + } + // Accessor keys: the summary is an input of the bit. + let acc = scope.root_string_ptr(key_of(b"pos_fact_acc")); + for o in objs.iter().skip(2).step_by(3) { + let (ov, kv) = ( + o.with_const_ptr(|p: *const ObjectHeader| { + crate::value::js_nanbox_pointer(p as i64) + }), + acc.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64) + }), + ); + let undef = f64::from_bits(crate::value::TAG_UNDEFINED); + crate::object::js_object_define_accessor(ov, kv, undef, undef); + } + let (n, positional, accessor, bad) = crate::object::shapes::test_positional_census(); + assert!( + positional > 0 && positional < n, + "premise: the slab holds both answerable and unanswerable shapes ({positional} of {n})" + ); + assert!( + accessor > 0, + "premise: an ordinary shape with an accessor key was minted" + ); + assert!( + bad.is_empty(), + "INVARIANT: the stored positional bit equals its definition; {} of {n} records disagree: {bad:x?}", + bad.len() + ); + } + /// A plain own data read that has never primed: the entry must fall all the /// way through to the miss handler, answer the field, and leave the site /// primed exactly as the old `js_object_get_field_ic_miss_packed` edge did. diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 4f87d4d02e..c21a47a509 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -249,13 +249,11 @@ impl ShapeRecordRef { /// shape's own canonical key list — or `None` when the shape cannot answer /// by position alone. /// - /// The position of a key in the keys list is its slot exactly when the - /// shape is `Ordinary`, generation 0 (no descriptor/prototype mutation - /// minted it) and hole-free; a position below `live_inline_slot_count` is - /// an inline slot of every receiver carrying the shape (the invariant the - /// read cache's prime already relies on). The list is bounded by the - /// SHAPE's key count, never the backing's length (#10969: one backing per - /// growth chain). + /// The shape's [`ShapeRecord::position_bound`] says how many leading key + /// positions ARE inline slots of every receiver carrying it (0 for a + /// dictionary, class, descriptor/prototype-generation or tombstoned + /// shape). The list is bounded by that — never by the backing's length + /// (#10969: one backing per growth chain). /// /// Key compares go identity first: canonical lists hold their text's ATOM /// (`string::intern::AtomTable`), which is also what a read site's pooled @@ -270,11 +268,8 @@ impl ShapeRecordRef { hint: usize, ) -> Option { let r = &*self.0.as_ptr(); - if r.object_kind() != ShapeObjectKind::Ordinary - || r.semantic_generation != 0 - || r.hole_count != 0 - || r.keys == 0 - { + let bound = r.position_bound() as usize; + if bound == 0 { return None; } let (slots, len) = @@ -282,11 +277,9 @@ impl ShapeRecordRef { if slots.is_null() { return None; } - let bound = len - .min(r.logical_key_count as usize) - .min(r.live_inline_slot_count as usize); + let bound = bound.min(len); let heap_bits = crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits(); - // The site's slot guess first: the receiver's shape confirms it. + // The site's slot guess, confirmed by the receiver's own key. if hint < bound && (*slots.add(hint)).to_bits() == heap_bits { return Some(hint); } @@ -374,6 +367,113 @@ unsafe fn stored_key_matches(key: *const crate::StringHeader, bits: u64) -> bool } } +/// The position bound of `obj`'s shape (S3c), or `None` when its word names +/// no record in this agent. +#[cfg(test)] +pub(crate) fn test_position_bound_of(obj: *const crate::object::ObjectHeader) -> Option { + let id = unsafe { object_shape_stamp(obj) }; + shape_record_by_id(id).map(|r| unsafe { (*r.0.as_ptr()).position_bound() }) +} + +/// Walk every present record of this agent's slab: `(records, positional, +/// ordinary records with an accessor key, disagreements)`, where a disagreement is a record whose stored positional +/// bit differs from [`ShapeRecord::positional_by_facts`]. +#[cfg(test)] +pub(crate) fn test_positional_census() -> (usize, usize, usize, Vec) { + let table = &crate::state::state().shapes; + let (mut n, mut positional, mut accessor, mut bad) = (0usize, 0usize, 0usize, Vec::new()); + table.slab().for_each(|id, p| { + let r = unsafe { &*p }; + if !r.present() { + return; + } + n += 1; + if r.positional_bit() { + positional += 1; + } + if r.object_kind() == ShapeObjectKind::Ordinary + && r.summary() & crate::object::key_attrs::SUMMARY_ACCESSOR != 0 + { + accessor += 1; + } + if r.positional_bit() != r.positional_by_facts() { + bad.push(id); + } + }); + (n, positional, accessor, bad) +} + +/// `(stored positional bit, its definition)` for shape `id`. +#[cfg(test)] +pub(crate) fn test_positional_of_id(id: u32) -> Option<(bool, bool)> { + shape_record_by_id(id).map(|r| unsafe { + let r = &*r.0.as_ptr(); + (r.positional_bit(), r.positional_by_facts()) + }) +} + +/// The position bound of shape `id` (S3c), or `None` when it names no record. +#[cfg(test)] +pub(crate) fn test_position_bound_of_id(id: u32) -> Option { + shape_record_by_id(id).map(|r| unsafe { (*r.0.as_ptr()).position_bound() }) +} + +/// The megamorphic read's slot-guess confirm: when `obj` carries an ordinary +/// ShapeId of this agent whose record says key position `guess` is inline slot +/// `guess` (`position_bound`), and the key AT that position is `key` itself +/// (one pointer compare: canonical lists hold their text's atom), the value in +/// the receiver's slot `guess`. `None` for anything else — a wrong or stale +/// guess, another text, a dictionary/class/descriptor/tombstoned shape, an id +/// that names no record — and the caller takes its ordinary path. +/// +/// The guess decides nothing: the receiver's own shape confirms it or it is +/// ignored. Allocation-free, no user code. +/// +/// # Safety +/// `obj` is a heap pointer above the handle band (its `+4` word is read); +/// `key` is a heap `StringHeader`. +#[inline] +pub(crate) unsafe fn confirm_slot_guess( + obj: *const crate::object::ObjectHeader, + key: *const crate::StringHeader, + guess: usize, +) -> Option { + if !slot_guess_confirmed((*obj).parent_class_id, key, guess) { + return None; + } + Some( + *((obj as *const u8).add(std::mem::size_of::() + guess * 8) + as *const f64), + ) +} + +/// Does shape `shape_id` of this agent store `key` at inline slot `guess`, +/// by position? See [`confirm_slot_guess`]. +/// +/// # Safety +/// `key` is a heap `StringHeader`. +#[inline] +pub(crate) unsafe fn slot_guess_confirmed( + shape_id: u32, + key: *const crate::StringHeader, + guess: usize, +) -> bool { + let record = ShapeSlab::ordinary_record(shape_id); + if record.is_null() { + return false; + } + let r = &*record; + if guess >= r.position_bound() as usize { + return false; + } + // A bound > 0 means the record names a live keys array (the collector + // marks through and rewrites `keys`) holding at least `bound` logical + // keys; logical element `i` sits past the array's front offset. + let arr = r.keys as usize as *const ArrayHeader; + let slots = crate::array::array_elements_ptr(arr) as *const u64; + *slots.add(guess) == crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits() +} + /// Byte equality without a libc call for the short keys property names are. #[inline] unsafe fn bytes_eq(a: *const u8, b: *const u8, n: usize) -> bool { diff --git a/crates/perry-runtime/src/object/shapes_slot_list.rs b/crates/perry-runtime/src/object/shapes_slot_list.rs index f90ab264d6..37651aa907 100644 --- a/crates/perry-runtime/src/object/shapes_slot_list.rs +++ b/crates/perry-runtime/src/object/shapes_slot_list.rs @@ -578,6 +578,8 @@ pub(crate) unsafe fn try_update_stable_tombstone_shape_cached( record.logical_key_count = logical_key_count; record.live_inline_slot_count = live_inline_slot_count; record.hole_count = hole_count; + // The hole count is an input of the positional bit. + record.refresh_positional(); super::debug_assert_object_shape_parity(obj); Some(id) } diff --git a/crates/perry-runtime/src/object/shapes_store.rs b/crates/perry-runtime/src/object/shapes_store.rs index c0bafd1766..08fbb37016 100644 --- a/crates/perry-runtime/src/object/shapes_store.rs +++ b/crates/perry-runtime/src/object/shapes_store.rs @@ -72,7 +72,9 @@ pub(crate) struct ShapeRecord { pub(super) hole_count: u32, /// Low 8 bits: the `RECORD_FLAG_*` set. Bits 8-10: the `ShapeObjectKind` /// discriminant. Bits 11-14: the births a keyless birth shape served while - /// tracking its width (#10905). Bit 15: reserved. Bits 16-23: the + /// tracking its width (#10905). Bit 15: ANSWERABLE BY POSITION (see + /// [`ShapeRecord::position_bound`]), derived from the record's own facts. + /// Bits 16-23: the /// attribute SUMMARY byte (`key_attrs::SUMMARY_*`), an identity fact. /// Bits 24-31: the inline width a keyless birth shape's descendants grow /// to (#10905). The two #10905 fields are learned facts of the record, @@ -107,6 +109,10 @@ const RECORD_BIRTHS_MASK: u32 = 0xF << RECORD_BIRTHS_SHIFT; /// #10905 (`shapes_birth_width`): the learned descendant width, bits 24-31. const RECORD_WIDTH_SHIFT: u32 = 24; const RECORD_WIDTH_MASK: u32 = 0xFF << RECORD_WIDTH_SHIFT; +/// Bit 15: the shape answers by position ([`ShapeRecord::position_bound`]). +/// A function of the record's facts, rewritten by +/// [`ShapeRecord::refresh_positional`] wherever an input changes. +const RECORD_POSITIONAL: u32 = 1 << 15; // The fields of `flags_and_kind` are pairwise disjoint. const _: () = { let fields = [ @@ -115,6 +121,7 @@ const _: () = { RECORD_BIRTHS_MASK, RECORD_SUMMARY_MASK, RECORD_WIDTH_MASK, + RECORD_POSITIONAL, ]; let mut i = 0; while i < fields.len() { @@ -187,6 +194,8 @@ impl ShapeRecord { pub(super) fn with_summary(mut self, summary: u8) -> ShapeRecord { self.flags_and_kind = (self.flags_and_kind & !RECORD_SUMMARY_MASK) | (u32::from(summary) << RECORD_SUMMARY_SHIFT); + // The summary is an input of the positional bit (an accessor key). + self.refresh_positional(); self } @@ -247,7 +256,7 @@ impl ShapeRecord { // because `facts_match` compares the full enum. let kind_bits = (object_kind.code() as u32) << RECORD_KIND_SHIFT; debug_assert!(kind_bits & !RECORD_KIND_MASK == 0, "kind does not fit"); - ShapeRecord { + let mut record = ShapeRecord { keys, semantic_generation, proto_id: 0, @@ -255,7 +264,74 @@ impl ShapeRecord { live_inline_slot_count, hole_count, flags_and_kind: u32::from(flags) | kind_bits, + }; + record.refresh_positional(); + record + } + + /// How many leading keys of this shape's list sit AT their own inline + /// slot: logical key position `i < bound` IS inline slot `i` of every + /// receiver carrying the shape. 0 when the shape cannot answer by position + /// at all. + /// + /// Whether it can is a FACT OF THE RECORD, stored in bit 15 + /// (`RECORD_POSITIONAL`) and read here with one load: the megamorphic + /// read asks it on every latched miss. [`Self::positional_by_facts`] is + /// its definition; every write of one of its inputs is followed by + /// [`Self::refresh_positional`] (construction, `with_summary`, slab + /// insert, the in-place stable-tombstone update), and debug builds assert + /// the stored bit against the definition on every read. + /// + /// The bound is `min(logical_key_count, live_inline_slot_count)`: a key + /// past the key count is another list's (canonical backings are shared by + /// a growth chain), and one past the live inline count is spilled. + #[inline] + pub(crate) fn position_bound(&self) -> u32 { + debug_assert_eq!( + self.flags_and_kind & RECORD_POSITIONAL != 0, + self.positional_by_facts(), + "the positional bit disagrees with the record's facts: {self:?}" + ); + if self.flags_and_kind & RECORD_POSITIONAL == 0 { + return 0; } + self.logical_key_count.min(self.live_inline_slot_count) + } + + /// The definition of the positional bit. The conjuncts are + /// `js_shape_ordinary_inline_slot_for_key`'s: + /// + /// * `Ordinary` — a class shape's slots are its class layout, and a + /// DICTIONARY shape keeps its id across layout changes, so a dictionary + /// receiver must never be matched by position; + /// * generation 0 — a descriptor/prototype mutation minted this layout; + /// * no tombstones — the answer is only claimed for hole-free lists; + /// * no ACCESSOR key in the attribute summary — an accessor key's slot + /// holds its accessor pair, not a value; + /// * a keys array at all. + #[inline] + pub(super) fn positional_by_facts(&self) -> bool { + self.object_kind() == ShapeObjectKind::Ordinary + && self.semantic_generation == 0 + && self.hole_count == 0 + && self.keys != 0 + && self.summary() & crate::object::key_attrs::SUMMARY_ACCESSOR == 0 + } + + /// Rewrite the positional bit from the record's facts. + #[inline] + pub(super) fn refresh_positional(&mut self) { + if self.positional_by_facts() { + self.flags_and_kind |= RECORD_POSITIONAL; + } else { + self.flags_and_kind &= !RECORD_POSITIONAL; + } + } + + /// The stored positional bit, for the agreement test. + #[cfg(test)] + pub(super) fn positional_bit(&self) -> bool { + self.flags_and_kind & RECORD_POSITIONAL != 0 } /// The same record for a receiver whose [[Prototype]] identity is @@ -465,6 +541,19 @@ fn new_page() -> Page { .unwrap_or_else(|_| unreachable!("page vector has PAGE_LEN slots")) } +/// This thread's ordinary page directory as `(page pointers, page count)`: +/// `ShapeSlab::pages`' element pointer and length, republished after every +/// change to `pages` (`ShapeSlab::publish_dir`) and cleared before the slab +/// is dropped. The megamorphic read's slot-guess confirm +/// ([`ShapeSlab::ordinary_record`]) reads it with one thread-local load +/// instead of resolving the runtime state and walking `record_ptr`. +/// `#[thread_local]` (const, no destructor) rather than `thread_local!`: a +/// late read during thread teardown sees the cleared pair, and the access +/// compiles to one thread-pointer-relative load. +#[thread_local] +static ORDINARY_DIR: std::cell::Cell<(*const Option, usize)> = + std::cell::Cell::new((std::ptr::null(), 0)); + /// The by-id descriptor store. See the module docs. /// Two page directories: ordinary ShapeIds index from `SHAPE_ID_BASE`, and the /// dictionary band (`shapes::DICTIONARY_SHAPE_ID_BASE`) from its own base. One @@ -473,6 +562,14 @@ fn new_page() -> Page { /// that one ~196 KB allocation moved the GC arena's pages relative to the /// page-class table window and cost +2.3% instructions (1.65 M vs 0.20 M /// registered-page misses in `classify_heap_generation`). +impl Drop for ShapeSlab { + fn drop(&mut self) { + if ORDINARY_DIR.get().0 == self.pages.as_ptr() { + ORDINARY_DIR.set((std::ptr::null(), 0)); + } + } +} + pub(crate) struct ShapeSlab { pages: Vec>, dict_pages: Vec>, @@ -587,18 +684,26 @@ impl ShapeSlab { /// Install `record` under `id`, allocating the page and chunk on first /// touch. Returns the record it replaced, if the id was already present. pub(super) fn insert(&mut self, id: u32, mut record: ShapeRecord) -> Option { - let (dict, index) = + let (band, index) = Self::index_of(id).expect("ShapeSlab::insert: id outside the ShapeId range"); record.set(RECORD_FLAG_PRESENT, true); let (page, chunk, slot) = Self::split(index); - let dir = self.dir_mut(dict); + let dir = self.dir_mut(band); if page >= dir.len() { dir.resize_with(page + 1, || None); + // Only the ordinary band is mirrored (`ORDINARY_DIR`). + if band == 0 { + self.publish_dir(); + } } + let dir = self.dir_mut(band); let page = dir[page].get_or_insert_with(new_page); let chunk = page[chunk].get_or_insert_with(new_chunk); let cell = chunk[slot].get_mut(); let previous = cell.present().then_some(*cell); + // A retire-and-reinsert edits facts on a removed copy: the positional + // bit follows them. + record.refresh_positional(); *cell = record; if previous.is_none() { self.len += 1; @@ -686,6 +791,39 @@ impl ShapeSlab { } dir.shrink_to_fit(); } + self.publish_dir(); + } + + /// Publish `pages` for [`Self::ordinary_record`] (see [`ORDINARY_DIR`]). + fn publish_dir(&self) { + ORDINARY_DIR.set((self.pages.as_ptr(), self.pages.len())); + } + + /// The record of ordinary ShapeId `id` in THIS thread's slab, or null — + /// the fast twin of [`Self::record_ptr`] for the megamorphic read: one + /// thread-local load, two dependent directory loads, no `state()`. A + /// dictionary- or exotic-band id indexes past the ordinary directory's + /// length. The + /// record may be absent (`EMPTY`): its position bound is 0. + #[inline(always)] + pub(super) fn ordinary_record(id: u32) -> *const ShapeRecord { + let (pages, len) = ORDINARY_DIR.get(); + let index = id.wrapping_sub(SHAPE_ID_BASE) as usize; + let (page, chunk, slot) = Self::split(index); + if page >= len { + return std::ptr::null(); + } + // SAFETY: `pages` holds `len` entries of this thread's slab, current + // as of the last change to it; nothing here can change it. + unsafe { + let Some(page) = (*pages.add(page)).as_ref() else { + return std::ptr::null(); + }; + match page[chunk].as_ref() { + Some(chunk) => chunk[slot].get(), + None => std::ptr::null(), + } + } } #[cfg(test)] @@ -693,6 +831,7 @@ impl ShapeSlab { self.pages.clear(); self.dict_pages.clear(); self.exotic_pages.clear(); + self.publish_dir(); self.len = 0; } diff --git a/crates/perry-runtime/src/object/tombstone_tests.rs b/crates/perry-runtime/src/object/tombstone_tests.rs index d156c14d22..9e18f364d1 100644 --- a/crates/perry-runtime/src/object/tombstone_tests.rs +++ b/crates/perry-runtime/src/object/tombstone_tests.rs @@ -1118,3 +1118,65 @@ fn stable_tombstone_bound_move_mints_a_new_shape_id() { assert_eq!(after.hole_count, shape.hole_count); } } + +/// The two in-place stable-tombstone updaters rewrite the hole count (and, in +/// the uncached one, the summary) of a live record without reinserting it, +/// and both are inputs of the positional bit (`ShapeRecord::position_bound`). +/// Both refresh the bit, but today no update can FLIP it: the updaters only +/// accept a stable-tombstone receiver's detached record, which carries a +/// private-epoch (nonzero) semantic generation, so it is never positional, +/// hole or not. Driven to zero holes directly, each updater must leave the bit +/// equal to its definition, and the premise that pins the bit false is +/// asserted, so the day an updater admits a generation-0 record this test +/// is where the refresh starts to matter. +#[test] +fn in_place_tombstone_updates_keep_the_positional_bit_equal_to_its_facts() { + super::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = scopeguard_tombstone_flag(); + let _global = crate::gc::global_side_table_test_lock(); + unsafe { + for cached in [true, false] { + let name: &[u8] = if cached { + b"posbit_cached" + } else { + b"posbit_plain" + }; + let (obj, shape) = stable_receiver_one_hole(0, name); + let id = super::shapes::object_shape_stamp(obj); + let keys = shape.keys as usize as *mut crate::ArrayHeader; + let updated = if cached { + super::shapes::try_update_stable_tombstone_shape_cached( + obj, + shape, + shape.logical_key_count, + shape.live_inline_slot_count, + 0, + ) + } else { + super::shapes::try_update_stable_tombstone_shape( + obj, + keys, + shape.logical_key_count, + shape.live_inline_slot_count, + 0, + ) + }; + assert_eq!( + updated, + Some(id), + "premise: the updater (cached={cached}) ran in place" + ); + let after = super::shapes::object_shape_descriptor(obj).unwrap(); + assert_eq!(after.hole_count, 0, "premise: the update wrote zero holes"); + assert_ne!( + after.semantic_generation, 0, + "premise: an in-place-updatable record is a private epoch (nonzero generation)" + ); + let (bit, facts) = super::shapes::test_positional_of_id(id).unwrap(); + assert_eq!( + bit, facts, + "INVARIANT: the in-place update (cached={cached}) left the positional bit stale" + ); + } + } +} diff --git a/scripts/thread_exit_address_globals.json b/scripts/thread_exit_address_globals.json index db97d8f5b0..a9bc9c2681 100644 --- a/scripts/thread_exit_address_globals.json +++ b/scripts/thread_exit_address_globals.json @@ -4086,6 +4086,14 @@ ], "verdict": "no_heap_address", "why": "Holds (Handle, Parked) rows: a Handle is a native-registry id in [1, 0x40000) minted by the process-global shared pool (perry_ffi::shared_handle_id_pool), never an arena address, and Parked is {kind, NativeRegistrationIdentity{domain, serial, numeric_id}, epoch}. Payloads live in the process-global common HANDLES map, not in any thread's arena. A row cannot name a different object after reuse: release() retires only via begin_retirement_of(identity), which rejects any registration whose serial differs, and ids are reissued only after a full trace proves them unreferenced (#11453)." + }, + { + "file": "crates/perry-runtime/src/object/shapes_store.rs", + "names": [ + "ORDINARY_DIR" + ], + "verdict": "per_thread", + "why": "`#[thread_local]` static (not `thread_local!`, so the megamorphic read reaches it with one thread-pointer-relative load): each thread mirrors ITS OWN ShapeSlab page directory (a Rust-heap Vec pointer and length, never an arena address), republished on every change to `pages` and cleared by ShapeSlab::drop before the Vec is freed; const-initialised to (null, 0) with no drop glue." } ] } diff --git a/test-files/test_gap_megamorphic_slot_guess.ts b/test-files/test_gap_megamorphic_slot_guess.ts new file mode 100644 index 0000000000..a8af38d056 --- /dev/null +++ b/test-files/test_gap_megamorphic_slot_guess.ts @@ -0,0 +1,101 @@ +// S3c: the inline shape-confirmed slot guess at a megamorphic read site. +// +// One read site (`readKind`) sees 60+ shapes, so it latches megamorphic and +// its reads take the inline guess: the site's slot guess is confirmed against +// the RECEIVER's own shape key list before any slot is loaded. Every receiver +// family below is built so that a guess which is NOT confirmed by the +// receiver's own shape would return a wrong value: +// * `kind` at slot 2 in most shapes and at other slots in some (the guess +// is right for some receivers, wrong for others); +// * a DIFFERENT key sitting at the guessed slot; +// * dictionary-mode receivers (many keys added one by one); +// * receivers that deleted a key (tombstones / compaction); +// * an accessor installed with defineProperty (descriptor shape); +// * `kind` inherited from a prototype, and `kind` absent; +// * class instances. +// The output is a per-family checksum and must equal node's. + +const EXTRA = []; +for (let i = 0; i < 48; i++) EXTRA.push("x" + i); + +function plain(i: number): any { + const o: any = { pos: i, end: i + 1, kind: i % 7 }; + o[EXTRA[i % 48]] = i; + return o; +} +function shifted(i: number): any { + // `kind` at slot 4: the site's guess (2) names `flags` here. + const o: any = { pos: i, end: i + 1, flags: 1000 + i, parent: null, kind: 50 + (i % 5) }; + o[EXTRA[(i + 7) % 48]] = i; + return o; +} +function dictionary(i: number): any { + const o: any = { pos: i, end: i + 1, kind: 200 + (i % 3) }; + for (let k = 0; k < 200; k++) o["d" + i + "_" + k] = k; + return o; +} +function deleted(i: number): any { + const o: any = { pos: i, end: i + 1, kind: 300 + (i % 4), gone: 1, also: 2 }; + o[EXTRA[(i + 3) % 48]] = i; + delete o.also; + delete o.end; + return o; +} +function accessor(i: number): any { + const o: any = { pos: i, end: i + 1, kind: -1 }; + Object.defineProperty(o, "kind", { get() { return 400 + (i % 6); }, enumerable: true }); + return o; +} +const proto = { kind: 500 }; +function inherited(i: number): any { + const o: any = Object.create(proto); + o.pos = i; + o.end = i + 1; + o.flags = i; + return o; +} +function absent(i: number): any { + const o: any = { pos: i, end: i + 1, flags: 600 + i }; + o[EXTRA[(i + 11) % 48]] = i; + return o; +} +class Node3 { + pos: number; end: number; kind: number; + constructor(i: number) { this.pos = i; this.end = i + 1; this.kind = 700 + (i % 9); } +} + +function readKind(node: any): any { + return node.kind; +} + +const families: [string, (i: number) => any][] = [ + ["plain", plain], ["shifted", shifted], ["dictionary", dictionary], ["deleted", deleted], + ["accessor", accessor], ["inherited", inherited], ["absent", absent], + ["class", (i: number) => new Node3(i)], +]; +const pool: [number, any][] = []; +for (let f = 0; f < families.length; f++) { + for (let i = 0; i < 64; i++) pool.push([f, families[f][1](i)]); +} +// Interleave so the site sees every family while latched. +const order: number[] = []; +for (let i = 0; i < pool.length; i++) order.push((i * 37) % pool.length); + +const sums: number[] = families.map(() => 0); +let undef = 0; +for (let round = 0; round < 50; round++) { + for (const j of order) { + const [f, o] = pool[j]; + const v = readKind(o); + if (v === undefined) undef++; + else sums[f] += v; + } + // Mutate some receivers between rounds: a moved key, a re-added key. + if (round === 20) { + for (const [f, o] of pool) { + if (f === 0 && o.pos % 5 === 0) { delete o.kind; o.kind = 900; } + } + } +} +for (let f = 0; f < families.length; f++) console.log(families[f][0] + " " + sums[f]); +console.log("undefined " + undef); From e8a6c403210c02b3b10f92ac109cd60e607177cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 27 Sep 2026 09:12:50 +0000 Subject: [PATCH 03/16] fix(runtime): an atom is key identity, never interned-key eligibility Minting atoms through the intern cache flagged every pool literal GC_FLAG_INTERNED, which silently admitted literal keys to the interned-only own-property lanes (read lane, set fast paths, chain store, proxy put). On Zod the widened read lane misses for inherited keys: keys_find_slot_by_key_ptr 5014 -> 8022 calls, +0.3%. Atoms are now plain allocations, and the intern cache neither adopts nor hands them out. --- .../src/gc/tests/canonical_keys_holders.rs | 15 +++-- .../object/field_get_set/ic_miss/ic_slow.rs | 20 +++++- crates/perry-runtime/src/string/intern.rs | 66 +++++++------------ 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs b/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs index c70c672600..0656c379f6 100644 --- a/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs +++ b/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs @@ -671,9 +671,9 @@ fn atom_bits(atom: usize) -> u64 { /// S3b: a key text has ONE string object in an agent — its atom — and every /// canonical list written after the atom exists holds it. The atom table holds /// its strings strongly and REWRITES them on a move (the intern-table root -/// scanner), so after a moving minor the table, the list, a fresh intern and a -/// fresh pool mint all name the atom at its NEW address, and none names the -/// address it moved away from. +/// scanner), so after a moving minor the table, the list and a fresh pool mint +/// all name the atom at its NEW address, and none names the address it moved +/// away from. Interning is separate: the intern cache never hands out an atom. #[test] fn an_atom_and_the_lists_holding_it_follow_a_moving_minor() { let _guard = CopyingNurseryTestGuard::new(0); @@ -723,12 +723,13 @@ fn an_atom_and_the_lists_holding_it_follow_a_moving_minor() { atom_bits(moved), "INVARIANT: the list follows its atom through the move" ); - // After the move: interning another copy, and minting again, both - // return the moved atom — never a third string. - assert_eq!( + // After the move: minting again returns the moved atom — never a third + // string. Interning a copy does NOT: an atom is identity, not + // eligibility (`string::intern::AtomTable`). + assert_ne!( crate::string::js_string_intern(nursery_key("atom_mv_kind"), hash) as usize, moved, - "a copy interns to the moved atom" + "INVARIANT: the intern cache never hands out an atom" ); assert_eq!( crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize, diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index f8b7be174e..78c7b67b99 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -704,14 +704,30 @@ mod tests { "INVARIANT: receiver {i}'s shape holds the atom, not the copy it grew with" ); } - // A computed key with the text interns TO the atom. + // Identity is not eligibility: the atom is not an interned string, and + // a computed key with the text interns to its own string, never to the + // atom (`GC_FLAG_INTERNED` admits keys to the own-property lanes; an + // atom must not widen them). + let atom_flags = atom.with_const_ptr(|p: *const crate::StringHeader| unsafe { + (*((p as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader)) + .gc_flags + }); + assert_eq!( + atom_flags & crate::gc::GC_FLAG_INTERNED, + 0, + "INVARIANT: an atom is never flagged interned" + ); let interned = copy.with_const_ptr(|p: *const crate::StringHeader| { crate::string::js_string_intern( p, crate::object::key_bytes_hash(text.as_ptr(), text.len()), ) as usize }); - assert_eq!(interned, addr(&atom), "a copy interns to the atom"); + assert_ne!( + interned, + addr(&atom), + "INVARIANT: interning never returns the atom" + ); // The site holds the atom: every latched read is the receiver's own // value, answered by its shape. let by_atom = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); diff --git a/crates/perry-runtime/src/string/intern.rs b/crates/perry-runtime/src/string/intern.rs index 5083c2e22c..792ab0e34c 100644 --- a/crates/perry-runtime/src/string/intern.rs +++ b/crates/perry-runtime/src/string/intern.rs @@ -57,10 +57,19 @@ crate::perry_thread_local! { /// collector rewrites the entries on move through the intern-table root /// scanner (`scan_intern_table_roots_mut`), exactly like the cache's. /// -/// Two funnels consult it: the intern cache's miss paths (so a computed key -/// with an atom's text interns TO the atom), and canonical key lists when they -/// write a key (`canonical_keys::Appended::atomized`), so a read site's pooled -/// key and the receiver's shape key are one pointer. +/// One funnel consults it: canonical key lists when they write a key +/// (`canonical_keys::Appended::atomized`), so a read site's pooled key and the +/// receiver's shape key are one pointer. +/// +/// An atom is NOT an interned string. It is minted by a plain allocation and +/// never carries `GC_FLAG_INTERNED`, and the intern cache neither adopts nor +/// hands out atoms. `GC_FLAG_INTERNED` is an ELIGIBILITY bit: the own-property +/// read lane, the set fast paths, the chain store and the proxy put paths +/// admit only interned keys. Minting atoms as interned strings silently widened +/// every one of those lanes to every pool-literal key, and on Zod the widened +/// read lane MISSES (the key is inherited, not own) at ~330 instructions each: +/// +0.3% instructions, measured, with the atom table itself inert. Identity is +/// the atom's job; eligibility stays exactly what it was. /// /// The table never decides an answer. A pointer match proves equal text; a /// pointer MISmatch proves nothing (a list written before its atom existed @@ -205,9 +214,10 @@ pub(crate) unsafe fn atom_for_key( /// precomputed FNV-1a hash of the bytes — the same function as every other /// key hash here. /// -/// Adopts the intern cache's string when it already holds this text, so the -/// keys runtime code interned before this module initialised keep matching. -/// Longer literals are not keys worth an atom and take the plain allocation. +/// A plain allocation, exactly what the pool minted before atoms existed: the +/// atom is not interned and does not adopt the intern cache's string (see +/// `AtomTable`: identity, never eligibility). Longer literals are not keys +/// worth an atom and take the plain allocation without a table entry. #[no_mangle] pub extern "C" fn js_string_pool_atom( bytes: *const u8, @@ -226,20 +236,18 @@ pub extern "C" fn js_string_pool_atom( if let Some(atom) = atom_lookup(input, hash) { return atom as *mut StringHeader; } - // Finds the cache's string for this text or allocates one, marking it - // interned and immutable (`refcount = 0`). Nothing is held across the - // allocation: `bytes` is read-only data in the compiled image. - let atom = intern_dispatch_bytes(0, bytes, len as usize, 0, is_wtf8 != 0); - if atom.is_null() { - return js_string_from_bytes(bytes, len); - } + // Nothing is held across the allocation: `bytes` is read-only data in the + // compiled image. Shared (`refcount = 0`) like every pool literal. + let atom: *const StringHeader = if is_wtf8 != 0 { + js_string_from_wtf8_bytes(bytes, len) + } else { + js_string_from_bytes(bytes, len) + }; let _ = ATOMS.try_with(|t| unsafe { (*t.get()).insert(hash, atom) }); atom as *mut StringHeader } -/// Test hook: evict `bytes` from the intern CACHE (a collision would), so the -/// next atom mint for that text allocates a new string instead of adopting -/// the cached one. +/// Test hook: evict `bytes` from the intern CACHE (a collision would). #[cfg(test)] pub(crate) fn test_evict_interned(bytes: &[u8]) { let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); @@ -300,20 +308,6 @@ pub extern "C" fn js_string_intern(key: *const StringHeader, hash: u64) -> *cons return existing; } - // An atom owns this text: the cache slot takes the ATOM, so a computed - // key interns to the same string a read site and a shape key list hold - // (see `AtomTable`), never to a second copy. - let bytes = std::slice::from_raw_parts(string_data(key), byte_len as usize); - if let Some(atom) = atom_lookup(bytes, hash) { - with_intern_table(|table| { - (*table)[slot] = InternEntry { - hash, - string_ptr: atom as usize, - }; - }); - return atom; - } - // Miss or collision — insert (evict on collision) with_intern_table(|table| { (*table)[slot] = InternEntry { @@ -386,16 +380,6 @@ pub(crate) fn intern_dispatch_bytes( if let Some(existing) = hit { return existing; } - // An atom owns this text (see `AtomTable`): hand it out rather than a copy. - if let Some(atom) = atom_lookup(input, hash) { - with_intern_table(|table| unsafe { - (*table)[slot] = InternEntry { - hash, - string_ptr: atom as usize, - }; - }); - return atom; - } let key = if is_wtf8 { js_string_from_wtf8_bytes(bytes, byte_len as u32) From e8e53077919317c6aecabc20d2cf01b29de242b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 01:43:57 +0000 Subject: [PATCH 04/16] docs(changelog): megamorphic reads confirm the slot guess by key atom --- changelog.d/megamorphic-read-key-atoms.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog.d/megamorphic-read-key-atoms.md diff --git a/changelog.d/megamorphic-read-key-atoms.md b/changelog.d/megamorphic-read-key-atoms.md new file mode 100644 index 0000000000..26ae826a5c --- /dev/null +++ b/changelog.d/megamorphic-read-key-atoms.md @@ -0,0 +1,5 @@ +Megamorphic property reads confirm the site's last slot against the receiver's +shape with one pointer compare: property-key literals are now one string per +key text (key atoms), and shape key lists hold that string. A 40-shape +`o.kind` read drops from ~300 to ~140 instructions. Atoms do not change which +keys count as interned. From d98497fd1c1b45b8b3c26b20b0d7a925132cc1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 15:36:36 +0200 Subject: [PATCH 05/16] changelog: name the fragment after PR #11633 --- ...phic-read-key-atoms.md => 11633-megamorphic-read-key-atoms.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{megamorphic-read-key-atoms.md => 11633-megamorphic-read-key-atoms.md} (100%) diff --git a/changelog.d/megamorphic-read-key-atoms.md b/changelog.d/11633-megamorphic-read-key-atoms.md similarity index 100% rename from changelog.d/megamorphic-read-key-atoms.md rename to changelog.d/11633-megamorphic-read-key-atoms.md From 71406b4953362ac546db7793f6aa211aefb0a356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 16:24:11 +0200 Subject: [PATCH 06/16] fix(runtime): an SSO key slot is its own atom; say so in code for the SSO unbox inventory atomized() replaced heap-string key slots with their atom and left every other slot alone. That was correct for short (SSO) strings, whose bits are their identity, but only implicitly, so the SSO unbox inventory (#11627) counted it as a new heap-only string reader. The SSO arm is now explicit. --- crates/perry-runtime/src/object/canonical_keys.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/perry-runtime/src/object/canonical_keys.rs b/crates/perry-runtime/src/object/canonical_keys.rs index 00704f8880..1a826ae6ec 100644 --- a/crates/perry-runtime/src/object/canonical_keys.rs +++ b/crates/perry-runtime/src/object/canonical_keys.rs @@ -697,6 +697,10 @@ impl Appended { Some(atom) => Appended::Key(atom), None => self, }, + // An SSO short string carries its bytes in the value itself, so + // its bits ARE its identity: equal texts are already equal words + // and there is no heap string to replace with an atom. + Appended::Slot(v) if v.is_short_string() => self, Appended::Slot(v) if v.is_string() => { match crate::string::atom_for_key(v.as_string_ptr(), h) { Some(atom) => Appended::Slot(JSValue::string_ptr(atom as *mut StringHeader)), From 805b222678243498e3e72905c50a31f6f8e7b6fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 21:07:35 +0000 Subject: [PATCH 07/16] regen: js_string_pool_atom in the wasm ABI table and the linux gc-call-effects table --- crates/perry-codegen/src/gc_effects/linux-x86_64.tsv | 1 + crates/perry-codegen/src/wasm32/runtime_abi.tsv | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv index b71c7ad644..6625c52a2a 100644 --- a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv @@ -3128,6 +3128,7 @@ js_string_normalize Reenters js_string_pad_end Reenters js_string_pad_fill Reenters js_string_pad_start Reenters +js_string_pool_atom Reenters js_string_position_to_index Leaf js_string_print Leaf js_string_raw Reenters diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index 059313f6b0..c3e2693778 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -3616,6 +3616,7 @@ js_string_normalize ptr ptr,f64 js_string_pad_end ptr ptr,f64,ptr js_string_pad_fill ptr f64 js_string_pad_start ptr ptr,f64,ptr +js_string_pool_atom ptr ptr,i32u,i64,i32s js_string_position_to_index i32s f64 js_string_print void ptr js_string_raw ptr f64,f64 From 5bfb01c7839842241f1412250c01a0f44385f0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 21:14:37 +0000 Subject: [PATCH 08/16] test(runtime): atoms survive a moving minor via the atom young log --- .../src/gc/tests/minor_fixed_cost.rs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs b/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs index 008f666f7b..b62f32743e 100644 --- a/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs +++ b/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs @@ -178,3 +178,44 @@ fn small_int_cache_writer_publishing_a_young_string_is_caught() { crate::string::test_write_small_int_cache_slot(255, young); crate::string::debug_assert_small_string_caches_not_minor_relevant(); } + +/// An atom minted from a young string and reachable ONLY through the atom +/// table survives a moving minor: the table names the forwarded, live copy +/// (found again by text and by `atom_for_key`), not from-space. The atom +/// young log is what makes the minor visit that slot. +#[test] +fn young_atom_is_rewritten_through_the_atom_young_log() { + let _guard = CopyingNurseryTestGuard::new(0); + let _clear = ClearTablesOnDrop; + gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut); + crate::string::test_clear_intern_table(); + + let bytes = b"minor-fixed-cost-young-atom"; + let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); + let young = crate::string::js_string_pool_atom(bytes.as_ptr(), bytes.len() as u32, hash, 0); + assert!(crate::arena::pointer_in_nursery(young as usize)); + assert_eq!( + crate::string::atom_lookup(bytes, hash), + Some(young as *const _) + ); + + let _ = gc_collect_minor(); + + let tabled = crate::string::atom_lookup(bytes, hash).expect("the atom must stay tabled"); + assert_ne!( + tabled as usize, young as usize, + "the table must name the evacuated copy, not from-space" + ); + unsafe { + assert_string_bytes(tabled, bytes); + let fresh = crate::string::js_string_pool_atom(bytes.as_ptr(), bytes.len() as u32, hash, 0); + assert_eq!( + fresh as usize, tabled as usize, + "the pool must reuse the live atom" + ); + assert!(crate::string::is_atom_for_test(tabled)); + // A second string with the same text resolves to the same atom. + let other = crate::string::js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32); + assert_eq!(crate::string::atom_for_key(other, hash), Some(tabled)); + } +} From 93aee48d9a10526aa0da287a3dba6982c1f19ceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 23:00:47 +0000 Subject: [PATCH 09/16] perf(runtime): POSBOUND, the shape record's position bound as one field The megamorphic read asks a receiver's shape record whether key position `guess` is inline slot `guess`. #11633 answered with bit 15 of flags_and_kind plus `min(logical_key_count, live_inline_slot_count)` on every ask. POSBOUND stores the answer: `position_bound: u32` at offset 40, 0 when the shape cannot answer by position, else the min. It replaces bit 15 (reserved again), is rewritten by `refresh_positional` wherever an input changes, and debug builds assert it against its definition on every read. The census test now compares the stored bound with the definition. The record grows 40 -> 48 bytes (4 bytes of tail padding). The slab's fast lookup takes the ordinary directory mirror's address (`ordinary_record_in`), so a caller that already holds it reads no thread-local. --- crates/perry-runtime/src/object/shapes.rs | 20 ++- .../perry-runtime/src/object/shapes_store.rs | 160 +++++++++++------- 2 files changed, 112 insertions(+), 68 deletions(-) diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index c21a47a509..02edd45432 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -376,8 +376,9 @@ pub(crate) fn test_position_bound_of(obj: *const crate::object::ObjectHeader) -> } /// Walk every present record of this agent's slab: `(records, positional, -/// ordinary records with an accessor key, disagreements)`, where a disagreement is a record whose stored positional -/// bit differs from [`ShapeRecord::positional_by_facts`]. +/// ordinary records with an accessor key, disagreements)`, where a +/// disagreement is a record whose stored POSBOUND differs from +/// [`ShapeRecord::position_bound_by_facts`]. #[cfg(test)] pub(crate) fn test_positional_census() -> (usize, usize, usize, Vec) { let table = &crate::state::state().shapes; @@ -388,7 +389,7 @@ pub(crate) fn test_positional_census() -> (usize, usize, usize, Vec) { return; } n += 1; - if r.positional_bit() { + if r.stored_position_bound() > 0 { positional += 1; } if r.object_kind() == ShapeObjectKind::Ordinary @@ -396,19 +397,19 @@ pub(crate) fn test_positional_census() -> (usize, usize, usize, Vec) { { accessor += 1; } - if r.positional_bit() != r.positional_by_facts() { + if r.stored_position_bound() != r.position_bound_by_facts() { bad.push(id); } }); (n, positional, accessor, bad) } -/// `(stored positional bit, its definition)` for shape `id`. +/// `(stored POSBOUND, its definition)` for shape `id`. #[cfg(test)] -pub(crate) fn test_positional_of_id(id: u32) -> Option<(bool, bool)> { +pub(crate) fn test_positional_of_id(id: u32) -> Option<(u32, u32)> { shape_record_by_id(id).map(|r| unsafe { let r = &*r.0.as_ptr(); - (r.positional_bit(), r.positional_by_facts()) + (r.stored_position_bound(), r.position_bound_by_facts()) }) } @@ -458,12 +459,13 @@ pub(crate) unsafe fn slot_guess_confirmed( key: *const crate::StringHeader, guess: usize, ) -> bool { - let record = ShapeSlab::ordinary_record(shape_id); + // SAFETY: this thread's own mirror. + let record = unsafe { ShapeSlab::ordinary_record_in(ShapeSlab::ordinary_dir_addr(), shape_id) }; if record.is_null() { return false; } let r = &*record; - if guess >= r.position_bound() as usize { + if guess >= r.position_bound_raw() as usize { return false; } // A bound > 0 means the record names a live keys array (the collector diff --git a/crates/perry-runtime/src/object/shapes_store.rs b/crates/perry-runtime/src/object/shapes_store.rs index 08fbb37016..0997be562f 100644 --- a/crates/perry-runtime/src/object/shapes_store.rs +++ b/crates/perry-runtime/src/object/shapes_store.rs @@ -72,10 +72,9 @@ pub(crate) struct ShapeRecord { pub(super) hole_count: u32, /// Low 8 bits: the `RECORD_FLAG_*` set. Bits 8-10: the `ShapeObjectKind` /// discriminant. Bits 11-14: the births a keyless birth shape served while - /// tracking its width (#10905). Bit 15: ANSWERABLE BY POSITION (see - /// [`ShapeRecord::position_bound`]), derived from the record's own facts. - /// Bits 16-23: the - /// attribute SUMMARY byte (`key_attrs::SUMMARY_*`), an identity fact. + /// tracking its width (#10905). Bit 15: reserved (it held the + /// answerable-by-position bit, which is now [`Self::position_bound`]'s + /// zero). Bits 16-23: the attribute SUMMARY byte (`key_attrs::SUMMARY_*`), an identity fact. /// Bits 24-31: the inline width a keyless birth shape's descendants grow /// to (#10905). The two #10905 fields are learned facts of the record, /// never identity. @@ -85,6 +84,18 @@ pub(crate) struct ShapeRecord { /// 8-aligned (asserted below) and the slab geometry is unchanged — the /// kind field is free, it lives in padding that was already paid for. flags_and_kind: u32, + /// POSBOUND: how many leading key positions ARE inline slots of every + /// receiver carrying this shape — `min(logical_key_count, + /// live_inline_slot_count)` when the shape answers by position + /// ([`Self::positional_by_facts`]), 0 otherwise. A function of the + /// record's facts, rewritten by [`Self::refresh_positional`] wherever an + /// input changes, read by [`Self::position_bound`]. + /// + /// ONE field so the megamorphic read confirm (`js_object_read_confirm`) + /// answers "is the guess a position of this shape" with one compare — + /// `guess < position_bound` — instead of a flag test and a `min`. + /// Offset 40; the record is 48 bytes with 4 bytes of tail padding. + position_bound: u32, } const RECORD_KIND_SHIFT: u32 = 8; @@ -109,10 +120,6 @@ const RECORD_BIRTHS_MASK: u32 = 0xF << RECORD_BIRTHS_SHIFT; /// #10905 (`shapes_birth_width`): the learned descendant width, bits 24-31. const RECORD_WIDTH_SHIFT: u32 = 24; const RECORD_WIDTH_MASK: u32 = 0xFF << RECORD_WIDTH_SHIFT; -/// Bit 15: the shape answers by position ([`ShapeRecord::position_bound`]). -/// A function of the record's facts, rewritten by -/// [`ShapeRecord::refresh_positional`] wherever an input changes. -const RECORD_POSITIONAL: u32 = 1 << 15; // The fields of `flags_and_kind` are pairwise disjoint. const _: () = { let fields = [ @@ -121,7 +128,6 @@ const _: () = { RECORD_BIRTHS_MASK, RECORD_SUMMARY_MASK, RECORD_WIDTH_MASK, - RECORD_POSITIONAL, ]; let mut i = 0; while i < fields.len() { @@ -137,8 +143,9 @@ const _: () = assert!( super::shapes_birth_width::TRACKING_BIRTHS <= RECORD_BIRTHS_MASK >> RECORD_BIRTHS_SHIFT ); -const _: () = assert!(std::mem::size_of::() == 40); +const _: () = assert!(std::mem::size_of::() == 48); const _: () = assert!(std::mem::align_of::() == 8); +const _: () = assert!(std::mem::offset_of!(ShapeRecord, position_bound) == 40); impl ShapeRecord { const EMPTY: ShapeRecord = ShapeRecord { @@ -149,6 +156,7 @@ impl ShapeRecord { live_inline_slot_count: 0, hole_count: 0, flags_and_kind: 0, + position_bound: 0, }; #[inline] @@ -264,6 +272,7 @@ impl ShapeRecord { live_inline_slot_count, hole_count, flags_and_kind: u32::from(flags) | kind_bits, + position_bound: 0, }; record.refresh_positional(); record @@ -274,25 +283,32 @@ impl ShapeRecord { /// receiver carrying the shape. 0 when the shape cannot answer by position /// at all. /// - /// Whether it can is a FACT OF THE RECORD, stored in bit 15 - /// (`RECORD_POSITIONAL`) and read here with one load: the megamorphic - /// read asks it on every latched miss. [`Self::positional_by_facts`] is - /// its definition; every write of one of its inputs is followed by - /// [`Self::refresh_positional`] (construction, `with_summary`, slab - /// insert, the in-place stable-tombstone update), and debug builds assert - /// the stored bit against the definition on every read. - /// - /// The bound is `min(logical_key_count, live_inline_slot_count)`: a key - /// past the key count is another list's (canonical backings are shared by - /// a growth chain), and one past the live inline count is spilled. + /// It is a FACT OF THE RECORD, stored in the `position_bound` field + /// (POSBOUND) and read here with one load: the megamorphic read confirm + /// compares a site's slot guess against it on every latched read. + /// [`Self::position_bound_by_facts`] is its definition; every write of one + /// of its inputs is followed by [`Self::refresh_positional`] + /// (construction, `with_summary`, slab insert, the in-place + /// stable-tombstone update), and debug builds assert the stored bound + /// against the definition on every read. #[inline] pub(crate) fn position_bound(&self) -> u32 { debug_assert_eq!( - self.flags_and_kind & RECORD_POSITIONAL != 0, - self.positional_by_facts(), - "the positional bit disagrees with the record's facts: {self:?}" + self.position_bound, + self.position_bound_by_facts(), + "the position bound disagrees with the record's facts: {self:?}" ); - if self.flags_and_kind & RECORD_POSITIONAL == 0 { + self.position_bound + } + + /// The definition of POSBOUND: `min(logical_key_count, + /// live_inline_slot_count)` for a shape that answers by position, else 0. + /// A key past the key count is another list's (canonical backings are + /// shared by a growth chain), and one past the live inline count is + /// spilled. + #[inline] + pub(super) fn position_bound_by_facts(&self) -> u32 { + if !self.positional_by_facts() { return 0; } self.logical_key_count.min(self.live_inline_slot_count) @@ -318,20 +334,25 @@ impl ShapeRecord { && self.summary() & crate::object::key_attrs::SUMMARY_ACCESSOR == 0 } - /// Rewrite the positional bit from the record's facts. + /// Rewrite POSBOUND from the record's facts. #[inline] pub(super) fn refresh_positional(&mut self) { - if self.positional_by_facts() { - self.flags_and_kind |= RECORD_POSITIONAL; - } else { - self.flags_and_kind &= !RECORD_POSITIONAL; - } + self.position_bound = self.position_bound_by_facts(); } - /// The stored positional bit, for the agreement test. + /// The stored POSBOUND without the debug agreement assert, for the + /// agreement test. #[cfg(test)] - pub(super) fn positional_bit(&self) -> bool { - self.flags_and_kind & RECORD_POSITIONAL != 0 + pub(super) fn stored_position_bound(&self) -> u32 { + self.position_bound + } + + /// The stored POSBOUND for the megamorphic read confirm, which must stay + /// a GC leaf with no formatting path: the agreement is asserted by + /// [`Self::position_bound`] everywhere else and by the census test. + #[inline(always)] + pub(crate) fn position_bound_raw(&self) -> u32 { + self.position_bound } /// The same record for a receiver whose [[Prototype]] identity is @@ -541,18 +562,23 @@ fn new_page() -> Page { .unwrap_or_else(|_| unreachable!("page vector has PAGE_LEN slots")) } +/// The ordinary directory mirror's type: `(page pointers, page count)`. +type OrdinaryDir = std::cell::Cell<(*const Option, usize)>; + /// This thread's ordinary page directory as `(page pointers, page count)`: /// `ShapeSlab::pages`' element pointer and length, republished after every /// change to `pages` (`ShapeSlab::publish_dir`) and cleared before the slab /// is dropped. The megamorphic read's slot-guess confirm -/// ([`ShapeSlab::ordinary_record`]) reads it with one thread-local load -/// instead of resolving the runtime state and walking `record_ptr`. -/// `#[thread_local]` (const, no destructor) rather than `thread_local!`: a -/// late read during thread teardown sees the cleared pair, and the access -/// compiles to one thread-pointer-relative load. +/// ([`ShapeSlab::ordinary_record_in`]) reads it through its ADDRESS, which +/// emitted code passes from the agent's pointer block, instead of resolving +/// the runtime state and walking `record_ptr`: the confirm itself then +/// touches no thread-local (a runtime thread-local access is a +/// `__tls_get_addr` call on ELF and a TLV thunk call on Darwin, which would +/// give the stub a frame). `#[thread_local]` (const, no destructor) rather +/// than `thread_local!`: the address is stable for the thread's life, and a +/// late read during thread teardown sees the cleared pair. #[thread_local] -static ORDINARY_DIR: std::cell::Cell<(*const Option, usize)> = - std::cell::Cell::new((std::ptr::null(), 0)); +static ORDINARY_DIR: OrdinaryDir = std::cell::Cell::new((std::ptr::null(), 0)); /// The by-id descriptor store. See the module docs. /// Two page directories: ordinary ShapeIds index from `SHAPE_ID_BASE`, and the @@ -794,20 +820,38 @@ impl ShapeSlab { self.publish_dir(); } - /// Publish `pages` for [`Self::ordinary_record`] (see [`ORDINARY_DIR`]). + /// Publish `pages` for [`Self::ordinary_record_in`] (see [`ORDINARY_DIR`]). fn publish_dir(&self) { ORDINARY_DIR.set((self.pages.as_ptr(), self.pages.len())); } - /// The record of ordinary ShapeId `id` in THIS thread's slab, or null — - /// the fast twin of [`Self::record_ptr`] for the megamorphic read: one - /// thread-local load, two dependent directory loads, no `state()`. A - /// dictionary- or exotic-band id indexes past the ordinary directory's - /// length. The - /// record may be absent (`EMPTY`): its position bound is 0. + /// The address of THIS thread's [`ORDINARY_DIR`] mirror, as an opaque + /// pointer for [`Self::ordinary_record_in`]. Stable for the thread's + /// life (a const-initialised `#[thread_local]` with no destructor), so an + /// agent publishes it once into its `PERRY_AGENT_PTRS` slot + /// (`agent_ptrs::perry_shape_dir_cell`) and emitted code hands it to the + /// megamorphic read confirm, which then reads no thread-local at all. + #[inline] + pub(crate) fn ordinary_dir_addr() -> *const u8 { + &ORDINARY_DIR as *const OrdinaryDir as *const u8 + } + + /// The record of ordinary ShapeId `id` in the slab whose [`ORDINARY_DIR`] + /// mirror is at `dir` (an [`Self::ordinary_dir_addr`] of this thread, or + /// null), or null — the fast twin of [`Self::record_ptr`] for the + /// megamorphic read: two dependent directory loads, no `state()`, no + /// thread-local access. A dictionary- or exotic-band id indexes past the + /// ordinary directory's length. The record may be absent (`EMPTY`): its + /// position bound is 0. + /// + /// # Safety + /// `dir` is null or this thread's [`Self::ordinary_dir_addr`]. #[inline(always)] - pub(super) fn ordinary_record(id: u32) -> *const ShapeRecord { - let (pages, len) = ORDINARY_DIR.get(); + pub(super) unsafe fn ordinary_record_in(dir: *const u8, id: u32) -> *const ShapeRecord { + if dir.is_null() { + return std::ptr::null(); + } + let (pages, len) = (*(dir as *const OrdinaryDir)).get(); let index = id.wrapping_sub(SHAPE_ID_BASE) as usize; let (page, chunk, slot) = Self::split(index); if page >= len { @@ -815,14 +859,12 @@ impl ShapeSlab { } // SAFETY: `pages` holds `len` entries of this thread's slab, current // as of the last change to it; nothing here can change it. - unsafe { - let Some(page) = (*pages.add(page)).as_ref() else { - return std::ptr::null(); - }; - match page[chunk].as_ref() { - Some(chunk) => chunk[slot].get(), - None => std::ptr::null(), - } + let Some(page) = (*pages.add(page)).as_ref() else { + return std::ptr::null(); + }; + match page[chunk].as_ref() { + Some(chunk) => chunk[slot].get(), + None => std::ptr::null(), } } @@ -1364,7 +1406,7 @@ mod tests { /// (`proto_id`), and a 64-bit prototype identity does not fit the padding. #[test] fn the_record_geometry_is_free_and_facts_key_is_o1() { - assert_eq!(std::mem::size_of::(), 40, "record grew"); + assert_eq!(std::mem::size_of::(), 48, "record grew"); assert_eq!(std::mem::align_of::(), 8, "record realigned"); // `facts_key` folds the keys ADDRESS; it must never dereference it. From c519ea9969a948481c5815c1afafa61c853609db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 01:03:33 +0000 Subject: [PATCH 10/16] perf: one GC-leaf miss front per generic read site (D3, D3b) A generic property read keeps only the ShapeId compare and the slot load inline. The compare's false edge makes one plain call to the GC-leaf js_object_get_field_ic_front(dir, handle, key_bits, cache_slot, packed), tests its answer against TAG_HOLE and, only on a decline, branches to the unchanged collecting js_object_get_field_ic_slow. Receiver-validation failures skip the front. --typed-feedback builds keep the old edge. The front (read_confirm.rs) answers from shape facts only, in order: - a polymorphic way (PIC_ID_TOKEN_BIT | ShapeId, slot); - a spill entry: the compact word holds the ShapeId flipped by PACKED_SPILL_FLIP, and the un-flipped id must be a real ShapeId; - a latched megamorphic site (D3): the slot guess in the compact word's high half, confirmed by the receiver's shape record (guess < POSBOUND and one key-atom word compare); a wrong guess gets one bounded scan of the first 32 positional keys, and the found position re-aims the site word unless it holds a stamp (D3b). It allocates, collects, locks, throws and calls nothing, so it is Leaf in the call-effects tables and nothing is spilled or relocated across it. The slow entry asks the inherited-read cache for a never-primed site, then runs the miss body. The directory operand is PERRY_AGENT_PTRS slot 0, which is never null (statically PERRY_EMPTY_SHAPE_DIR until the slab publishes its mirror): one initial-exec load on ELF executables, the TEB TLS array plus the runtime's PERRY_AGENT_PTRS_SECREL on Windows x86-64, the HotTls TSD read on Apple aarch64, and the perry_shape_dir_cell leaf accessor elsewhere (x86-64 Darwin, ELF dylib/staticlib outputs, wasm). A `length` site passes the empty directory. Absent directory pages and chunks are shared all-EMPTY statics, so the walk has no null tests. tsc: -0.40% instructions, .text -9.0% (127.28 -> 115.81 MB), RSS -1.2%; lead_mega1 213.1 -> 164.3 instr/iter, lead_poly4 at base. --- changelog.d/megamorphic-read-miss-front.md | 11 + crates/perry-abi/src/lib.rs | 6 +- crates/perry-codegen/src/expr/agent_ptr.rs | 126 ++- .../expr/property_get/array_length_tests.rs | 21 +- .../src/expr/property_get/generic_dispatch.rs | 478 ++------- .../src/expr/property_get/tests.rs | 987 +++++++++--------- .../perry-codegen/src/expr/receiver_range.rs | 9 +- crates/perry-codegen/src/gc_call_effects.rs | 19 + .../src/gc_effects/linux-x86_64.tsv | 2 + .../src/gc_effects/macos-aarch64.tsv | 2 + .../src/gc_effects/windows-x86_64.tsv | 2 + crates/perry-codegen/src/module/linkage.rs | 8 + crates/perry-codegen/src/root_reload.rs | 2 + .../src/runtime_decls/objects.rs | 21 + .../perry-codegen/src/wasm32/runtime_abi.tsv | 2 + crates/perry-runtime/src/agent_ptrs.rs | 58 +- .../perry-runtime/src/object/field_get_set.rs | 5 + .../object/field_get_set/ic_miss/ic_slow.rs | 108 +- .../field_get_set/ic_miss/read_confirm.rs | 523 ++++++++++ crates/perry-runtime/src/object/shapes.rs | 111 +- .../perry-runtime/src/object/shapes_store.rs | 262 +++-- 21 files changed, 1697 insertions(+), 1066 deletions(-) create mode 100644 changelog.d/megamorphic-read-miss-front.md create mode 100644 crates/perry-runtime/src/object/field_get_set/ic_miss/read_confirm.rs diff --git a/changelog.d/megamorphic-read-miss-front.md b/changelog.d/megamorphic-read-miss-front.md new file mode 100644 index 0000000000..13d0cb22ad --- /dev/null +++ b/changelog.d/megamorphic-read-miss-front.md @@ -0,0 +1,11 @@ +A generic property read keeps only the ShapeId compare and the slot load +inline. Its miss makes one GC-leaf call, `js_object_get_field_ic_front`, which +answers a polymorphic way, a spill entry, or a latched megamorphic site whose +slot guess the receiver's own shape record confirms (one POSBOUND bound +compare and one key-atom word compare, with a bounded second chance over the +first 32 positional keys that re-aims the guess). Only what the front declines +reaches the collecting slow entry, which also asks the inherited-read cache +for a never-primed site. Nothing is spilled or relocated across the front +call, and the site reads its agent's shape directory without a call on ELF +executables, Windows x86-64 and Apple aarch64. The shape record grows from 40 +to 48 bytes; tsc's `.text` shrinks by 9%. diff --git a/crates/perry-abi/src/lib.rs b/crates/perry-abi/src/lib.rs index 0cb02c6c2a..eb1ae100cc 100644 --- a/crates/perry-abi/src/lib.rs +++ b/crates/perry-abi/src/lib.rs @@ -10,8 +10,12 @@ pub const ARRAY_HEADER_SIZE: usize = 8; /// `agent_ptrs::PERRY_AGENT_PTRS`: the number of per-agent pointer slots. -/// Slot 0 is reserved (the megamorphic follow-up's shape-record directory). pub const AGENT_PTR_SLOTS: usize = 4; +/// Slot 0: the address of this agent's ordinary shape-directory mirror +/// (`shapes_store::ORDINARY_DIR`), which a generic read site passes to its +/// GC-leaf miss front (`js_object_get_field_ic_front`) so the front reads no +/// thread-local. +pub const AGENT_PTR_SHAPE_DIR: usize = 0; /// Slot 1: the address of this agent's implicit-`this` cell /// (`tls_hot::HotTls::implicit_this`), which a direct method call binds. pub const AGENT_PTR_IMPLICIT_THIS: usize = 1; diff --git a/crates/perry-codegen/src/expr/agent_ptr.rs b/crates/perry-codegen/src/expr/agent_ptr.rs index 7ec3f6bdb0..fcb5db0b45 100644 --- a/crates/perry-codegen/src/expr/agent_ptr.rs +++ b/crates/perry-codegen/src/expr/agent_ptr.rs @@ -15,10 +15,22 @@ //! model (every thread-local access is a TLV thunk call), so the block's //! address is read from the runtime's `HotTls` cache through the pthread //! TSD fast path (`hot_tls.rs`), at `HOT_TLS_AGENT_PTRS_OFFSET`. -//! * [`AgentPtrAccess::Call`] — everything else (Windows, wasm, arm64_32, -//! x86-64 Darwin, dylib/staticlib outputs): the runtime accessor. +//! * [`AgentPtrAccess::WindowsTeb`] — Windows x86-64, any output kind: the +//! same sequence the compiler emits for a native thread-local, spelled out +//! because the runtime cannot export the block under a stable name there +//! (`agent_ptrs.rs`): `gs:[0x58]` (the TEB's `ThreadLocalStoragePointer`) +//! indexed by the image's `_tls_index` gives this thread's TLS block for +//! the image, and the block sits at `PERRY_AGENT_PTRS_SECREL` (a `.secrel32` +//! the runtime emits for its own static) inside it. Emitted code and the +//! runtime are linked into ONE image, so `_tls_index` is theirs. +//! * [`AgentPtrAccess::Call`] — everything else (wasm, arm64_32, Windows +//! aarch64, x86-64 Darwin, ELF dylib/staticlib outputs): the runtime +//! accessor. x86-64 Darwin has no call-free thread-local model (every +//! Mach-O thread-local access is a TLV thunk call) and the runtime's +//! pthread-TSD fast path (`HotTls`, the Apple aarch64 route) is built for +//! aarch64 only. //! -//! In both inline forms a null slot means "not published yet" and takes the +//! In every inline form a null slot means "not published yet" and takes the //! accessor call, which publishes it; so the inline forms and the call are //! equivalent by construction. @@ -35,10 +47,18 @@ pub(crate) const AGENT_PTRS_SYMBOL: &str = "PERRY_AGENT_PTRS"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum AgentPtrAccess { InitialExec, + WindowsTeb, AppleTsd, Call, } +/// `PERRY_AGENT_PTRS`'s offset in the image's TLS block on Windows x86-64 +/// (`agent_ptrs.rs`), and the image's TLS index. +pub(crate) const AGENT_PTRS_SECREL_SYMBOL: &str = "PERRY_AGENT_PTRS_SECREL"; +pub(crate) const TLS_INDEX_SYMBOL: &str = "_tls_index"; +/// `NT_TIB64`/`TEB64.ThreadLocalStoragePointer`, off `gs`. +const TEB_TLS_POINTER_OFFSET: &str = "88"; + thread_local! { /// Whether the module being compiled is linked into an EXECUTABLE (set per /// module by `codegen::compile_module`); anything else must not assume @@ -62,12 +82,104 @@ pub(crate) fn agent_ptr_access(ctx: &FnCtx<'_>) -> AgentPtrAccess { if elf && OUTPUT_IS_EXECUTABLE.with(|c| c.get()) { return AgentPtrAccess::InitialExec; } + if triple.starts_with("x86_64") && triple.contains("windows") { + return AgentPtrAccess::WindowsTeb; + } if super::hot_tls::inline_hot_tls_enabled(ctx) { return AgentPtrAccess::AppleTsd; } AgentPtrAccess::Call } +/// The current value of per-agent pointer `slot`, for a GC-leaf callee that +/// accepts `absent` (a constant operand meaning "not available here") in its +/// place: one initial-exec load in an ELF executable (the slot must never be +/// null there); the `HotTls` read on Apple aarch64, `absent` when the direct +/// TSD path is unavailable or the block is not published; the slot's `gc-leaf` +/// runtime accessor everywhere else. No null test and no fallback call on the +/// inline forms, so a site pays only the read. Ends in the block where the +/// returned register holds the value. +pub(crate) fn emit_agent_ptr_or(ctx: &mut FnCtx<'_>, slot: usize, absent: &str) -> String { + debug_assert!(slot < AGENT_PTR_SLOTS); + let slot_off = (slot * 8).to_string(); + let access = agent_ptr_access(ctx); + match access { + AgentPtrAccess::InitialExec | AgentPtrAccess::WindowsTeb => { + let at = emit_slot_addr(ctx, access, &slot_off); + ctx.block().load(PTR, &at) + } + AgentPtrAccess::AppleTsd => { + let lookup = super::hot_tls::emit_hot_tls_lookup(ctx, "agent_ptr"); + let field = super::hot_tls::hot_tls_field( + ctx, + &lookup.hot, + &HOT_TLS_AGENT_PTRS_OFFSET.to_string(), + ); + let blk = ctx.block(); + let block_ptr = blk.load(PTR, &field); + let at = blk.gep( + crate::types::I8, + &block_ptr, + &[(crate::types::I64, &slot_off)], + ); + let val = blk.load(PTR, &at); + let fast_pred = blk.label.clone(); + let join_idx = ctx.new_block("agent_ptr.join"); + let join_label = ctx.block_label(join_idx); + ctx.block().br(&join_label); + ctx.current_block = lookup.slow_idx; + let slow_pred = ctx.block().label.clone(); + ctx.block().br(&join_label); + ctx.current_block = join_idx; + ctx.block() + .phi(PTR, &[(&val, &fast_pred), (absent, &slow_pred)]) + } + AgentPtrAccess::Call => ctx.block().call(PTR, agent_ptr_accessor(slot), &[]), + } +} + +/// The address of the slot `slot_off` bytes into this thread's block, for the +/// two forms that name the block through the thread pointer (module docs). +fn emit_slot_addr(ctx: &mut FnCtx<'_>, access: AgentPtrAccess, slot_off: &str) -> String { + let blk = ctx.block(); + let block = match access { + AgentPtrAccess::InitialExec => format!("@{AGENT_PTRS_SYMBOL}"), + AgentPtrAccess::WindowsTeb => { + // `mov gs:[0x58]` — a plain load in the x86 `gs` address space + // (256). Plain, not volatile: it is re-read after every call, and + // a thread switch happens only inside a call. + let tls_array = blk.next_reg(); + blk.emit_raw(format!( + " {tls_array} = load ptr, ptr addrspace(256) inttoptr (i64 {TEB_TLS_POINTER_OFFSET} to ptr addrspace(256)), align 8" + )); + let index = blk.load(crate::types::I32, &format!("@{TLS_INDEX_SYMBOL}")); + let index = blk.zext(crate::types::I32, &index, crate::types::I64); + let entry = blk.gep(PTR, &tls_array, &[(crate::types::I64, &index)]); + let image_block = blk.load(PTR, &entry); + let secrel = blk.load(crate::types::I32, &format!("@{AGENT_PTRS_SECREL_SYMBOL}")); + let secrel = blk.zext(crate::types::I32, &secrel, crate::types::I64); + blk.gep( + crate::types::I8, + &image_block, + &[(crate::types::I64, &secrel)], + ) + } + AgentPtrAccess::AppleTsd | AgentPtrAccess::Call => { + unreachable!("{access:?} does not name the block through the thread pointer") + } + }; + blk.gep(crate::types::I8, &block, &[(crate::types::I64, slot_off)]) +} + +/// The `gc-leaf` runtime accessor of per-agent pointer `slot`. +fn agent_ptr_accessor(slot: usize) -> &'static str { + match slot { + crate::runtime_abi::AGENT_PTR_SHAPE_DIR => "perry_shape_dir_cell", + crate::runtime_abi::AGENT_PTR_IMPLICIT_THIS => "perry_implicit_this_cell", + _ => unreachable!("agent pointer slot {slot} has no accessor"), + } +} + /// Emit a load of per-agent pointer `slot`, falling back to `fallback_fn` /// (a `gc-leaf` runtime accessor `() -> ptr` that also publishes it). Ends /// in a fresh block where the returned register holds the pointer. @@ -79,15 +191,11 @@ pub(crate) fn emit_agent_ptr(ctx: &mut FnCtx<'_>, slot: usize, fallback_fn: &str } let slot_off = (slot * 8).to_string(); let (fast_pred, fast_val, slow_idx) = match access { - AgentPtrAccess::InitialExec => { + AgentPtrAccess::InitialExec | AgentPtrAccess::WindowsTeb => { let slow_idx = ctx.new_block("agent_ptr.slow"); let fast_idx = ctx.new_block("agent_ptr.fast"); + let at = emit_slot_addr(ctx, access, &slot_off); let blk = ctx.block(); - let at = blk.gep( - crate::types::I8, - &format!("@{AGENT_PTRS_SYMBOL}"), - &[(crate::types::I64, &slot_off)], - ); let val = blk.load(PTR, &at); let ok = blk.icmp_ne(PTR, &val, "null"); let fast_label = ctx.block_label(fast_idx); diff --git a/crates/perry-codegen/src/expr/property_get/array_length_tests.rs b/crates/perry-codegen/src/expr/property_get/array_length_tests.rs index d664efe8d6..51819db528 100644 --- a/crates/perry-codegen/src/expr/property_get/array_length_tests.rs +++ b/crates/perry-codegen/src/expr/property_get/array_length_tests.rs @@ -329,7 +329,26 @@ fn a_length_read_serves_a_live_plain_array_off_the_shape_compare() { // cannot heal in one edge — continues exactly where the compare's false // edge used to go. let refused = assert_plain_array_arm(&blocks, "pget.array_kind", true); - assert_eq!(strip_suffix(&refused), "pic.token.miss"); + assert_eq!(strip_suffix(&refused), "pic.miss.front"); + + // S6: a `length` site's miss front is handed the runtime's EMPTY + // directory, so its latched edge is never confirmed from the receiver's + // shape. An Array-subclass receiver serves `length` from its elements + // store; the `length` its shape may name is not the answer + // (`read_confirm::tests::a_length_site_is_never_confirmed_from_the_shape` + // is the runtime half). + let front = ir + .lines() + .find(|l| l.contains(" = call double @js_object_get_field_ic_front(")) + .unwrap_or_else(|| panic!("expected the miss front call:\n{ir}")); + assert!( + front.contains("@js_object_get_field_ic_front(ptr @PERRY_EMPTY_SHAPE_DIR, "), + "a `length` site must pass the empty directory:\n{front}" + ); + assert!( + !ir.contains("ptr @PERRY_AGENT_PTRS, i64 0"), + "a `length` site reads no directory at all:\n{ir}" + ); // The merge takes the arm's value. let (load_label, load_body) = block(&blocks, "pget.array_length"); diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 7dead6729d..3bcc25f880 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -19,8 +19,8 @@ use crate::types::{DOUBLE, I1, I32, I64, I8, PTR}; /// Since #9708 codegen emits only the 8-byte slot (`@perry_ic_N = private /// global ptr null`) and the runtime allocates the words itself, sized from /// its own `PicCache` — so the constant is no longer an emission width, but -/// the emitted way GEPs (`PIC_WAY_BASE + PIC_WAYS * 2` words) must still -/// land inside that allocation. perry-codegen does not depend on +/// the runtime's miss entry reads its ways at `PIC_WAY_BASE + PIC_WAYS * 2` +/// words, which the pairing tests keep inside that allocation. perry-codegen does not depend on /// perry-runtime (the same reason `INLINE_SLOT_FLOOR` is duplicated in /// `target_layout`), so the pairing is held by `pic_cache_layout_matches_runtime` /// here and `pic_cache_words_match_codegen` in the runtime: change one and both @@ -29,9 +29,11 @@ use crate::types::{DOUBLE, I1, I32, I64, I8, PTR}; pub(crate) const PIC_CACHE_WORDS: usize = 12; /// First word of the polymorphic way array (words 0..2 are the MRU entry and /// word 3 is the gate). Mirrors the runtime's `PIC_WAY_BASE`. +#[cfg(test)] pub(crate) const PIC_WAY_BASE: usize = 4; /// `(token, slot)` ways beyond the MRU entry; a site resolves `PIC_WAYS + 1` /// shapes inline. Mirrors the runtime's `PIC_WAYS`. +#[cfg(test)] pub(crate) const PIC_WAYS: usize = 4; /// The value a per-site compact MRU word (`@perry_ic_N_packed_get`) holds /// before anything has primed it. @@ -63,14 +65,17 @@ pub(crate) const PACKED_GET_EMPTY: i64 = 0xFFFF_FFFF; /// The hit path's compare therefore REFUSES a spill entry without asking a /// question of its own, which is what lets the overflow-bit test (a 10-byte /// `movabs`, a `test` and a branch, on every read of every site) leave the hit -/// path entirely. The spill entry is still served: `pic.token.miss` un-flips -/// the bit, and a match branches straight to the slow entry, which decodes the -/// same word. See the design note at the head of this function. +/// path entirely. The spill entry is still served: the site's one miss call +/// (`js_object_get_field_ic_slow`) un-flips the bit first thing. Codegen no +/// longer reads the word's encoding; the mirror is kept for the pairing tests. +#[cfg(test)] pub(crate) const PACKED_SPILL_FLIP: i64 = 0xC000_0000; /// Way-state word: `> 0` means at least one way is populated and the compares /// are worth running; `0` (fresh) and a negative megamorphic countdown -/// both skip them. Mirrors the runtime's `PIC_WAY_STATE`. +/// both skip them. Mirrors the runtime's `PIC_WAY_STATE` (read there only, +/// by the miss entry; kept here for the pairing tests). +#[cfg(test)] pub(crate) const PIC_WAY_STATE: usize = 3; // Word 2 is unused: it held the Array-subclass named-prefix token, site state // not derived from one shape, retired by S6. A site holds `(ShapeId, slot)` @@ -90,6 +95,11 @@ pub(crate) const PIC_WAY_STATE: usize = 3; /// re-reading it at each consumer is not merely cheap, it is the correct /// reading: every cold block sees the pool's current address rather than one /// captured before whatever collected. +/// The runtime's never-written empty shape directory (`shapes_store.rs`), +/// which confirms nothing: a `length` site's front operand, and the value +/// where the agent's own directory is not readable inline. +const EMPTY_SHAPE_DIR: &str = "@PERRY_EMPTY_SHAPE_DIR"; + fn emit_key_handle(ctx: &mut FnCtx<'_>, key_handle_global: &str) -> String { let blk = ctx.block(); let key_box = blk.load(DOUBLE, key_handle_global); @@ -776,9 +786,7 @@ pub(crate) fn lower_generic_property_get( // the hit path pays a `jmp` to the survivor instead of falling through. let hit_live_idx = crate::expr::typed_feedback_emission_enabled().then(|| ctx.new_block("pic.hit.live")); - let miss_idx = ctx.new_block("pic.miss"); let hit_label = ctx.block_label(hit_idx); - let miss_label = ctx.block_label(miss_idx); // Small-handle receivers (native-module registry ids) must never be // dereferenced. Pre-#7883 they were kept out of the loads by selecting a // sentinel address and AND-ing `is_real_ptr` into `hit`; the branch does @@ -877,12 +885,11 @@ pub(crate) fn lower_generic_property_get( ctx.block().inttoptr(I64, &pcid_addr) } }; - // The hot ShapeId load has exactly ONE use: the compare. The two cold - // consumers of the same word — the spill compare in `pic.token.miss` and - // the way token in `pic.ways` — read it AGAIN there, through an atomic - // load that GVN will not merge with this one. That is a deliberate - // re-derivation on the miss path (one load, on a path that is about to - // spend hundreds), and it is what lets isel fold this load into the + // The hot ShapeId load has exactly ONE use: the compare. The cold + // consumers of the same word — the spill compare and the way tokens — + // live in the miss front, which reads it AGAIN from the receiver. That is + // a deliberate re-derivation on the miss path (one load, on a path that + // is about to make a call), and it is what lets isel fold this load into the // compare itself: `cmp %ecx, 4(%rdi)` instead of a `mov` and a `cmp`, // one instruction fewer on every hit. With the word live into the cold // blocks it had to sit in a register. @@ -893,11 +900,27 @@ pub(crate) fn lower_generic_property_get( // proves the shape without a discriminator OR or a wide token mask. let packed_stamp = ctx.block().trunc(I64, &packed_word, I32); let token_eq = ctx.block().icmp_eq(I32, &pcid, &packed_stamp); - let token_miss_idx = ctx.new_block("pic.token.miss"); - let token_miss_label = ctx.block_label(token_miss_idx); + // What stays inline is the hit: the ShapeId compare and the load + // (first-read D3). Everything else a miss can be — a spill entry, a + // polymorphic way, a latched site's slot guess, an inherited read, the + // collecting miss — is answered by a runtime call. + // + // Outside profiling builds, the compare's false edge first makes ONE + // GC-leaf call, `js_object_get_field_ic_front` (`pic.miss.front`): it + // answers a spill entry, a polymorphic way and a latched site's + // shape-confirmed slot guess, and returns `TAG_HOLE` for anything else. + // Only then does the site branch to the collecting slow call, so the + // statepoint spills and reloads that call needs sit on that cold edge + // alone. Receiver-validation failures skip the front: it answers + // nothing for a receiver that is not a real object. + let front_idx = + (!crate::expr::typed_feedback_emission_enabled()).then(|| ctx.new_block("pic.miss.front")); + let token_miss_label = front_idx + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| cold_label.clone()); // `.length` on a plain Array (#10714), tested on the compare's FALSE edge // and nowhere earlier. See `emit_plain_array_length_arm` for what it - // answers and what it leaves to `pic.token.miss`. + // answers and what it leaves to the miss front (`pic.miss.front`). // // An Array can never take the hit: its `+4` word is `capacity`, a count // rule 3 (#10828) bounds below the ShapeId floor, so the compare above @@ -931,102 +954,6 @@ pub(crate) fn lower_generic_property_get( .cond_br(&token_eq, &hit_label, &token_miss_label); } - ctx.current_block = token_miss_idx; - // The cold re-read of the ShapeId word — see the hot load above. - let pcid = ctx.block().load_atomic_monotonic(I32, &pcid_ptr, 4); - let pcid64 = ctx.block().zext(I32, &pcid, I64); - // pic_prime_get is the only production writer of get-cache tokens and - // refuses the zero-ShapeId token. All remaining tokens carry a valid, - // never-reused ShapeId; vacant entries are zero. Equality therefore - // proves a nonzero stamp without another check on every property read. - // Keyless Object.create(proto) receivers still miss and walk prototypes. - let token = ctx.block().or(I64, &pcid64, "4611686018427387904"); - // The SPILL entry — tested HERE, and nowhere on the hit path. - // - // A key past the object's inline region used to publish its slot into the - // compact word with `IC_SLOT_OVERFLOW_BIT` set, and every read of every - // site paid to ask whether the bit was there: LLVM folds - // `((packed >> 32) & (1 << 30)) == 0` into `packed & (1 << 62)`, which is - // a 10-byte `movabs`, a `test` and a branch on the hit path of sites whose - // field is inline and can never see the bit. - // - // Now a spill entry publishes the SAME ShapeId with `PACKED_SPILL_FLIP` - // flipped into it, which lands it outside the ShapeId range, so the hit - // path's compare refuses it for free. Un-flipping the bit here recognises - // it in three instructions ON THE MISS PATH ONLY, and a match is served - // by `pic.spill.hit` below — skipping the full cache's resolution and the - // polymorphic ways, neither of which can serve a spill key anyway - // (`pic_prime_get` refuses to cascade an encoded slot into a way). - let spill_stamp = ctx - .block() - .xor(I32, &packed_stamp, &PACKED_SPILL_FLIP.to_string()); - let is_spill = ctx.block().icmp_eq(I32, &pcid, &spill_stamp); - let ways_entry_idx = ctx.new_block("pic.token.ways"); - let ways_entry_label = ctx.block_label(ways_entry_idx); - let spill_hit_idx = ctx.new_block("pic.spill.hit"); - let spill_hit_label = ctx.block_label(spill_hit_idx); - ctx.block() - .cond_br(&is_spill, &spill_hit_label, &ways_entry_label); - - // S5: the SPILL hit. The flipped entry is a `(ShapeId, index)` fact like - // the inline one, and the ShapeId alone proves where the value is: the - // key list and the live inline-slot bound it names fix the key's - // position, a position at or past the bound IS its index in the spill - // buffer, and every carrier of the shape has that storage (the runtime - // reserves it for a key claimed without a value, keeps a stored - // `undefined` across buffer growth, and publishes no spill entry while - // spill storage is disabled — `spill_reserve_claimed`, - // `spill_get_present`, `packed_get::prime_get`). So the hit is two - // dependent loads to reach the buffer and one at the fixed index, with no - // null, bound or hole test: - // - // meta = [handle + META] ObjectHeader.meta - // spill = [meta + 32] ObjectMeta.spill - // value = [spill + 8 + index * 8] past the u32 length/capacity words - // - // Nothing here allocates or can collect, so the receiver needs no root. - ctx.current_block = spill_hit_idx; - crate::expr::receiver_range::emit_route_note( - ctx.block(), - crate::expr::receiver_range::Route::GenericSpillHit, - ); - let (val_spill, spill_end_label) = emit_spill_hit( - ctx, - fused_recv.as_ref(), - &entry_handle, - &packed_word, - &merge_label, - ); - - // Every way load still requires a resolved full cache. A site that has - // never primed has no cache, so there is nothing to compare against. - // - // That "never primed" edge is also exactly where an INHERITED read lives: - // a key on the prototype chain is never an own slot on the receiver's - // shape, so a site that only ever reads it never resolves its cache, and - // every read of it reaches this branch with `present` false. So that - // edge, and no other, asks the inherited-read cache (#10834/#10842) - // before calling out — see `pic.miss.inherited` below. Every other path - // to the exit (a small handle, a spill entry, an MRU or way miss at a site - // that HAS primed) is unchanged to the instruction; the first placement - // asked on all of them and cost every own-key miss the price of a - // declining probe (+88 on a megamorphic site, +89 on a spill read). - // - // Under `--typed-feedback` the edge keeps its old target: the recording - // blocks put a guard-fail and a fallback-call record on precisely this - // edge, and a read served without a call would have to change one of - // those records. Feedback builds are profiling builds; they keep their - // signal byte-identical and go without the hook. - ctx.current_block = ways_entry_idx; - let token_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); - let inherited_idx = (!crate::expr::typed_feedback_emission_enabled()) - .then(|| ctx.new_block("pic.miss.inherited")); - let never_primed_label = inherited_idx - .map(|idx| ctx.block_label(idx)) - .unwrap_or_else(|| cold_label.clone()); - ctx.block() - .cond_br(&token_cache.present, &miss_label, &never_primed_label); - // `js_object_get_field_ic_miss` primes only slots below the descriptor's // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact // token hit permanently proves that the cached slot remains live and @@ -1040,7 +967,7 @@ pub(crate) fn lower_generic_property_get( } // A matched compact word is now, by construction, an INLINE slot: a // spill-located key publishes its ShapeId flipped by `PACKED_SPILL_FLIP` - // and is recognised in `pic.token.miss` instead. The overflow-bit test + // and is recognised by the miss front instead. The overflow-bit test // that used to stand between this shift and the load is gone from the hit // path — see the note there for what it cost and where it went. let slot = ctx.block().lshr(I64, &packed_word, "32"); @@ -1107,169 +1034,6 @@ pub(crate) fn lower_generic_property_get( } }; - // PIC miss on the MRU entry — before paying for the call, try the - // polymorphic ways (#7753). - // - // The slow entry is not a cheap fallback: it re-derives the receiver kind - // from scratch (proxy band, closure magic, registered-buffer and - // typed-array registries, small-handle dispatch), reads the - // accessors-in-use thread-local, then linear-scans the keys array with a - // `js_string_equals` per key. On a site whose receiver alternates between a - // handful of shapes — the shape of every discriminated-union dispatch — - // a single-entry cache misses on essentially every read and that whole - // ladder runs per field access. Measured on a tree-walking interpreter it - // was ~34% of run time. - // - // The ways are consulted only here, so a genuinely monomorphic site keeps - // the exact instruction sequence it had before this block existed. The - // typed-feedback counters are also recorded before the way compares, so a - // way hit still reports guard-fail + fallback-call exactly as it did when - // it was a real miss — the feedback heuristics see an unchanged signal - // (the site IS polymorphic; only the cost of that changed). - // - // # Why this block is DOMINATED by `pic.token` (#7907) - // - // Its only predecessor is `pic.token.miss`, which is `pic.token`'s. The - // exact descriptor identity proves cached-slot bounds, so `token` is - // everything the way compares need, and the cache pointer arrives on one - // edge rather than through a phi. - // - // #7883 could not rely on that: it routed the two receiver-validation - // failures here as well, which left the values live on only some edges, so - // the block **re-derived them** — header and identity loads, the token - // select, and a safe-address select for small-handle receivers. - // That was correct, and it was justified as cold. It is not cold: on a site - // whose receiver rotates over more shapes than the MRU entry holds — the - // shape #7753's ways exist for — this block runs on nearly every read, so - // the duplicate ladder sat on the hot path. Measured on `interp.ts`'s - // `evalNode`, the single hottest instruction in the whole program was the - // redundant receiver reconstruction inside this block. - ctx.current_block = miss_idx; - let cache_ref = token_cache.cache.clone(); - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_guard_fail", - &[(I64, &feedback_site_id)], - ); - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_fallback_call", - &[(I64, &feedback_site_id)], - ); - - // Every way contains a ShapeId token. A non-zero receiver token keeps an - // empty way from matching; no GC-epoch guard is necessary because ids are - // never reused and descriptor identity survives key relocation. - // - // The compares sit behind their own branch on `cache[PIC_WAY_STATE] > 0` - // rather than being folded into one flat predicate, because a site whose - // receiver rotation is WIDER than the ways hold never hits one and would - // otherwise pay four dependent loads on every read: measured at **+37%** on - // a 7-shape site, against a 2.5x speedup on a 5-shape one. `pic_prime_get` - // latches that state to `-1` once a site proves itself megamorphic, and a - // fresh site reads `0`, so for both the branch is one load, - // one compare, and a perfectly predicted fall-through to the call — which - // is exactly the pre-#7753 code path. - let state_ptr = ctx - .block() - .gep(I64, &cache_ref, &[(I64, &PIC_WAY_STATE.to_string())]); - let way_state = ctx.block().load(I64, &state_ptr); - let ways_live = ctx.block().icmp_sgt(I64, &way_state, "0"); - let ways_idx = ctx.new_block("pic.ways"); - let ways_label = ctx.block_label(ways_idx); - ctx.block().cond_br(&ways_live, &ways_label, &call_label); - - ctx.current_block = ways_idx; - // `is_object` is not ANDed in any more: it is statically true on every edge - // that reaches here (#7907 — see the dominance note above). - // Reduced as a BALANCED TREE, not as a left fold. At most one way can hold - // a given token (`pic_prime_get` evicts a duplicate before it writes one, - // and pic_prime_get excludes zero-ShapeId tokens), so the association is - // free to change — but the fold made `way_slot` a chain of `PIC_WAYS` - // dependent `csel`s whose last node is the operand of the bounds compare - // that gates the branch out of this block. On `interp.ts` that node was the - // hottest instruction in `evalNode` (#7907). The tree halves the chain. - let mut lanes: Vec<(String, String)> = Vec::with_capacity(PIC_WAYS); - for w in 0..PIC_WAYS { - let tok_ptr = ctx.block().gep( - I64, - &cache_ref, - &[(I64, &(PIC_WAY_BASE + w * 2).to_string())], - ); - let way_tok = ctx.block().load(I64, &tok_ptr); - let eq = ctx.block().icmp_eq(I64, &way_tok, &token); - let slot_ptr = ctx.block().gep( - I64, - &cache_ref, - &[(I64, &(PIC_WAY_BASE + w * 2 + 1).to_string())], - ); - let way_slot_val = ctx.block().load(I64, &slot_ptr); - let lane_slot = ctx.block().select(I1, &eq, I64, &way_slot_val, "0"); - lanes.push((eq, lane_slot)); - } - while lanes.len() > 1 { - let mut merged: Vec<(String, String)> = Vec::with_capacity(lanes.len().div_ceil(2)); - for pair in lanes.chunks(2) { - match pair { - [(a_any, a_slot), (b_any, b_slot)] => { - let any = ctx.block().or(I1, a_any, b_any); - let slot = ctx.block().select(I1, a_any, I64, a_slot, b_slot); - merged.push((any, slot)); - } - [single] => merged.push(single.clone()), - _ => unreachable!("chunks(2) yields one or two elements"), - } - } - lanes = merged; - } - let (way_any, way_slot) = lanes - .pop() - .expect("PIC_WAYS is non-zero, so the reduction leaves exactly one lane"); - let way_load_idx = ctx.new_block("pic.way.load"); - let way_load_label = ctx.block_label(way_load_idx); - ctx.block().cond_br(&way_any, &way_load_label, &call_label); - - ctx.current_block = way_load_idx; - if fused_recv.is_some() { - crate::expr::receiver_range::emit_route_note( - ctx.block(), - crate::expr::receiver_range::Route::GenericWayHit, - ); - } - let way_offset = ctx.block().shl(I64, &way_slot, "3"); - let way_handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); - let way_base = ctx.block().add(I64, &way_handle, &obj_header_size); - let way_field_addr = ctx.block().add(I64, &way_base, &way_offset); - let way_field_ptr = ctx.block().inttoptr(I64, &way_field_addr); - let val_way = ctx.block().load(DOUBLE, &way_field_ptr); - // The loaded value is the answer here too, for the reason the shape-gated - // hit above needs no `TAG_HOLE` compare (#10826: a successful delete - // ALWAYS moves the receiver's ShapeId, so an exact-id match proves the - // slot it names is live). - // - // A way pair is not a second kind of cache entry needing its own - // argument. `pic_prime_get` is the ONLY writer of a way, and the only - // values it ever writes into one are `prev_tok`/`prev_slot` — the pair - // that was sitting in the MRU entry. Every `(token, slot)` a way holds is - // therefore an MRU pair that aged out; the token it is compared against is - // the same receiver ShapeId word the MRU compare reads; and ShapeIds are - // never reused. Whatever makes the MRU pair safe to load without a hole - // check makes the way pair safe — the entry did not become weaker by - // moving one word over. - // - // The two ways in which a way pair differs from an MRU pair both narrow - // it: an overflow-encoded slot is refused entry to a way at all, and a way - // is consulted only after the MRU entry has already missed. - let way_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); - - // #7907: receiver-validation failure. A receiver that gets here can never - // match a way — the compares require a real pointer to a plain - // descriptor-free `ObjectHeader` — so it goes straight to the handler, - // which reproduces the whole ladder anyway (proxy band, closure magic, - // buffer/typed-array registries, small-handle dispatch). The typed-feedback - // counters are the same two records on the same edges, so the feedback - // signal is byte-identical to what the pre-T1 blocks reported. if let Some(cold_idx) = cold_idx { ctx.current_block = cold_idx; crate::expr::emit_typed_feedback_record_call( @@ -1285,48 +1049,14 @@ pub(crate) fn lower_generic_property_get( ctx.block().br(&call_label); } - // The inherited-read hook, on the never-primed edge only (see the branch - // that reaches it, in `pic.token.ways`). A read whose key lives on the - // prototype chain can never take the own-slot hit — the receiver's shape - // says the key is not own — so before this block it paid the slow entry's - // prologue and dispatch (79 of an inherited read's 204 instructions, - // measured by the inherited-reads lane) just to reach the same lookup - // inside `get_field_ic_miss_impl`. `js_inherited_read_cache_hit_f64` is - // a pure state read — it allocates nothing, triggers no GC and runs no - // user code — so it is a leaf in `gc_call_effects.rs` and - // `root_reload.rs`: no spill, no reload around it. `TAG_HOLE` is its - // decline sentinel, which no ordinary value can be, so the answer is one - // compare, with the SERVED edge as the true edge like every guard-passing - // edge in this tower (#7883); a decline continues to the one exit exactly - // as the never-primed edge did before. Nothing is primed from here: - // priming stays in the miss handler, the one place that already knows - // the key is not own without a second search. The versioned-loop deopt - // note is emitted here as it is on the exit, so entering either cold arm - // still records the bailout. - let inherited_arm = inherited_idx.map(|idx| { - ctx.current_block = idx; - crate::expr::emit_versioned_loop_callback_deopt(ctx); - let inh_key_handle = emit_key_handle(ctx, &key_handle_global); - let handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); - let recv_ptr = ctx.block().inttoptr(I64, &handle); - let key_ptr = ctx.block().inttoptr(I64, &inh_key_handle); - let val_inherited = ctx.block().call( - DOUBLE, - "js_inherited_read_cache_hit_f64", - &[(PTR, &recv_ptr), (PTR, &key_ptr)], - ); - let inherited_bits = ctx.block().bitcast_double_to_i64(&val_inherited); - let inherited_served = - ctx.block() - .icmp_ne(I64, &inherited_bits, crate::nanbox::TAG_HOLE_I64); - let inherited_end_label = ctx.block().label.clone(); - ctx.block() - .cond_br(&inherited_served, &merge_label, &cold_label); - (val_inherited, inherited_end_label) - }); - - // The object exit: one call reproducing every pointer-path arm this tower - // used to expand. + // The collecting exit. It receives what the miss front declined (and, + // without a front, every miss) with the same four operands as before + // first-read D3: a never-primed site's inherited-read cache + // (#10834/#10842) is asked inside it, then the full miss body runs. + // Under `--typed-feedback` every miss records guard-fail + fallback-call + // on the way in (`cold` above), the signal those builds always saw for a + // non-hit. The versioned-loop deopt note is emitted here as well, so + // entering this cold arm still records the bailout. ctx.current_block = call_idx; crate::expr::emit_versioned_loop_callback_deopt(ctx); let miss_key_handle = emit_key_handle(ctx, &key_handle_global); @@ -1344,6 +1074,53 @@ pub(crate) fn lower_generic_property_get( let miss_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); + // The front (see `token_miss_label`). Its operands: the agent's + // shape-directory mirror (`PERRY_AGENT_PTRS` slot 0), so the front reads + // no thread-local; the receiver; the key exactly as the pool global holds + // it — STRING-tagged, the form a canonical key list stores, so the + // latched confirm compares one word; and the site's two cache words. A + // `length` site passes the runtime's empty directory + // (`PERRY_EMPTY_SHAPE_DIR`): an Array-subclass receiver serves `length` + // from its elements store, which no key list names, so its latched edge + // must not be confirmed from the shape. The call is a + // `"gc-leaf-function"` (the front is `Leaf` in the generated call-effects + // table): nothing live across it is spilled or relocated. + let front_arm = front_idx.map(|front_idx| { + ctx.current_block = front_idx; + let dir = if property == "length" { + EMPTY_SHAPE_DIR.to_string() + } else { + crate::expr::agent_ptr::emit_agent_ptr_or( + ctx, + crate::runtime_abi::AGENT_PTR_SHAPE_DIR, + EMPTY_SHAPE_DIR, + ) + }; + let front_handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); + let key_box = ctx.block().load(DOUBLE, &key_handle_global); + let key_bits = ctx.block().bitcast_double_to_i64(&key_box); + let answered = ctx.block().call( + DOUBLE, + "js_object_get_field_ic_front", + &[ + (PTR, &dir), + (I64, &front_handle), + (I64, &key_bits), + (PTR, &cache_slot_ref), + (PTR, &packed_ref), + ], + ); + let answered_bits = ctx.block().bitcast_double_to_i64(&answered); + let served = ctx + .block() + .icmp_ne(I64, &answered_bits, crate::nanbox::TAG_HOLE_I64); + let front_end_label = ctx.block().label.clone(); + // The SERVED edge is the true edge, like every guard-passing edge in + // the tower (#7883). + ctx.block().cond_br(&served, &merge_label, &call_label); + (answered, front_end_label) + }); + // Native Map/Set `.size`: their common leading field was admitted only by // the exact live GC-kind checks above. Keep the read inline; calling // `js_map_size` / `js_set_size` would reclassify the same receiver again. @@ -1375,12 +1152,11 @@ pub(crate) fn lower_generic_property_get( ctx.current_block = merge_idx; let mut incoming: Vec<(&str, &str)> = vec![ (&val_hit, &hit_end_label), - (&val_way, &way_end_label), (&val_miss, &miss_end_label), (&val_nonptr, &nonptr_end_label), ]; - if let Some((val_inherited, inherited_end_label)) = inherited_arm.as_ref() { - incoming.push((val_inherited, inherited_end_label)); + if let Some((answered, front_end_label)) = front_arm.as_ref() { + incoming.push((answered, front_end_label)); } if let Some((sso_val, sso_end_label)) = sso_arm.as_ref() { incoming.push((sso_val, sso_end_label)); @@ -1394,63 +1170,5 @@ pub(crate) fn lower_generic_property_get( if let Some((len, array_end_label)) = array_length_arm.as_ref() { incoming.push((len, array_end_label)); } - incoming.push((&val_spill, &spill_end_label)); Ok(ctx.block().phi(DOUBLE, &incoming)) } - -/// `pic.spill.hit`'s loads (see the note at its branch): the value at spill -/// index `packed_word >> 32` of the receiver's spill buffer. Returns the value -/// and the label of the block that branches to `merge_label`. -fn emit_spill_hit( - ctx: &mut FnCtx<'_>, - fused_recv: Option<&crate::expr::receiver_range::FusedReceiver>, - entry_handle: &str, - packed_word: &str, - merge_label: &str, -) -> (String, String) { - let ilp32 = crate::target_layout::target_is_ilp32(ctx.target_triple); - let meta_offset = crate::target_layout::object_meta_slot_offset_bytes(ctx.target_triple); - let meta_slot = match fused_recv { - // `handle + META`, addressed from the biased value (`receiver_range`). - Some(f) => { - crate::expr::receiver_range::emit_field_ptr(ctx.block(), &f.biased, meta_offset as i64) - } - None => { - let addr = ctx.block().add(I64, entry_handle, &meta_offset.to_string()); - ctx.block().inttoptr(I64, &addr) - } - }; - let meta = if ilp32 { - let narrow = ctx.block().load(I32, &meta_slot); - ctx.block().zext(I32, &narrow, I64) - } else { - ctx.block().load(I64, &meta_slot) - }; - let meta_ptr = ctx.block().inttoptr(I64, &meta); - let spill_slot = ctx.block().gep( - I8, - &meta_ptr, - &[( - I64, - &crate::target_layout::OBJECT_META_SPILL_OFFSET_BYTES.to_string(), - )], - ); - // `ObjectMeta.spill` is a `u64` on every target (the buffer address, - // zero-extended on ILP32). - let spill = ctx.block().load(I64, &spill_slot); - let spill_ptr = ctx.block().inttoptr(I64, &spill); - let index = ctx.block().lshr(I64, packed_word, "32"); - let elements = ctx.block().gep( - I8, - &spill_ptr, - &[( - I64, - &crate::target_layout::ARRAY_HEADER_SIZE_BYTES.to_string(), - )], - ); - let value_ptr = ctx.block().gep(DOUBLE, &elements, &[(I64, &index)]); - let value = ctx.block().load(DOUBLE, &value_ptr); - let end_label = ctx.block().label.clone(); - ctx.block().br(merge_label); - (value, end_label) -} diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 928bed2a34..5f67320c82 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -200,6 +200,13 @@ fn no_call_location_without_debug_symbols() { /// #8067: the primary property-read PIC identity is the authoritative ShapeId /// only. Word 2 may carry the independent Array-subclass named-prefix proof, /// but it is consulted only after this exact ShapeId predicate fails. +/// +/// First-read D3: the hit is the receiver's `+4` word compared, as an `i32`, +/// with the compact word's low half — no discriminated token is formed at the +/// site at all. The polymorphic ways' `PIC_ID_TOKEN_BIT | ShapeId` tokens are +/// compared inside the miss front (`js_object_get_field_ic_front`), so the +/// token bit appearing in emitted IR again would mean a way compare crept back +/// inline. #[test] fn generic_property_get_hit_path_is_shape_id_only() { let ir = emit(false, None); @@ -207,9 +214,25 @@ fn generic_property_get_hit_path_is_shape_id_only() { ir.contains("@perry_ic_"), "test premise: the generic read reaches the inline monomorphic PIC:\n{ir}" ); + let token = ir + .find("\npic.token") + .unwrap_or_else(|| panic!("expected a pic.token block:\n{ir}")); + let token_body = &ir[token + ..ir[token + 1..] + .find("\n\n") + .map(|o| o + token + 1) + .unwrap_or(ir.len())]; + assert!( + token_body.contains("load i32") + && token_body.contains("trunc i64") + && token_body.contains("icmp eq i32"), + "the hit is the ShapeId word compared with the compact word's low \ + half:\n{token_body}" + ); assert!( - ir.contains("4611686018427387904"), - "hit path must form a discriminated ShapeId token:\n{ir}" + !ir.contains("4611686018427387904"), + "no discriminated way token may be formed at the site — the ways are \ + the miss front's:\n{ir}" ); assert!( !ir.contains("@PERRY_IC_EPOCH"), @@ -355,7 +378,13 @@ fn fs_promises_native_module_value_uses_submodule_singleton() { /// be the pre-#9708 shape coming back, with its 96 B of zero-fill per site. #[test] fn pic_cache_layout_matches_runtime() { - use crate::expr::property_get::generic_dispatch::{PIC_CACHE_WORDS, PIC_WAYS, PIC_WAY_BASE}; + use crate::expr::property_get::generic_dispatch::{ + PIC_CACHE_WORDS, PIC_WAYS, PIC_WAY_BASE, PIC_WAY_STATE, + }; + assert!( + PIC_WAY_STATE < PIC_WAY_BASE, + "the way-state word sits below the ways, as in perry-runtime" + ); assert_eq!( PIC_CACHE_WORDS, 12, "perry-runtime's PIC_CACHE_WORDS is 12; update both sides together" @@ -394,10 +423,22 @@ fn pic_cache_layout_matches_runtime() { fills on the first prime (#9708), got:\n{def}\n\nIR:\n{ir}" ); } + // The full cache is the runtime's to dereference: the site hands the slot's + // ADDRESS to the miss front and the slow entry, which test it for null + // (`read_confirm::tests::the_front_answers_a_way_and_declines_a_null_cache`, + // `ic_slow::tests::an_unresolved_cache_slot_is_never_dereferenced`). A + // site that loaded the slot itself would have to prove it non-null first. assert!( - ir.contains("load ptr, ptr @perry_ic_") && ir.contains("icmp ne ptr "), - "the full-cache fallback must prove the slot non-null before reading \ - a cache word:\n{ir}" + !ir.contains("load ptr, ptr @perry_ic_"), + "the site must not dereference the full-cache slot:\n{ir}" + ); + let front = ir + .lines() + .find(|l| l.contains(" = call double @js_object_get_field_ic_front(")) + .unwrap_or_else(|| panic!("expected the miss front call:\n{ir}")); + assert!( + front.contains("ptr @perry_ic_") && front.contains("_packed_get)"), + "the front must receive the cache slot and the compact word:\n{front}" ); } @@ -454,253 +495,244 @@ fn array_subclass_named_prefix_proof_is_reached_through_the_one_exit() { ); } -/// #7753: the polymorphic ways must be consulted BEFORE the miss call, and the -/// monomorphic path must not have grown any work. +/// The emitted function holding the generic tower, split into +/// `(label, trimmed body lines)` blocks. Register names restart in every +/// function, so every def/use question must be asked inside this one. +fn tower_blocks(ir: &str) -> Vec<(String, Vec)> { + let func = ir + .split("\ndefine ") + .find(|f| f.contains("\npic.miss.call")) + .unwrap_or_else(|| panic!("no function contains the generic tower:\n{ir}")); + let mut blocks: Vec<(String, Vec)> = Vec::new(); + for line in func.lines() { + if !line.starts_with(' ') && line.ends_with(':') { + blocks.push((line.trim_end_matches(':').to_string(), Vec::new())); + } else if let Some((_, body)) = blocks.last_mut() { + if !line.trim().is_empty() { + body.push(line.trim().to_string()); + } + } + } + blocks +} + +/// The one block whose label starts with `prefix`. +fn tower_block<'b>(blocks: &'b [(String, Vec)], prefix: &str) -> (&'b str, &'b [String]) { + let found: Vec<_> = blocks + .iter() + .filter(|(l, _)| l.starts_with(prefix)) + .collect(); + assert_eq!(found.len(), 1, "expected one `{prefix}` block: {blocks:?}"); + (found[0].0.as_str(), &found[0].1) +} + +/// `(cond, true target, false target)` of a block's `br i1` terminator. +fn tower_cond_br(body: &[String]) -> (String, String, String) { + let term = body.last().expect("a terminated block"); + let parts: Vec<&str> = term + .strip_prefix("br i1 ") + .unwrap_or_else(|| panic!("expected a conditional branch: {term}")) + .split(", ") + .collect(); + let label = |s: &str| s.trim_start_matches("label %").to_string(); + (parts[0].to_string(), label(parts[1]), label(parts[2])) +} + +/// #7753: the polymorphic ways must be consulted BEFORE the collecting miss +/// call, and the monomorphic path must not have grown any work. /// /// A one-entry cache misses on essentially every read at a site whose receiver /// alternates between shapes — the shape of every discriminated-union dispatch -/// — and each miss runs the full `js_object_get_field_ic_miss` ladder -/// (proxy/closure/buffer/typed-array probes, an accessors thread-local, then a -/// linear keys scan with a `js_string_equals` per key). If the way block is -/// ever deleted or floated below the call it stops paying for itself entirely, -/// and nothing else in the suite would show it — the program still computes the -/// right answer, just slowly. So assert the ORDER, not merely the presence. +/// — and each miss that reaches the collecting slow entry pays its statepoint +/// and the full miss ladder. If the ways are ever moved behind that call they +/// stop paying for themselves, and nothing else in the suite would show it — +/// the program still computes the right answer, just slowly. So assert the +/// ORDER, not merely the presence. +/// +/// First-read D3: the ways are asked by the GC-leaf miss front +/// (`js_object_get_field_ic_front`, ways first — pinned by +/// `read_confirm::tests::the_front_answers_a_way_and_declines_a_null_cache`), +/// so the order is a CFG fact here: the ShapeId compare's false edge is the +/// front, the front's SERVED edge is the merge, and the slow call is reached +/// from the front only on its decline edge. #[test] fn generic_property_get_tries_ways_before_calling_the_miss_handler() { let ir = emit(false, None); + let blocks = tower_blocks(&ir); + let (_, token) = tower_block(&blocks, "pic.token"); + let (_, on_hit, on_miss) = tower_cond_br(token); + assert!(on_hit.starts_with("pic.hit"), "{token:?}"); assert!( - ir.contains("@perry_ic_"), - "test premise: the generic read reaches the inline PIC:\n{ir}" + on_miss.starts_with("pic.miss.front"), + "the compare's miss edge must reach the front (the ways) first: {token:?}" ); - use crate::expr::property_get::generic_dispatch::{PIC_WAYS, PIC_WAY_BASE, PIC_WAY_STATE}; - - // Block *text* order is an artifact of emission order, so assert the CFG - // instead: the block that calls the miss handler must be reachable only as - // a branch target of the way block, never straight-line after it. - let ways = ir - .find("\npic.ways") - .unwrap_or_else(|| panic!("expected a pic.ways block:\n{ir}")); - let way_load = ir - .find("\npic.way.load") - .unwrap_or_else(|| panic!("expected a pic.way.load block:\n{ir}")); - let call_block = ir - .find("\npic.miss.call") - .unwrap_or_else(|| panic!("expected a pic.miss.call block:\n{ir}")); - let ways_body = &ir[ways..[way_load, call_block, ir.len()] - .into_iter() - .filter(|&x| x > ways) - .min() - .unwrap()]; + let (front_label, front) = tower_block(&blocks, "pic.miss.front"); assert!( - ways_body.contains("pic.way.load") && ways_body.contains("pic.miss.call"), - "pic.ways must end in a branch choosing between the way load and the \ - miss call — otherwise the compares are not gating anything:\n{ways_body}" + front + .iter() + .any(|l| l.contains("call double @js_object_get_field_ic_front(")), + "{front:?}" ); assert!( - !ways_body.contains("call double @js_object_get_field_ic"), - "the slow call must not sit inside the way block:\n{ways_body}" - ); - // The way compares read (token, slot) pairs at words PIC_WAY_BASE.. and the - // gate reads the state word — all inside pic.ways, none anywhere else. - for w in 0..PIC_WAYS { - for word in [PIC_WAY_BASE + w * 2, PIC_WAY_BASE + w * 2 + 1] { - assert!( - ways_body.contains(&format!("i64 {word}\n")), - "way word {word} is never read in the way block:\n{ways_body}" - ); - } - } + !front + .iter() + .any(|l| l.contains("@js_object_get_field_ic_slow(")), + "the slow call must not sit inside the front block: {front:?}" + ); + let (_, served, declined) = tower_cond_br(front); + assert!(served.starts_with("pget.recv_merge"), "{front:?}"); + assert!(declined.starts_with("pic.miss.call"), "{front:?}"); + // Every way of reaching the slow call from the object path goes through + // the front: its only other predecessor is the receiver-validation + // failure, which the front could not answer (no real object). + let (call_label, _) = tower_block(&blocks, "pic.miss.call"); + let preds: Vec<&str> = blocks + .iter() + .filter(|(_, body)| { + body.iter() + .any(|l| l.starts_with("br ") && l.contains(&format!("label %{call_label}"))) + }) + .map(|(l, _)| l.as_str()) + .collect(); assert!( - ir.contains(&format!("i64 {PIC_WAY_STATE}\n")), - "the megamorphic gate must read the way-state word:\n{ir}" + preds.contains(&front_label) + && preds + .iter() + .all(|p| *p == front_label || p.starts_with("pget.recv_")), + "the slow call is reached from the front's decline or a receiver \ + failure only: {preds:?}" ); } -/// #7907: `pic.miss` must be DOMINATED by `pic.token`, so the way compares can -/// use the values that block already computed instead of re-deriving them. +/// #7907: the miss path must be DOMINATED by `pic.token`, so it can use the +/// values that block already computed instead of re-deriving them. /// /// #7883 routed all four failure edges — small-handle receiver, non-object /// receiver, MRU token mismatch, cached slot out of bounds — into one block, -/// which left `token` / `token_nonnull` / `shape_id_eq` live on only some of them -/// and forced the block to reload the whole header ladder. That block is not -/// cold: on a receiver rotation wider than the MRU entry it runs on nearly -/// every read, so the duplicate ladder was hot code. The fix is purely -/// structural — send the two receiver-validation failures to `pic.miss.cold` -/// (they can never resolve a way, since `way_hit` requires a real object) and -/// the dominance follows. +/// which left the token values live on only some of them and forced the block +/// to reload the whole header ladder. That block is not cold: on a receiver +/// rotation wider than the MRU entry it runs on nearly every read, so the +/// duplicate ladder was hot code. The receiver-validation failures go to the +/// slow exit directly (they can never resolve a way: a way hit requires a real +/// object), and the dominance follows. /// -/// Assert the *consequences*, not the block names alone: a re-derivation would -/// show up as duplicate header loads or the small-handle sentinel `select`. +/// First-read D3: that block is `pic.miss.front`, the GC-leaf front call. It +/// must have exactly ONE predecessor, `pic.token`'s false edge, and the site +/// must not re-derive a receiver predicate for it: the front re-reads the +/// ShapeId word itself, so the hot load keeps a single use (the compare) and +/// isel folds it into `cmp %ecx, 4(%rdi)`. #[test] fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { let ir = emit(false, None); - let main_start = ir - .find("define i32 @main()") - .expect("entry module should define main"); - let main_rest = &ir[main_start..]; - let main_end = main_rest - .find("\n}\n") - .expect("main should have a closing brace"); - let main = &main_rest[..main_end]; - assert!( - main.contains("@perry_ic_"), - "test premise: the generic read reaches the inline PIC:\n{ir}" - ); - // T1: the landing block is now the single slow exit itself, and the - // dominance is structural — `pic.miss` has exactly ONE predecessor, - // `pic.token.miss`, which `pic.token` dominates. Assert that directly: - // routing any receiver-validation failure back into `pic.miss` would add a - // predecessor and immediately re-introduce the phis #7907 removed. - // `pic.miss` carries a numeric suffix and `pic.miss.call` starts with the - // same text, so match the block's own label exactly and then count the - // branches whose TARGET is that label (a `br i1` naming both blocks counts - // once, for the right one). - let miss_label = main - .lines() - .filter(|l| !l.starts_with(' ') && l.ends_with(':')) - .map(|l| l.trim_end_matches(':')) - .find(|l| { - l.strip_prefix("pic.miss.") - .is_some_and(|tail| tail.chars().all(|c| c.is_ascii_digit())) - }) - .unwrap_or_else(|| panic!("expected a pic.miss block:\n{ir}")) - .to_string(); - let preds = main - .lines() - .filter(|l| l.trim_start().starts_with("br ")) - .filter(|l| { - l.split("label %") - .skip(1) - .any(|t| t.trim_end_matches(&[',', ' '][..]) == miss_label) + let blocks = tower_blocks(&ir); + let (front_label, _) = tower_block(&blocks, "pic.miss.front"); + let (token_label, token) = tower_block(&blocks, "pic.token"); + let preds: Vec<&str> = blocks + .iter() + .filter(|(_, body)| { + body.iter().any(|l| { + l.starts_with("br ") + && l.split("label %") + .skip(1) + .any(|t| t.trim_end_matches(&[',', ' '][..]) == front_label) + }) }) - .count(); + .map(|(l, _)| l.as_str()) + .collect(); assert_eq!( - preds, 1, - "pic.miss must have exactly one predecessor (pic.token.miss), or it is \ - no longer dominated by pic.token:\n{ir}" - ); - assert!( - main.contains("label %pic.miss.call"), - "every receiver-validation failure must land on the single slow \ - exit:\n{ir}" + preds, + vec![token_label], + "the front must have exactly one predecessor (pic.token), or it is no \ + longer dominated by it: {blocks:?}" ); + let all: Vec<&String> = blocks.iter().flat_map(|(_, b)| b.iter()).collect(); assert!( - !main.contains("@PERRY_IC_EPOCH"), - "the removed keys-pointer epoch global must not appear:\n{ir}" + !all.iter().any(|l| l.contains("@PERRY_IC_EPOCH")), + "the removed keys-pointer epoch global must not appear" ); assert!( - !main.contains("ptrtoint ptr @perry_ic_"), - "the small-handle sentinel select only existed because an invalid \ - receiver could reach the way compares; it must be gone:\n{ir}" + !all.iter().any(|l| l.contains("ptrtoint ptr @perry_ic_")), + "the small-handle sentinel select must be gone" ); - // The receiver predicates, exactly once each. `icmp eq i32 %` is two: the - // ShapeId identity compare on the hit path and the spill compare in - // `pic.token.miss` that replaced the hit path's overflow-bit test. There - // is no GC-kind compare at all any more (#10828), so a single `icmp eq - // i8` would mean the header load has crept back somewhere. + // The receiver predicates, exactly once each: the ShapeId identity + // compare is the only `icmp eq i32`, and there is no GC-kind compare at + // all (#10828). for (needle, what, expect) in [ ("icmp eq i8 ", "the GC_TYPE_OBJECT compare", 0), - ("icmp eq i32 %", "the ShapeId identity compare", 2), + ("icmp eq i32 %", "the ShapeId identity compare", 1), ] { - let n = main.matches(needle).count(); + let n = all.iter().filter(|l| l.contains(needle)).count(); assert_eq!( n, expect, "{what} appears {n} times, expected {expect} — a receiver \ - predicate is being re-derived or has crept back:\n{ir}" + predicate is being re-derived or has crept back: {blocks:?}" ); } - // The ONE re-derivation that is deliberate: `pic.token.miss` re-reads the - // ShapeId word through an atomic load rather than reusing the hot load's - // value, so that the hot load has a single use and isel folds it into - // the compare (`cmp %ecx, 4(%rdi)`). A plain second load would be merged - // back into the first by GVN and the hot word would be live into the - // cold blocks again. - let token_miss = main - .find("\npic.token.miss") - .unwrap_or_else(|| panic!("expected a pic.token.miss block:\n{ir}")); - let token_miss_body = &main[token_miss - ..main[token_miss + 1..] - .find("\npic.") - .map(|o| o + token_miss + 1) - .unwrap_or(main.len())]; - assert!( - token_miss_body.contains("load atomic i32"), - "pic.token.miss must re-read the ShapeId word atomically so the hot \ - load stays single-use:\n{token_miss_body}" + let shape_word = token + .iter() + .find(|l| l.contains(" = load i32, ")) + .and_then(|l| l.split_once(" = ")) + .map(|(r, _)| r.to_string()) + .unwrap_or_else(|| panic!("the ShapeId word load: {token:?}")); + let uses = all + .iter() + .filter(|l| { + l.split(|c: char| c == ',' || c == ' ' || c == '(' || c == ')') + .any(|t| t == shape_word) + }) + .count(); + assert_eq!( + uses, 2, + "the hot ShapeId load must have exactly one use (the compare), so it \ + folds into it and stays dead on the miss edge: {blocks:?}" ); } -/// S5: a matched SPILL entry is served inline. `pic.token.miss` branches to -/// `pic.spill.hit` (not to the slow exit) on the un-flipped compare, and that -/// block is exactly the three dependent loads the ShapeId licenses — -/// `ObjectHeader.meta`, `ObjectMeta.spill`, the element at the word's index — -/// with no call, no compare and no hole test, straight to the merge. +/// S5: a matched SPILL entry is served without reaching the collecting call. +/// +/// First-read D3: the miss front recognises it — the compact word holding the +/// receiver's ShapeId flipped by `PACKED_SPILL_FLIP` — and answers with the +/// three dependent loads the ShapeId licenses (`ObjectHeader.meta`, +/// `ObjectMeta.spill`, the element at the word's index); its behaviour is +/// `read_confirm::tests::the_front_serves_a_spill_entry_only_for_a_real_shape_id`. +/// The CODEGEN half: the site hands the front the compact word (the spill +/// index and flipped id live there), the front's SERVED edge lands on the +/// merge without a second call, and no spill arithmetic is expanded inline. #[test] -fn a_spill_entry_is_served_inline_by_three_loads() { +fn a_spill_entry_is_served_by_the_leaf_front_before_the_slow_call() { + use crate::expr::property_get::generic_dispatch::PACKED_SPILL_FLIP; let ir = emit(false, None); - let main_start = ir - .find("define i32 @main()") - .expect("entry module should define main"); - let main_rest = &ir[main_start..]; - let main = &main_rest[..main_rest.find("\n}\n").expect("main closes")]; - // A block: its label line (by prefix, labels carry a numeric suffix) and - // every indented line after it. - let block = |prefix: &str| -> String { - let mut lines = main - .lines() - .skip_while(|l| !(l.starts_with(prefix) && l.ends_with(':'))); - let label = lines - .next() - .unwrap_or_else(|| panic!("expected a {prefix} block:\n{main}")); - let body: Vec<&str> = lines.take_while(|l| l.starts_with(' ')).collect(); - format!("{label}\n{}", body.join("\n")) - }; - let token_miss = block("pic.token.miss"); - let spill_label = main - .lines() - .filter(|l| !l.starts_with(' ') && l.ends_with(':')) - .map(|l| l.trim_end_matches(':')) - .find(|l| l.starts_with("pic.spill.hit")) - .unwrap_or_else(|| panic!("expected a pic.spill.hit block:\n{main}")) - .to_string(); + let blocks = tower_blocks(&ir); + let (front_label, front) = tower_block(&blocks, "pic.miss.front"); + let call = front + .iter() + .find(|l| l.contains("@js_object_get_field_ic_front(")) + .unwrap_or_else(|| panic!("the front call: {front:?}")); assert!( - token_miss.contains(&format!("label %{spill_label}")), - "the spill compare in pic.token.miss must branch to {spill_label}, not \ - to the slow exit:\n{token_miss}" + call.ends_with("_packed_get)"), + "the front must receive the compact word it decodes a spill entry \ + from: {call}" ); + let answer = call.split_once(" = ").map(|(r, _)| r).unwrap(); + let (_, merge) = tower_block(&blocks, "pget.recv_merge"); + let phi = merge.iter().find(|l| l.contains(" = phi double ")).unwrap(); assert!( - !token_miss.contains("label %pic.miss.call"), - "a matched spill entry must no longer call out:\n{token_miss}" + phi.contains(&format!("[ {answer}, %{front_label} ]")), + "the merge must take the front's answer straight from its block: {phi}" ); - let hit = block(&spill_label); - let loads: Vec<&str> = hit.lines().filter(|l| l.contains(" = load ")).collect(); - assert_eq!( - loads.len(), - 3, - "the spill hit is exactly meta, spill and the value:\n{hit}" - ); - assert!(loads[0].contains("load i64") && loads[1].contains("load i64")); - assert!(loads[2].contains("load double"), "{hit}"); - for forbidden in ["call ", "icmp", "select", "atomic"] { - assert!( - !hit.contains(forbidden), - "the ShapeId match is the whole proof: no `{forbidden}` on the spill \ - hit:\n{hit}" - ); + for (label, body) in &blocks { + for gone in [ + PACKED_SPILL_FLIP.to_string(), + "pic.spill".to_string(), + "xor i32 ".to_string(), + ] { + assert!( + !label.starts_with(gone.as_str()) && !body.iter().any(|l| l.contains(&gone)), + "no spill recognition may be expanded at the site, found \ + `{gone}` in {label}: {body:?}" + ); + } } - let meta = crate::target_layout::object_meta_slot_offset_bytes("x86_64-unknown-linux-gnu"); - let spill = crate::target_layout::OBJECT_META_SPILL_OFFSET_BYTES; - let elems = crate::target_layout::ARRAY_HEADER_SIZE_BYTES; - assert_eq!((meta, spill, elems), (8, 32, 8)); - assert!( - hit.contains(&format!("i64 {spill}")) && hit.contains(&format!("i64 {elems}")), - "the loads use the paired layout constants:\n{hit}" - ); - assert!( - hit.contains("lshr i64") && hit.contains(", 32"), - "the spill index is the compact word's high half:\n{hit}" - ); - assert!(hit.contains("br label %pget.recv_merge"), "{hit}"); } /// #8067: an exact ShapeId match proves the cached slot's descriptor facts, so @@ -711,7 +743,7 @@ fn cached_slot_bound_comes_from_the_shape_descriptor_match() { let floor = crate::target_layout::INLINE_SLOT_FLOOR_LIT; let ir = emit(false, None); assert!( - ir.contains("4611686018427387904") && ir.contains("@perry_ic_"), + ir.contains("_packed_get") && ir.contains("@perry_ic_"), "test premise: the emitted read uses a ShapeId PIC:\n{ir}" ); assert!( @@ -722,38 +754,32 @@ fn cached_slot_bound_comes_from_the_shape_descriptor_match() { ); } -/// #7907: the way `(token, slot)` reduction is a balanced tree, so the slot -/// select chain is `log2(PIC_WAYS)` deep instead of `PIC_WAYS` deep. Its last -/// node feeds the bounds compare that gates the branch out of `pic.ways`, so -/// the chain depth is directly on the critical path. +/// #7907: the way `(token, slot)` reduction must not sit on the critical path +/// of a way hit. It used to be a balanced select tree expanded per site, whose +/// last node fed the bounds compare gating the branch out of `pic.ways`. /// -/// At most one way can hold a given token — `pic_prime_get` evicts a duplicate -/// before writing one, and a zero token is excluded by `token_nonnull` — so -/// reassociating is value-preserving. +/// First-read D3: the ways are compared in the miss front, which returns on +/// the first matching way (at most one way holds a given token — +/// `pic_prime_get` evicts a duplicate before writing one, and an empty way's 0 +/// cannot match), so there is no reduction left at all. Pin that the site +/// expands none: no `pic.ways` block and no `select` anywhere in the tower. #[test] -fn way_slot_reduction_is_a_balanced_tree() { - use crate::expr::property_get::generic_dispatch::PIC_WAYS; +fn way_slot_reduction_is_not_expanded_per_site() { let ir = emit(false, None); - let ways = ir - .find("\npic.ways") - .unwrap_or_else(|| panic!("expected a pic.ways block:\n{ir}")); - // Block labels carry a numeric suffix (`pic.ways.16:`), so the search for - // the NEXT block has to start past this one's own label or it matches - // itself and slices an empty body — which reads as "the tree is missing". - let end = ir[ways + 1..] - .find("\npic.") - .map(|o| o + ways + 1) - .unwrap_or(ir.len()); - let body = &ir[ways..end]; - // A left fold emits PIC_WAYS selects whose 3rd operand is the previous - // select; the tree emits PIC_WAYS lane selects against the literal 0 plus - // PIC_WAYS-1 merges. Count the "select against 0" lanes: a fold has one. - let lanes = body.matches(", i64 0\n").count(); - assert_eq!( - lanes, PIC_WAYS, - "expected one `select … , i64 , i64 0` per way (a balanced tree); \ - a left fold produces exactly one:\n{body}" - ); + let blocks = tower_blocks(&ir); + for (label, body) in &blocks { + assert!( + !label.starts_with("pic.way"), + "no way block may be expanded per site: {label}" + ); + if label.starts_with("pic.") || label.starts_with("pget.") { + assert!( + !body.iter().any(|l| l.contains(" = select ")), + "no way reduction may be expanded per site, found a select in \ + {label}: {body:?}" + ); + } + } } /// #7189 — `B.ns` where the imported module says `export * as ns from "./m.ts"`. @@ -1146,22 +1172,40 @@ fn generic_property_get_slot_load_is_reached_only_through_every_guard() { "the hit block must end in the slot load and an unconditional branch \ to the merge:\n{hit_body}" ); - let way_body = blocks + // A way hit is answered in the miss front now (first-read D3), from the + // same proof: a way holds an aged MRU pair compared against the same + // ShapeId word. The front's answer is branched on only to tell a served + // value from its `TAG_HOLE` decline — the served edge is the TRUE edge + // and lands on the merge — never to re-test a served slot. + assert!( + !blocks.iter().any(|(l, _)| l.starts_with("pic.way")), + "no way block may be expanded per site:\n{func}" + ); + let front_body = blocks .iter() - .find(|(l, _)| l.starts_with("pic.way.load")) + .find(|(l, _)| l.starts_with("pic.miss.front")) .map(|(_, body)| body.join("\n")) - .expect("the way load block"); + .expect("the miss front block"); + let term = front_body + .lines() + .rev() + .find(|l| l.trim_start().starts_with("br ")) + .unwrap(); assert!( - !way_body.contains(crate::nanbox::TAG_HOLE_I64), - "the way path must not compare the loaded slot against TAG_HOLE — a \ - way holds an aged MRU pair and its token is the same ShapeId word, \ - so a way hit carries the same liveness proof as an MRU hit:\n\ - {way_body}" + front_body.contains(&format!(", {}", crate::nanbox::TAG_HOLE_I64)) + && term.contains("br i1 ") + && term.contains(", label %pget.recv_merge") + && term.contains("label %pic.miss.call"), + "the front's decline test must send the served value to the merge on \ + the TRUE edge and the decline to the slow call:\n{front_body}" + ); + let (served_at, declined_at) = ( + term.find("label %pget.recv_merge").unwrap(), + term.find("label %pic.miss.call").unwrap(), ); assert!( - way_body.contains("load double") && way_body.contains("br label %"), - "the way load block must end in the slot load and an unconditional \ - branch to the merge:\n{way_body}" + served_at < declined_at, + "served must be the TRUE edge: {term}" ); } @@ -1438,6 +1482,56 @@ fn no_gc_header_load_on_any_target() { } } +/// First-read D3: the miss front's directory operand (`PERRY_AGENT_PTRS` +/// slot 0) is read WITHOUT a call wherever the target has a call-free +/// thread-pointer path: an initial-exec load on an ELF executable, the TEB's +/// TLS array on Windows x86-64 (`agent_ptr::AgentPtrAccess::WindowsTeb`), the +/// pthread TSD on Apple aarch64. x86-64 Darwin keeps the `gc-leaf` accessor: +/// Mach-O has no call-free thread-local model there (`agent_ptr.rs`). +#[test] +fn the_front_reads_its_directory_without_a_call_where_the_target_allows() { + for (target, inline_form) in [ + ( + "x86_64-unknown-linux-gnu", + Some("getelementptr i8, ptr @PERRY_AGENT_PTRS, i64 0"), + ), + ( + "x86_64-pc-windows-msvc", + Some("load ptr, ptr addrspace(256) inttoptr (i64 88 to ptr addrspace(256))"), + ), + ("aarch64-apple-darwin", Some("mrs $0, tpidrro_el0")), + ("x86_64-apple-darwin", None), + ] { + let mut opts = ir_opts(false, None); + opts.target = Some(target.to_string()); + let ir = + String::from_utf8(compile_module(&module_with_nullish_read(), opts).unwrap()).unwrap(); + let func = ir + .split("\ndefine ") + .find(|f| f.contains("\npic.miss.front")) + .unwrap_or_else(|| panic!("{target}: no function contains the front:\n{ir}")); + let dir_call = func.contains("call ptr @perry_shape_dir_cell("); + match inline_form { + Some(form) => { + assert!( + func.contains(form) && !dir_call, + "{target}: the directory must be read inline (`{form}`), \ + with no accessor call:\n{func}" + ); + } + None => assert!(dir_call, "{target}: the accessor call:\n{func}"), + } + if target.contains("windows") { + for global in ["@_tls_index", "@PERRY_AGENT_PTRS_SECREL"] { + assert!( + func.contains(&format!("load i32, ptr {global}")), + "{target}: the TEB form reads {global}:\n{func}" + ); + } + } + } +} + #[test] fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { use crate::expr::property_get::generic_dispatch::PACKED_GET_EMPTY; @@ -1457,7 +1551,7 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { // `icmp ne i64 %packed, 0` beside it any more: the sentinel above makes // the ShapeId compare prove the site is primed as well. Named by the // packed word's register: a blanket "no `icmp ne i64`" would now also - // forbid the inherited-read hook's decline compare on the exit edge, + // forbid the miss front's `TAG_HOLE` decline compare, // which is a different question about a different value. let packed = ir .lines() @@ -1472,15 +1566,16 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { "the compact word must not be tested against zero:\n{ir}" ); assert!( - ir.contains("pic.token.miss"), - "a full-cache dereference must still guard a null site: {ir}" + !ir.contains("load ptr, ptr @perry_ic_"), + "the full cache stays lazy: the site never dereferences its slot (the \ + front and the slow entry null-test it): {ir}" ); } -/// T1: the whole point — per untyped `obj.prop` the emitted tower is TWO calls -/// and a handful of blocks, with the inline hit and the polymorphic ways kept. +/// T1: per untyped `obj.prop` the emitted tower is a bounded handful of blocks +/// and calls, with only the inline hit kept inline. /// -/// This is a ratchet, so it is an EXACT count in both dimensions. The tower it +/// This is a ratchet, so it is an EXACT count in both dimensions. The tower T1 /// replaced expanded 33 tower blocks and SIX runtime call sites per site /// (`js_object_get_field_by_name_f64` twice, the feedback-wrapped class-ref /// helper, `js_throw_type_error_property_access`, @@ -1490,58 +1585,20 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { /// sites; a single arm creeping back inline is a regression measured in /// megabytes of `.text`, and nothing else in the suite would report it. /// -/// Two and not one: a single shared exit let SimplifyCFG fold the receiver-tag -/// test and the small-handle test into one flat predicate, costing +4.00 -/// instructions on every HIT (measured, before those two tests became the one -/// fused compare). The separate non-pointer callee still keeps the `.length` -/// tower's chain branchy, so the count below is 2 — and a change that makes it -/// 1 is a hit-path regression, not a size win. -/// A SPILL-located key must still be RECOGNISED — just not on the hit path. +/// Two collecting exits and not one: a single shared exit let SimplifyCFG fold +/// the receiver-tag test and the small-handle test into one flat predicate, +/// costing +4.00 instructions on every HIT (measured, before those two tests +/// became the one fused compare). The separate non-pointer callee still keeps +/// the `.length` tower's chain branchy — a change that makes it one is a +/// hit-path regression, not a size win. /// -/// Taking the overflow-bit test off the hit path is only sound if the entry it -/// used to catch is caught somewhere else. `pic.token.miss` un-flips -/// `PACKED_SPILL_FLIP` and branches straight to `pic.spill.hit` (S5), skipping -/// the full cache's resolution and the ways (neither can hold an encoded -/// slot). Without this test, deleting the spill compare would leave every -/// spill read correct-but-slow — it would walk the ways, miss, call out, and -/// re-scan the keys array on every read, which is invisible in program -/// output. -#[test] -fn a_spill_entry_is_recognised_in_the_token_miss_block_and_nowhere_else() { - use crate::expr::property_get::generic_dispatch::PACKED_SPILL_FLIP; - let ir = emit(false, None); - let func = ir - .split("\ndefine ") - .find(|f| f.contains("\npic.token.miss")) - .unwrap_or_else(|| panic!("no function contains the generic tower:\n{ir}")); - - // Split the function into blocks and find `pic.token.miss`'s body. - let mut body: Vec<&str> = Vec::new(); - let mut inside = false; - for line in func.lines() { - if !line.starts_with(' ') && line.ends_with(':') { - inside = line.trim_end_matches(':').starts_with("pic.token.miss"); - continue; - } - if inside { - body.push(line); - } - } - let body = body.join("\n"); - assert!( - body.contains("xor i32 ") && body.contains(&PACKED_SPILL_FLIP.to_string()), - "`pic.token.miss` must un-flip PACKED_SPILL_FLIP to recognise a spill \ - entry:\n{body}" - ); - assert!( - body.contains("label %pic.spill.hit"), - "a recognised spill entry must branch straight to the inline spill \ - hit, not walk the ways or call out:\n{body}" - ); -} - +/// First-read D3: the spill entry, the ways, the inherited-read cache and the +/// latched confirm are no longer expanded per site. The ShapeId compare's +/// false edge makes ONE plain call to the GC-leaf front +/// (`js_object_get_field_ic_front`), whose `TAG_HOLE` decline continues to the +/// collecting slow call. #[test] -fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { +fn the_generic_tower_is_one_leaf_call_two_exits_and_a_bounded_number_of_blocks() { let ir = emit(false, None); let func = ir .split("\ndefine ") @@ -1550,8 +1607,8 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { // Every call/invoke in the whole function, by callee. Feedback records are // compile-time gated and absent from this build; anything else must be the - // one exit (the fixture's module init contributes its own calls, so match - // on the property-GET family rather than on a total). + // front or one of the two exits (the fixture's module init contributes its + // own calls, so match on the property-GET family rather than on a total). let pget_calls: Vec<&str> = func .lines() .filter(|l| l.contains(" call ") || l.contains(" invoke ")) @@ -1560,6 +1617,7 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { .filter(|c| { c.starts_with("js_object_get_field") || c.starts_with("js_typed_feedback_object_get_field") + || c.starts_with("js_inherited_read_cache") || *c == "js_throw_type_error_property_access" }) .collect(); @@ -1568,10 +1626,34 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { assert_eq!( sorted, vec![ + "js_object_get_field_ic_front", "js_object_get_field_ic_nonptr", "js_object_get_field_ic_slow" ], - "the tower must expand exactly two property-GET call sites:\n{func}" + "the tower must expand the front and exactly two collecting exits:\n{func}" + ); + // The front is nounwind: a plain call, never an invoke (its GC-leaf + // classification is `gc_call_effects`' test). + let fronts: Vec<&str> = func + .lines() + .filter(|l| l.contains("@js_object_get_field_ic_front(")) + .collect(); + assert_eq!(fronts.len(), 1, "one front call per tower:\n{func}"); + assert!( + fronts[0].contains(" = call double "), + "the front is nounwind, a plain call:\n{func}" + ); + // A non-`length` site confirms from this agent's own directory: the dir + // operand is slot 0 of `PERRY_AGENT_PTRS` (one initial-exec load in this + // ELF executable), never the empty directory a `length` site passes. + assert!( + !fronts[0].contains("@PERRY_EMPTY_SHAPE_DIR"), + "only a `length` site passes the empty directory:\n{}", + fronts[0] + ); + assert!( + func.contains("getelementptr i8, ptr @PERRY_AGENT_PTRS, i64 0"), + "the dir operand is PERRY_AGENT_PTRS slot 0:\n{func}" ); let blocks: Vec<&str> = func @@ -1588,41 +1670,25 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { // its split (cold): a POINTER-tagged small handle fails the fused // receiver test too and goes on to the object exit from here "pget.recv_nonptr", - // `pic.recv_hdr` is GONE: it existed to load the GC header word, and - // the ShapeId compare in `pic.token` now proves the kind (#10828) and - // the descriptor state (#10824) that word was loaded for. + // `pic.recv_hdr` is GONE: the ShapeId compare in `pic.token` proves + // the kind (#10828) and the descriptor state (#10824). "pic.token", - "pic.token.miss", - // `pic.token.miss` recognises a SPILL-located key by un-flipping - // PACKED_SPILL_FLIP and branches to `pic.spill.hit` (S5: three - // dependent loads, no call); everything else continues here to the - // full cache and the ways. - "pic.spill.hit", - // `pic.hit.inline` is GONE: with spill entries - // refused by the ShapeId compare itself, the hit block has nothing to - // decide between and the load sits directly in `pic.hit`. - "pic.token.ways", - // The hit block ends in the slot load and a branch to the merge: - // `pic.hit.deleted` is GONE with the `TAG_HOLE` compare (#10826 made - // delete a shape transition, so a ShapeId hit proves the slot live); - // `pic.hit.live` exists only when typed feedback has something to - // record on the live edge. + // The hit block is the slot load and a branch to the merge: no + // overflow-bit test (a spill entry is refused by the compare itself) + // and no `TAG_HOLE` compare (#10826 made delete a shape transition). "pic.hit", - // the polymorphic ways, deliberately still inline (#7753) - "pic.miss", - "pic.ways", - // `pic.way.live` is GONE with the way path's `TAG_HOLE` compare: the - // load block has nothing left to decide and branches to the merge. - "pic.way.load", - // the inherited-read hook, on the never-primed edge out of - // `pic.token.ways` and nowhere else (`js_inherited_read_cache_hit_f64`, - // a leaf); a decline continues to the one exit - "pic.miss.inherited", + // First-read D3: the compare's false edge. One GC-leaf call answers + // a way, a spill entry or a latched site's confirmed guess; its + // decline continues to the one exit. `pic.token.miss`, + // `pic.spill.hit`, `pic.token.ways`, `pic.miss`, `pic.ways`, + // `pic.way.load`, `pic.not_ways`, `pic.mega` and `pic.miss.inherited` + // are GONE into it and into the slow entry. + "pic.miss.front", // the one exit, and the join "pic.miss.call", "pget.recv_merge", ]; - // Labels carry a numeric suffix (`pic.ways.16`); strip it for comparison. + // Labels carry a numeric suffix (`pic.token.6`); strip it for comparison. let mut normalized: Vec = blocks .iter() .map(|b| { @@ -1642,198 +1708,123 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { ); } +/// A SPILL-located key must still be RECOGNISED — just not on the hit path. +/// +/// Taking the overflow-bit test off the hit path is only sound if the entry it +/// used to catch is caught somewhere else. Without that, every spill read +/// would be correct-but-slow — it would miss, call out, and re-scan the keys +/// array on every read, which is invisible in program output. +/// +/// First-read D3: the miss front recognises it (un-flips `PACKED_SPILL_FLIP`, +/// checks the result is a real ShapeId, and loads the value — +/// `read_confirm::tests::the_front_serves_a_spill_entry_only_for_a_real_shape_id`), +/// so the site recognises it nowhere: the flip constant is not emitted, and the +/// only block the compare's false edge reaches is the front. +#[test] +fn a_spill_entry_is_recognised_by_the_front_and_nowhere_at_the_site() { + use crate::expr::property_get::generic_dispatch::PACKED_SPILL_FLIP; + let ir = emit(false, None); + let blocks = tower_blocks(&ir); + let flip = PACKED_SPILL_FLIP.to_string(); + let flip_i32 = (PACKED_SPILL_FLIP as i32).to_string(); + for (label, body) in &blocks { + assert!( + !body + .iter() + .any(|l| l.contains(&flip) || l.contains(&flip_i32)), + "PACKED_SPILL_FLIP must not be emitted at the site (`{label}`): {body:?}" + ); + } + let (_, token) = tower_block(&blocks, "pic.token"); + let (_, _, on_miss) = tower_cond_br(token); + assert!( + on_miss.starts_with("pic.miss.front"), + "the compare's false edge must reach the front, which recognises a \ + spill entry: {token:?}" + ); +} + /// The inherited-read cache (#10834/#10842) is asked on the NEVER-PRIMED edge /// and nowhere else. A read whose key lives on the prototype chain is never an /// own slot on the receiver's shape, so a site that only reads such a key never -/// resolves its per-site cache, and every read of it reaches `pic.token.ways` -/// with `present` false. That edge — which used to go straight to the exit — -/// now asks the cache before calling out. The first placement asked on EVERY -/// path into the exit and charged each own-key miss a declining probe (+88 on -/// a megamorphic site, +89 on a spill read, measured); this one costs every -/// other path zero instructions. +/// resolves its per-site cache. The first placement asked on EVERY path into +/// the exit and charged each own-key miss a declining probe (+88 on a +/// megamorphic site, +89 on a spill read, measured). /// -/// Five things are pinned, each of which would otherwise fail silently (the -/// program still computes the right value through the slow entry): +/// First-read D3: the probe moved into the slow entry +/// (`js_object_get_field_ic_slow`, which asks it only when the site's cache +/// slot is unresolved), behind the leaf front — so an own-key way, spill or +/// latched read never reaches it, and the site expands none of it. Pinned +/// here, each of which would otherwise fail silently (the program still +/// computes the right value through the slow entry): /// -/// 1. the hook call sits in `pic.miss.inherited` and in no other block, in -/// particular NOT on any path to the inline slot load (the CFG-walk test -/// asserts the same from the other side); -/// 2. that block is reached from `pic.token.ways` on the FALSE edge of the -/// cache-present test, and from nowhere else; -/// 3. its result is branched on with the SERVED edge as the true edge, the -/// tower's rule for every guard-passing edge, and the false edge is the -/// one exit; -/// 4. the slow entry is still called from `pic.miss.call` only, with the same -/// four operands; -/// 5. the merge phi takes the served value from `pic.miss.inherited`. +/// 1. no block of the site calls the hook — in particular none on a path to +/// the inline slot load (the CFG-walk test asserts the same from the other +/// side); +/// 2. the slow entry is called from `pic.miss.call` only, with the same four +/// operands (the never-primed test reads the cache slot); +/// 3. `pic.miss.call` is reached from the front only on its `TAG_HOLE` +/// decline, so a front-served read never pays the probe; +/// 4. the merge takes the slow entry's value from `pic.miss.call`. #[test] fn the_inherited_read_cache_is_asked_on_the_never_primed_edge_only() { let ir = emit(false, None); - let func = ir - .split("\ndefine ") - .find(|f| f.contains("\npic.miss.call")) - .unwrap_or_else(|| panic!("no function contains the generic tower:\n{ir}")); - let mut blocks: Vec<(String, Vec)> = Vec::new(); - let mut cur: Option<(String, Vec)> = None; - for line in func.lines() { - if !line.starts_with(' ') && line.ends_with(':') { - if let Some(b) = cur.take() { - blocks.push(b); - } - cur = Some((line.trim_end_matches(':').to_string(), Vec::new())); - continue; - } - if let Some((_, body)) = cur.as_mut() { - body.push(line.trim().to_string()); - } - } - if let Some(b) = cur.take() { - blocks.push(b); - } - // 1. one caller block, and it is the inherited arm. + let blocks = tower_blocks(&ir); + // 1. let holders: Vec<&str> = blocks .iter() .filter(|(_, body)| { body.iter() - .any(|l| l.contains("call double @js_inherited_read_cache_hit_f64(")) + .any(|l| l.contains("@js_inherited_read_cache_hit_f64(")) }) .map(|(l, _)| l.as_str()) .collect(); - assert_eq!( - holders.len(), - 1, - "the inherited hook must be called from exactly one block: {holders:?}\n{func}" - ); - let inh_label = holders[0]; assert!( - inh_label.starts_with("pic.miss.inherited"), - "the hook belongs on the never-primed edge, found it in `{inh_label}`:\n{func}" + holders.is_empty(), + "the inherited hook belongs to the slow entry, not the site: {holders:?}" ); - let (_, inh_body) = blocks.iter().find(|(l, _)| l == inh_label).unwrap(); - let hook_line = inh_body - .iter() - .find(|l| l.contains("@js_inherited_read_cache_hit_f64(")) - .unwrap(); - assert!( - hook_line.contains("(ptr %") && hook_line.matches(", ptr %").count() == 1, - "the hook takes the masked receiver and the interned key as two \ - pointers:\n{hook_line}" - ); - // 2. reached only from `pic.token.ways`, on the FALSE edge of `present`. - let preds: Vec<(&str, &str)> = blocks + // 2. + let slow_callers: Vec<(&str, &String)> = blocks .iter() .flat_map(|(l, body)| { body.iter() - .filter(|t| t.starts_with("br ") && t.contains(&format!("label %{inh_label}"))) - .map(move |t| (l.as_str(), t.as_str())) + .filter(|t| t.contains("@js_object_get_field_ic_slow(")) + .map(move |t| (l.as_str(), t)) }) .collect(); - assert_eq!( - preds.len(), - 1, - "exactly one edge may reach the hook: {preds:?}\n{func}" - ); - let (pred_label, pred_term) = preds[0]; - assert!( - pred_label.starts_with("pic.token.ways"), - "the hook's one predecessor must be the cache-present test: {pred_label}" - ); - let parts: Vec<&str> = pred_term - .strip_prefix("br i1 ") - .unwrap() - .split(", ") - .collect(); - assert!( - parts[1].starts_with("label %pic.miss") && !parts[1].starts_with("label %pic.miss.inh"), - "the TRUE edge of `present` must still be the way compares: {pred_term}" - ); - assert!( - parts[2].starts_with(&format!("label %{inh_label}")), - "the hook must sit on the FALSE (never-primed) edge: {pred_term}" - ); - let present_def = blocks - .iter() - .find(|(l, _)| l == pred_label) - .and_then(|(_, body)| { - body.iter() - .find(|l| l.starts_with(&format!("{} = ", parts[0]))) - }) - .unwrap_or_else(|| { - panic!( - "the branch condition {} must be defined in {pred_label}", - parts[0] - ) - }); - assert!( - present_def.contains("icmp ne ptr ") && present_def.ends_with(", null"), - "`present` is the cache slot's non-null test:\n{present_def}" - ); - // 3. polarity: `icmp ne , TAG_HOLE` is "served", served is the TRUE - // edge and lands on the merge; the false edge is the one exit. - let served = inh_body - .iter() - .find(|l| l.contains("icmp ne i64 ") && l.ends_with(crate::nanbox::TAG_HOLE_I64)) - .unwrap_or_else(|| panic!("the decline compare against TAG_HOLE:\n{func}")); - let cond = served.split_once(" = ").map(|(c, _)| c).unwrap(); - let term = inh_body - .iter() - .rev() - .find(|l| l.starts_with("br ")) - .unwrap(); - let parts: Vec<&str> = term - .strip_prefix("br i1 ") - .unwrap_or_else(|| panic!("the arm must branch on the hook's answer: {term}")) - .split(", ") - .collect(); - assert_eq!( - parts[0], cond, - "the branch must be on the served predicate: {term}" - ); - assert!( - parts[1].starts_with("label %pget.recv_merge"), - "the SERVED edge must be the true edge and land on the merge: {term}" - ); - assert!( - parts[2].starts_with("label %pic.miss.call"), - "the decline must be the false edge into the one exit: {term}" - ); - // 4. the slow entry: one caller, the exit, same operands. - let slow_callers: Vec<&str> = blocks - .iter() - .filter(|(_, body)| { - body.iter() - .any(|l| l.contains("@js_object_get_field_ic_slow(")) - }) - .map(|(l, _)| l.as_str()) - .collect(); assert_eq!(slow_callers.len(), 1, "{slow_callers:?}"); + let (call_label, slow_line) = slow_callers[0]; assert!( - slow_callers[0].starts_with("pic.miss.call"), + call_label.starts_with("pic.miss.call"), "the slow entry must be called from the one exit: {slow_callers:?}" ); - let (_, slow_body) = blocks.iter().find(|(l, _)| l == slow_callers[0]).unwrap(); - let slow_line = slow_body - .iter() - .find(|l| l.contains("@js_object_get_field_ic_slow(")) - .unwrap(); assert!( - slow_line.contains("ptr @perry_ic_") && slow_line.contains("_packed_get"), - "the slow entry must still receive the cache slot and the packed \ - word:\n{slow_line}" + slow_line.contains("(i64 %") + && slow_line.matches(", i64 %").count() == 1 + && slow_line.contains("ptr @perry_ic_") + && slow_line.contains("_packed_get"), + "the slow entry must still receive the receiver, the key, the cache \ + slot and the packed word:\n{slow_line}" ); - // 5. the merge takes the served value from the inherited arm. - let (_, merge_body) = blocks - .iter() - .find(|(l, _)| l.starts_with("pget.recv_merge")) - .unwrap(); - let phi = merge_body - .iter() - .find(|l| l.contains(" = phi double ")) - .unwrap(); - let served_value = hook_line.split_once(" = ").map(|(v, _)| v).unwrap(); + // 3. + let (_, front) = tower_block(&blocks, "pic.miss.front"); + let (cond, served, declined) = tower_cond_br(front); assert!( - phi.contains(&format!("[ {served_value}, %{inh_label} ]")), - "the merge must take the hook's value from `{inh_label}`:\n{phi}" + front + .iter() + .any(|l| l.starts_with(&format!("{cond} = icmp ne i64 ")) + && l.ends_with(crate::nanbox::TAG_HOLE_I64)), + "the front's branch must be on its TAG_HOLE decline: {front:?}" + ); + assert!(served.starts_with("pget.recv_merge"), "{front:?}"); + assert_eq!(declined, call_label, "{front:?}"); + // 4. + let (_, merge) = tower_block(&blocks, "pget.recv_merge"); + let phi = merge.iter().find(|l| l.contains(" = phi double ")).unwrap(); + let value = slow_line.split_once(" = ").map(|(v, _)| v).unwrap(); + assert!( + phi.contains(&format!("[ {value}, %{call_label} ]")), + "the merge must take the slow entry's value from `{call_label}`:\n{phi}" ); } diff --git a/crates/perry-codegen/src/expr/receiver_range.rs b/crates/perry-codegen/src/expr/receiver_range.rs index fafcd16a65..fdd820bbb5 100644 --- a/crates/perry-codegen/src/expr/receiver_range.rs +++ b/crates/perry-codegen/src/expr/receiver_range.rs @@ -88,7 +88,10 @@ pub(crate) enum Route { Generic = 0, /// ...and was served by the compact MRU word. GenericMruHit = 1, - /// ...and was served by one of the polymorphic ways. + /// ...and was served by one of the polymorphic ways. No longer emitted: + /// the ways are read by the runtime's miss entry (first-read D3). The + /// number stays reserved: it indexes `RECV_ROUTE_NAMES`. + #[allow(dead_code)] GenericWayHit = 2, /// A read region's receiver test passed (R1 part 1). Region = 3, @@ -101,7 +104,9 @@ pub(crate) enum Route { /// The cached field-index early return's receiver test passed. CachedFieldIndex = 7, /// A generic read served from the receiver's SPILL buffer by the compact - /// word's flipped entry (S5, `pic.spill.hit`). + /// word's flipped entry (S5). No longer emitted: the runtime's miss entry + /// serves it (first-read D3). The number stays reserved, as above. + #[allow(dead_code)] GenericSpillHit = 8, } diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index a73f514850..32cb1f97f7 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -762,6 +762,25 @@ mod tests { } } + /// First-read D3: a generic read's miss front must be a proven GC leaf + /// (`read_confirm.rs` says why). If the generated table ever classifies + /// it otherwise, the front grew a collecting path — a design error in the + /// front, not a table update. Its decline continuation still collects. + #[test] + fn the_generic_read_miss_front_is_leaf_and_its_continuation_collects() { + assert_eq!( + runtime_class("js_object_get_field_ic_front"), + RuntimeClass::Leaf + ); + assert!(external_callee_cannot_collect( + "js_object_get_field_ic_front" + )); + assert!(external_callee_cannot_collect("perry_shape_dir_cell")); + assert!(!external_callee_cannot_collect( + "js_object_get_field_ic_slow" + )); + } + #[test] fn register_global_root_tracks_the_barrier_it_wraps() { assert_eq!( diff --git a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv index 6625c52a2a..f441dd70eb 100644 --- a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv @@ -2123,6 +2123,7 @@ js_object_get_field_f64 Leaf js_object_get_field_ic Reenters js_object_get_field_ic_fast Leaf js_object_get_field_ic_fast_miss Reenters +js_object_get_field_ic_front Leaf js_object_get_field_ic_miss Reenters js_object_get_field_ic_miss_packed Reenters js_object_get_field_ic_nonptr Reenters @@ -3958,6 +3959,7 @@ perry_resolve_static_plugin Reenters perry_runtime_widget_init Leaf perry_safe_area_insets_make Reenters perry_set_wake_callback Leaf +perry_shape_dir_cell Leaf perry_store_census_arm Leaf perry_string_header_abi_revision Leaf perry_stub_warn_ffi Leaf diff --git a/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv b/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv index 405bf6af51..f5029b857e 100644 --- a/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv +++ b/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv @@ -2123,6 +2123,7 @@ js_object_get_field_f64 Leaf js_object_get_field_ic Reenters js_object_get_field_ic_fast Leaf js_object_get_field_ic_fast_miss Reenters +js_object_get_field_ic_front Leaf js_object_get_field_ic_miss Reenters js_object_get_field_ic_miss_packed Reenters js_object_get_field_ic_nonptr Reenters @@ -3959,6 +3960,7 @@ perry_resolve_static_plugin Leaf perry_runtime_widget_init Reenters perry_safe_area_insets_make Reenters perry_set_wake_callback Leaf +perry_shape_dir_cell Leaf perry_store_census_arm Leaf perry_string_header_abi_revision Leaf perry_stub_warn_ffi Leaf diff --git a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv index 8334ff53d5..1c3a08b73a 100644 --- a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv @@ -2123,6 +2123,7 @@ js_object_get_field_f64 Leaf js_object_get_field_ic Reenters js_object_get_field_ic_fast Leaf js_object_get_field_ic_fast_miss Reenters +js_object_get_field_ic_front Leaf js_object_get_field_ic_miss Reenters js_object_get_field_ic_miss_packed Reenters js_object_get_field_ic_nonptr Reenters @@ -3953,6 +3954,7 @@ perry_resolve_static_plugin Leaf perry_runtime_widget_init Leaf perry_safe_area_insets_make Reenters perry_set_wake_callback Leaf +perry_shape_dir_cell Leaf perry_store_census_arm Leaf perry_string_header_abi_revision Leaf perry_stub_warn_ffi Leaf diff --git a/crates/perry-codegen/src/module/linkage.rs b/crates/perry-codegen/src/module/linkage.rs index 118c04f4ed..dad9da83ee 100644 --- a/crates/perry-codegen/src/module/linkage.rs +++ b/crates/perry-codegen/src/module/linkage.rs @@ -326,6 +326,14 @@ pub(crate) fn helper_decl_attrs(name: &str) -> &'static str { // operands are i64 handles) orders it against every GC-capable call. // js_string_compare_value is NOT eligible: number coercion allocates. "js_string_compare" => " #3", + // First-read D3 (`object/field_get_set/ic_miss/read_confirm.rs`): a + // generic read site's miss front. Loads, compares and at most one + // store (D3b re-aims the site's compact word); no allocation, lock, + // throw, call or unbounded loop (the ways and the second-chance scan + // are bounded). NOT readonly: it writes the site word. Also a proven + // `Leaf` in the generated call-effects table, so the call is + // `"gc-leaf-function"`. + "js_object_get_field_ic_front" => " #4", // NOUNWIND+WILLRETURN only (#4, repsel Phase 4a.0) — each verified // (`typed_feedback.rs` / `array/header.rs`): no `js_throw` (longjmp) // anywhere in the body, every loop bounded by the 16M length/capacity diff --git a/crates/perry-codegen/src/root_reload.rs b/crates/perry-codegen/src/root_reload.rs index b106fb3a42..bced92be6b 100644 --- a/crates/perry-codegen/src/root_reload.rs +++ b/crates/perry-codegen/src/root_reload.rs @@ -205,6 +205,8 @@ const NON_COLLECTING: &[&str] = &[ "js_inherited_read_cache_hit_f64", // S2 GC-leaf IC hits; proven `Leaf` by the generated call-effects table. "js_object_get_field_ic_fast", + // First-read D3: a generic read's miss front, proven `Leaf` likewise. + "js_object_get_field_ic_front", "js_class_field_get_ic_fast", "js_class_field_set_ic_fast", "js_put_value_set_packed_fast", diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index 1d0e82583a..d1b3eb8dc3 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -85,6 +85,10 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // by the inline lookup in `expr::hot_tls` (Apple aarch64 targets only; // the declaration is unreferenced, and therefore inert, elsewhere). module.add_external_global("PERRY_HOT_TSD_KEY", I64); + // shapes_store — the never-written empty shape directory, a generic read + // site's front operand for `length` and where the agent's own directory + // is not readable inline (`property_get/generic_dispatch.rs`). + module.add_external_global("PERRY_EMPTY_SHAPE_DIR", I64); // #5525 follow-up: the process-global typed-array kind cache + the // "any exotic views live" guard, exported from perry-runtime so the codegen // can emit a guarded *inline* typed-array element load at the access site @@ -433,6 +437,18 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // Heap-pointer receiver: (masked obj_handle, key_handle, per-site IC cache // SLOT, per-site packed MRU word) -> field value. module.declare_function("js_object_get_field_ic_slow", DOUBLE, &[I64, I64, PTR, PTR]); + // First-read D3: a generic read site's ShapeId miss asks this GC leaf + // first (the agent's shape-directory mirror or null, receiver payload, + // the key as its pool global holds it, the site's cache slot and compact + // word); `TAG_HOLE` = declined, and the site calls the slow entry. + // `perry_shape_dir_cell` is the directory's accessor where emitted code + // cannot read the agent's pointer block inline. Both GC leaves. + module.declare_function( + "js_object_get_field_ic_front", + DOUBLE, + &[PTR, I64, I64, PTR, PTR], + ); + module.declare_function("perry_shape_dir_cell", PTR, &[]); // Object rest destructuring: copy all properties from src except excluded keys. // Takes a src object ptr and an array of NaN-boxed strings (the excluded keys), // returns a new object pointer. @@ -628,6 +644,11 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { &format!("[{} x ptr]", crate::expr::agent_ptr::AGENT_PTR_SLOTS), "initialexec", ); + // Windows x86-64 reaches the same block through the TEB (`agent_ptr.rs`, + // `WindowsTeb`): the image TLS index and the block offset in that image's + // TLS block. Declarations only; no other target references them. + module.add_external_global(crate::expr::agent_ptr::TLS_INDEX_SYMBOL, I32); + module.add_external_global(crate::expr::agent_ptr::AGENT_PTRS_SECREL_SYMBOL, I32); module.declare_function("perry_implicit_this_cell", PTR, &[]); module.declare_function( "js_method_site_miss", diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index c3e2693778..a7c6aa78df 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -2588,6 +2588,7 @@ js_object_get_field_f64 f64 ptr,i32u js_object_get_field_ic f64 i64,ptr,i64,ptr js_object_get_field_ic_fast f64 i64,ptr,i64,ptr js_object_get_field_ic_fast_miss f64 i64,ptr,i64,ptr +js_object_get_field_ic_front f64 ptr,i64,i64,ptr,ptr js_object_get_field_ic_miss f64 ptr,ptr,ptr js_object_get_field_ic_miss_packed f64 ptr,ptr,ptr,ptr js_object_get_field_ic_nonptr f64 i64,ptr,i64 @@ -4686,6 +4687,7 @@ perry_resolve_static_plugin f64 ptr perry_runtime_widget_init void perry_safe_area_insets_make f64 f64,f64,f64,f64 perry_set_wake_callback void ptr,ptr +perry_shape_dir_cell ptr perry_store_census_arm void perry_string_header_abi_revision i32u perry_stub_warn_ffi void ptr,ptr,ptr diff --git a/crates/perry-runtime/src/agent_ptrs.rs b/crates/perry-runtime/src/agent_ptrs.rs index d994e55873..7ac6e2f7f2 100644 --- a/crates/perry-runtime/src/agent_ptrs.rs +++ b/crates/perry-runtime/src/agent_ptrs.rs @@ -14,7 +14,10 @@ //! `agent_ptrs` field holds this block's address — Mach-O has no //! initial-exec model, so a direct thread-local access would be a TLV thunk //! call; -//! * every other target, and any image that can be `dlopen`ed (a dylib or +//! * Windows x86-64: the native TLS sequence spelled out — `gs:[0x58]` +//! indexed by the image's `_tls_index`, plus [`PERRY_AGENT_PTRS`]'s offset +//! in the image's TLS block, published below as `PERRY_AGENT_PTRS_SECREL`; +//! * every other target, and any ELF image that can be `dlopen`ed (a dylib or //! staticlib output, where initial-exec TLS may not fit the static TLS //! block): the runtime accessor call. //! @@ -24,8 +27,10 @@ use std::cell::Cell; -/// Slot 1: this agent's implicit-`this` cell. (Slot 0 is reserved.) +/// Slot 1: this agent's implicit-`this` cell. pub use crate::codegen_abi::AGENT_PTR_IMPLICIT_THIS; +/// Slot 0: the address of this agent's ordinary shape-directory mirror. +pub use crate::codegen_abi::AGENT_PTR_SHAPE_DIR; /// Slots in the block. **Must equal `AGENT_PTR_SLOTS` in /// `perry-codegen/src/expr/agent_ptr.rs`** (the emitted global's type). pub use crate::codegen_abi::AGENT_PTR_SLOTS; @@ -42,11 +47,38 @@ pub struct AgentPtrs([Cell<*const u8>; AGENT_PTR_SLOTS]); /// rustc emits a thread-local shim for the static under the same symbol name /// and a `#[no_mangle]` static fails to build ("symbol `PERRY_AGENT_PTRS` is /// already defined"). Only ELF executables name it -/// (`perry-codegen/src/expr/agent_ptr.rs`); Windows takes the accessor call. +/// (`perry-codegen/src/expr/agent_ptr.rs`); Windows x86-64 reaches it through +/// `PERRY_AGENT_PTRS_SECREL` instead (below). #[cfg_attr(not(windows), no_mangle)] #[thread_local] -pub static PERRY_AGENT_PTRS: AgentPtrs = - AgentPtrs([const { Cell::new(std::ptr::null()) }; AGENT_PTR_SLOTS]); +pub static PERRY_AGENT_PTRS: AgentPtrs = AgentPtrs({ + let mut slots = [const { Cell::new(std::ptr::null()) }; AGENT_PTR_SLOTS]; + // The shape-directory slot is never null: until this agent publishes its + // own mirror it names the shared empty directory, so the generic-read + // miss front reads it without a null test. + slots[AGENT_PTR_SHAPE_DIR] = + Cell::new(std::ptr::addr_of!(crate::object::shapes::PERRY_EMPTY_SHAPE_DIR) as *const u8); + slots +}); + +// Windows x86-64: `PERRY_AGENT_PTRS`'s section-relative offset in this image's +// TLS block (`.tls$`), as a 4-byte constant under a stable name. The static +// itself cannot be exported there (above), but `sym` names it under whatever +// symbol rustc gave it, and `.secrel32` is exactly the relocation rustc's own +// access to it uses (`mov _tls_index; mov gs:[0x58]; mov [..+idx*8]; +// sym@SECREL32`). Generated code performs that same sequence with this +// constant, so it reads the block without a call +// (`perry-codegen/src/expr/agent_ptr.rs`, `AgentPtrAccess::WindowsTeb`). +#[cfg(all(windows, target_arch = "x86_64"))] +core::arch::global_asm!( + ".section .rdata,\"dr\"", + ".globl PERRY_AGENT_PTRS_SECREL", + ".p2align 2", + "PERRY_AGENT_PTRS_SECREL:", + ".secrel32 {agent_ptrs}", + ".text", + agent_ptrs = sym PERRY_AGENT_PTRS, +); /// Publish (or clear, with null) one of this agent's pointers. #[inline] @@ -60,6 +92,22 @@ pub(crate) fn hot_addr() -> *mut u8 { &PERRY_AGENT_PTRS as *const AgentPtrs as *mut u8 } +/// The address of this agent's ordinary shape-directory mirror +/// (`ShapeSlab::ordinary_dir_addr`), published into slot +/// [`AGENT_PTR_SHAPE_DIR`]. A generic read site passes it to its GC-leaf miss +/// front (`read_confirm::js_object_get_field_ic_front`) so the front reads no +/// thread-local; this is the accessor for targets where emitted code cannot +/// read the block inline. The slab also publishes the slot whenever it +/// publishes its directory, so the inline reads see it once any shape exists +/// on this agent. The address is stable for the thread's life, so the slot is +/// never cleared; the mirror it names is. A leaf: one TLS read and one store. +#[no_mangle] +pub extern "C" fn perry_shape_dir_cell() -> *const u8 { + let dir = crate::object::shapes::ordinary_dir_addr(); + publish(AGENT_PTR_SHAPE_DIR, dir); + dir +} + /// The address of this agent's implicit-`this` cell, published into slot /// [`AGENT_PTR_IMPLICIT_THIS`] for emitted code that binds `this` around a /// direct method call. A leaf: one TLS read and one store. diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index eba8ac46d8..6b4662156f 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -343,6 +343,11 @@ pub use ic_slow::{js_object_get_field_ic_nonptr, js_object_get_field_ic_slow}; #[path = "field_get_set/ic_miss/outline_split.rs"] mod outline_split; pub use outline_split::{js_object_get_field_ic_fast, js_object_get_field_ic_fast_miss}; +/// First-read D3: the megamorphic read confirm, a GC-leaf stub a latched +/// generic read site calls before the slow entry. +#[path = "field_get_set/ic_miss/read_confirm.rs"] +mod read_confirm; +pub use read_confirm::js_object_get_field_ic_front; #[cfg(test)] mod buffer_ic_miss_tests { diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 78c7b67b99..e4f119976c 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -216,8 +216,8 @@ unsafe fn overflow_arm( super::ic_miss::get_field_ic_miss_impl(obj, key, cache_slot, std::ptr::null()) } -/// The exit for a receiver that IS a heap pointer — every failing guard on the -/// emitted site's object path lands here. +/// The exit for a receiver that IS a heap pointer — every failing guard and +/// every MRU miss on the emitted site's object path lands here (one call). /// /// * `obj_handle` — the receiver with the NaN-box tag already masked off. The /// caller has established the POINTER/STRING tag; this entry re-establishes @@ -238,62 +238,32 @@ pub extern "C-unwind" fn js_object_get_field_ic_slow( cache_slot: *mut PicCacheSlot, packed: *const AtomicU64, ) -> f64 { - // --- 0. a MEGAMORPHIC site's slot guess, confirmed by the receiver ------ - // - // A site whose way state is latched negative sees more shapes than any - // per-site entry can name, and every read that misses its compact word - // lands here. The site may still hold one thing: a slot GUESS (the compact - // word's high half — the slot the receiver's shape answered last, step 2b), - // which the RECEIVER'S own shape confirms or refutes - // (`shapes::confirm_slot_guess`: the position bound says key position - // `guess` is inline slot `guess`, and the key there IS this key, one - // pointer compare). Everything the guess cannot answer continues in - // `ic_slow_body` unchanged. Asked first, and only at a latched site, so a - // site that can still be primed is primed exactly as before. `length` is - // excluded as in step 2b: an Array-subclass receiver serves it from its - // elements store. Kept in this frameless entry, with the body out of line, - // so the confirmed read pays no prologue for the arms below. - if let Some(value) = unsafe { megamorphic_slot_guess(obj_handle, key, cache_slot, packed) } { - return value; - } - ic_slow_body(obj_handle, key, cache_slot, packed) -} - -/// Step 0 of [`js_object_get_field_ic_slow`]: the latched site's slot guess. -/// -/// # Safety -/// The entry's contract: a POINTER receiver handle, this site's cache slot -/// and packed word. -#[inline(always)] -unsafe fn megamorphic_slot_guess( - obj_handle: i64, - key: *const crate::StringHeader, - cache_slot: *mut PicCacheSlot, - packed: *const AtomicU64, -) -> Option { - let obj = obj_handle as usize as *const ObjectHeader; - if packed.is_null() - || key.is_null() - || !crate::value::addr_class::is_above_handle_band(obj as usize) + // First-read D3: the site asked its GC-leaf front + // (`read_confirm::js_object_get_field_ic_front`) first; what reaches this + // entry is what the front declined. A never-primed site asks the + // inherited-read cache (#10834/#10842) — the one edge an inherited read + // ever takes — and everything else runs the collecting body. + let addr = obj_handle as usize; + if !key.is_null() + && crate::value::addr_class::is_above_handle_band(addr) + // SAFETY: the site passes its own cache slot or null. + && unsafe { crate::object::pic_slot_peek(cache_slot) }.is_null() { - return None; - } - let cache = crate::object::pic_slot_peek(cache_slot); - if cache.is_null() - || (*cache)[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] >= 0 - || key_is_length(key) - { - return None; + // SAFETY: a POINTER-tagged payload above the handle band. + let v = unsafe { + crate::object::inherited_read_cache::js_inherited_read_cache_hit_f64( + addr as *const ObjectHeader, + key, + ) + }; + if v.to_bits() != crate::value::TAG_HOLE { + return v; + } } - let guess = ((*packed).load(Ordering::Relaxed) >> 32) as usize; - let value = crate::object::shapes::confirm_slot_guess(obj, key, guess)?; - #[cfg(test)] - crate::object::shapes::SHAPE_ANSWERED_READS.fetch_add(1, Ordering::Relaxed); - Some(value) + ic_slow_body(obj_handle, key, cache_slot, packed) } -/// Everything after step 0 of [`js_object_get_field_ic_slow`]. -#[inline(never)] +/// The body of [`js_object_get_field_ic_slow`]. fn ic_slow_body( obj_handle: i64, key: *const crate::StringHeader, @@ -434,7 +404,7 @@ fn ic_slow_body( /// `key` spells `length` — six bytes, compared directly (no UTF-8 validation). #[inline] -unsafe fn key_is_length(key: *const crate::StringHeader) -> bool { +pub(super) unsafe fn key_is_length(key: *const crate::StringHeader) -> bool { (*key).byte_len == 6 && std::slice::from_raw_parts(crate::string::string_data(key), 6) == b"length" } @@ -444,6 +414,10 @@ mod tests { use super::*; use crate::object::{PicCache, PIC_CACHE_WORDS}; + /// The read an emitted site performs on its miss edge: the confirm stub + /// first at a latched site, then the slow entry. + use super::super::read_confirm::test_site_miss_read as site_read; + /// The compact word an emitted site is born holding. const PACKED_GET_EMPTY_WORD: u64 = 0xFFFF_FFFF; @@ -503,7 +477,7 @@ mod tests { let packed = AtomicU64::new(0); let read = |o: &crate::gc::RuntimeHandle<'_>, slot: &mut PicCacheSlot| { o.with_mut_ptr(|p: *mut ObjectHeader| { - kind.with_const_ptr(|k| js_object_get_field_ic_slow(handle(p), k, slot, &packed)) + kind.with_const_ptr(|k| unsafe { site_read(handle(p), k, slot, &packed) }) }) }; // Drive the site until it latches. @@ -567,17 +541,13 @@ mod tests { for _ in 0..4 { for o in &objs { o.with_mut_ptr(|p: *mut ObjectHeader| { - kind.with_const_ptr(|k| { - js_object_get_field_ic_slow(handle(p), k, &mut slot, &packed) - }) + kind.with_const_ptr(|k| unsafe { site_read(handle(p), k, &mut slot, &packed) }) }); } } for o in &objs { let v = o.with_mut_ptr(|p: *mut ObjectHeader| { - absent.with_const_ptr(|k| { - js_object_get_field_ic_slow(handle(p), k, &mut slot, &packed) - }) + absent.with_const_ptr(|k| unsafe { site_read(handle(p), k, &mut slot, &packed) }) }); assert_eq!( v.to_bits(), @@ -642,7 +612,7 @@ mod tests { let packed = AtomicU64::new(0); let read = |o: &crate::gc::RuntimeHandle<'_>, slot: &mut PicCacheSlot| { o.with_mut_ptr(|p: *mut ObjectHeader| { - key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(p), k, slot, &packed)) + key.with_const_ptr(|k| unsafe { site_read(handle(p), k, slot, &packed) }) }) }; for round in 0..4 { @@ -874,9 +844,13 @@ mod tests { /// says key position `guess` is inline slot `guess` and holds exactly the /// key `site_key_bits` names, `None` for a decline. unsafe fn guess_walk(shape_id: u32, guess: u64, site_key_bits: u64) -> Option { - let key = (site_key_bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader; - crate::object::shapes::slot_guess_confirmed(shape_id, key, guess as usize) - .then_some(guess as usize) + crate::object::shapes::slot_guess_confirmed( + crate::object::shapes::ordinary_dir_addr(), + shape_id, + site_key_bits, + guess as usize, + ) + .then_some(guess as usize) } fn stamp_of(o: &crate::gc::RuntimeHandle<'_>) -> u32 { @@ -1470,7 +1444,7 @@ mod tests { let packed = AtomicU64::new(PACKED_GET_EMPTY_WORD); let read = |o: &crate::gc::RuntimeHandle<'_>, slot: &mut PicCacheSlot| { o.with_mut_ptr(|p: *mut ObjectHeader| { - key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(p), k, slot, &packed)) + key.with_const_ptr(|k| unsafe { site_read(handle(p), k, slot, &packed) }) }) }; let want = |i: usize| { diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/read_confirm.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/read_confirm.rs new file mode 100644 index 0000000000..6ce271109e --- /dev/null +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/read_confirm.rs @@ -0,0 +1,523 @@ +//! First-read D3: everything a generic read site's miss can answer WITHOUT +//! collecting, in one GC-leaf call. +//! +//! Emitted code keeps exactly one thing inline for a read: the receiver's +//! ShapeId compared against the site's compact word, and the slot load. A +//! ShapeId miss makes ONE call to [`js_object_get_field_ic_front`], which +//! answers, in this order and only from shape facts: +//! +//! 1. a polymorphic way (#7753): `(ShapeId token, slot)` pairs in the site's +//! full cache; +//! 2. a SPILL entry: the compact word holds the receiver's ShapeId flipped by +//! `PACKED_SPILL_FLIP`; the ShapeId fixes the key's index in the spill +//! buffer (`packed_get::prime_get`, `spill_reserve_claimed`); +//! 3. a LATCHED megamorphic site (way state negative): its slot guess (the +//! compact word's high half), confirmed by the receiver's own shape — the +//! shape record's `POSBOUND` and its canonical key list compared with the +//! key atom — and, on a wrong guess, one bounded scan of that key list that +//! re-aims the guess (D3b). +//! +//! Anything else answers `TAG_HOLE`, and only then does the site branch to +//! its cold block and call the collecting `js_object_get_field_ic_slow` with +//! its usual operands (a never-primed site's inherited-read cache is asked +//! there). The front never allocates, collects, enters JS, throws, takes a +//! lock or makes a call — not even a thread-local access: the site passes the +//! agent's shape-directory mirror (`PERRY_AGENT_PTRS` slot 0, never null) as +//! an operand. +//! It is therefore `Leaf` in the generated call-effects table, the site's call +//! is a plain `"gc-leaf-function"` call, and nothing live across it is +//! spilled or relocated: statepoint spills exist only on the cold slow edge. + +use super::ic_miss::{PACKED_GET_EMPTY, PACKED_SPILL_FLIP, PIC_WAYS, PIC_WAY_BASE, PIC_WAY_STATE}; +use crate::object::shapes::{is_shape_id, positional_key_words, PIC_ID_TOKEN_BIT}; +use crate::object::{ObjectHeader, PicCacheSlot}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// D3b's second chance scans at most this many key positions. +const SECOND_CHANCE_POSITIONS: usize = 32; + +#[inline(always)] +fn hole() -> f64 { + f64::from_bits(crate::value::TAG_HOLE) +} + +/// Inline slot `slot` of `obj`. +#[inline(always)] +unsafe fn inline_slot(obj: *const ObjectHeader, slot: usize) -> f64 { + *((obj as *const u8).add(std::mem::size_of::() + slot * 8) as *const f64) +} + +/// A generic read site's ShapeId miss (module docs): the answer, or +/// `TAG_HOLE` for the site's collecting slow call. +/// +/// * `dir` — this agent's shape-directory mirror +/// (`shapes::ordinary_dir_addr`, published in `PERRY_AGENT_PTRS` slot +/// `AGENT_PTR_SHAPE_DIR`), or `PERRY_EMPTY_SHAPE_DIR`, which confirms +/// nothing; never null. A `length` site passes the empty one on purpose: an +/// Array-subclass receiver serves `length` from its elements store, which no +/// key list names. +/// * `obj_handle` — the receiver's payload. The site calls only on the +/// ShapeId compare's false edge, which its small-handle test dominates, so +/// this is a real object pointer (its `+4` word was just loaded). +/// * `key_bits` — the site's key exactly as its pool global holds it: the +/// interned key, STRING-tagged, which is also how a canonical key list +/// stores it, so the confirm compares one word. +/// * `cache_slot`, `packed` — the site's full-cache slot (its global, never +/// null) and compact word. +/// +/// The ways are asked first (a polymorphic site's common miss), then the +/// spill entry, then a latched site's confirm: each answer is proven on its +/// own, so the order only decides who pays for which test. +/// +/// # Safety +/// The operands as a generic read site passes them (above). +#[no_mangle] +pub unsafe extern "C" fn js_object_get_field_ic_front( + dir: *const u8, + obj_handle: i64, + key_bits: u64, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, +) -> f64 { + let obj = obj_handle as usize as *const ObjectHeader; + let shape_id = (*obj).parent_class_id; + // `pic_slot_peek` without its null test: the slot is the site's global. + let cache = (*(cache_slot as *const std::sync::atomic::AtomicPtr)) + .load(Ordering::Acquire); + // A never-primed site (no cache) can still hold a spill entry in its + // compact word; what else can serve it (the inherited-read cache) is + // asked on the slow edge. + let state = if cache.is_null() { + 0 + } else { + (*cache)[PIC_WAY_STATE] + }; + if state > 0 { + // 1. The ways. A way token is `PIC_ID_TOKEN_BIT | ShapeId`; an empty + // way is 0 and cannot match. + let token = (shape_id as u64 | PIC_ID_TOKEN_BIT) as i64; + for w in 0..PIC_WAYS { + if (*cache)[PIC_WAY_BASE + 2 * w] == token { + return inline_slot(obj, (*cache)[PIC_WAY_BASE + 2 * w + 1] as usize); + } + } + } + let word = (*packed).load(Ordering::Relaxed); + // 2. Spill. Equality with a real ShapeId proves the receiver is an + // ordinary object of that shape (#10828 rule 3); the range test keeps an + // unflipped non-id word (a zeroed word flips to the synthetic-class floor) + // from matching an unstamped receiver. Nested, not `&&`: the common miss + // leaves on the first compare. + let spill_id = (word as u32) ^ PACKED_SPILL_FLIP; + if shape_id == spill_id { + if !is_shape_id(spill_id) { + return hole(); + } + let meta = (*obj).meta; + let spill = (*meta).spill as usize as *const u8; + let index = (word >> 32) as usize; + return *(spill.add(std::mem::size_of::() + index * 8) + as *const f64); + } + if state < 0 { + // 3. Latched. + return confirm_in(dir, obj, shape_id, packed, word, key_bits); + } + hole() +} + +/// The latched site's confirm (module docs, 3.), for tests that drive it +/// with an explicit directory. +#[cfg(test)] +pub(crate) fn read_confirm( + dir: *const u8, + obj_handle: i64, + shape_id: u32, + packed: *const AtomicU64, + key_bits: u64, +) -> f64 { + unsafe { + let word = (*packed).load(Ordering::Relaxed); + confirm_in( + dir, + obj_handle as usize as *const ObjectHeader, + shape_id, + packed, + word, + key_bits, + ) + } +} + +#[inline(always)] +unsafe fn confirm_in( + dir: *const u8, + obj: *const ObjectHeader, + shape_id: u32, + packed: *const AtomicU64, + word: u64, + key_bits: u64, +) -> f64 { + let Some((keys, bound)) = positional_key_words(dir, shape_id) else { + return hole(); + }; + let guess = (word >> 32) as usize; + // `guess < bound` proves POSBOUND nonzero before the keys are touched. + let slot = if guess < bound && *keys.words().add(guess) == key_bits { + guess + } else { + // D3b: the receiver's own key list, bounded; the found position is + // published as the new guess unless the word holds a stamp. + let scan = bound.min(SECOND_CHANCE_POSITIONS); + if scan == 0 { + return hole(); + } + let words = keys.words(); + let Some(found) = (0..scan).find(|&i| *words.add(i) == key_bits) else { + return hole(); + }; + if word as u32 == PACKED_GET_EMPTY as u32 { + (*packed).store( + ((found as u64) << 32) | (PACKED_GET_EMPTY as u32 as u64), + Ordering::Relaxed, + ); + } + found + }; + #[cfg(test)] + crate::object::shapes::SHAPE_ANSWERED_READS.fetch_add(1, Ordering::Relaxed); + inline_slot(obj, slot) +} + +/// A generic read site's miss as emitted code performs it: the leaf front +/// with this agent's directory, then the slow entry on `TAG_HOLE`. +#[cfg(test)] +pub(crate) unsafe fn test_site_miss_read( + obj_handle: i64, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, +) -> f64 { + let dir = if super::ic_slow::key_is_length(key) { + std::ptr::addr_of!(crate::object::shapes::PERRY_EMPTY_SHAPE_DIR) as *const u8 + } else { + crate::object::shapes::ordinary_dir_addr() + }; + let key_bits = key as usize as u64 | crate::value::STRING_TAG; + let v = js_object_get_field_ic_front(dir, obj_handle, key_bits, cache_slot, packed); + if v.to_bits() != crate::value::TAG_HOLE { + return v; + } + super::js_object_get_field_ic_slow(obj_handle, key, cache_slot, packed) +} + +#[cfg(test)] +mod tests { + use super::read_confirm as js_object_read_confirm; + use crate::gc::RuntimeHandle; + use crate::object::ObjectHeader; + use std::sync::atomic::{AtomicU64, Ordering}; + + fn atom(text: &[u8]) -> *mut crate::StringHeader { + let hash = crate::object::key_bytes_hash(text.as_ptr(), text.len()); + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) + } + + /// The confirm's answer bits and the site word after the call. + fn confirm( + obj: &RuntimeHandle<'_>, + key: &RuntimeHandle<'_>, + word: u64, + dir: *const u8, + ) -> (u64, u64) { + let packed = AtomicU64::new(word); + let bits = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k: *const crate::StringHeader| unsafe { + js_object_read_confirm( + dir, + o as i64, + (*o).parent_class_id, + &packed, + crate::value::js_nanbox_string(k as i64).to_bits(), + ) + .to_bits() + }) + }); + (bits, packed.load(Ordering::Relaxed)) + } + + fn guess(slot: u64) -> u64 { + (slot << 32) | 0xFFFF_FFFF + } + + /// The confirm answers exactly one thing: the receiver's own inline slot at + /// the guessed position, when the receiver's shape names the site's key + /// atom there. Every other input declines with `TAG_HOLE` — including a + /// key of the same TEXT that is not the atom (a pointer mismatch proves + /// nothing, so the slow entry decides), a guess past the position bound, + /// an id that names no ordinary record, and a null directory. + #[test] + fn the_confirm_answers_only_what_the_receivers_shape_names() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let a = scope.root_string_ptr(atom(b"d3_confirm_a")); + let b = scope.root_string_ptr(atom(b"d3_confirm_b")); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + for (k, v) in [(&a, 11.0), (&b, 22.0)] { + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + let dir = crate::object::shapes::ordinary_dir_addr(); + let hole = crate::value::TAG_HOLE; + let (v11, v22) = (11.0f64.to_bits(), 22.0f64.to_bits()); + // The guess, confirmed: answered, the word untouched. + assert_eq!(confirm(&obj, &a, guess(0), dir), (v11, guess(0)), "a at 0"); + assert_eq!(confirm(&obj, &b, guess(1), dir), (v22, guess(1)), "b at 1"); + // A wrong guess, or one past the position bound: the second chance + // finds the atom in the receiver's own key list, answers it, and + // re-aims the guess. + assert_eq!( + confirm(&obj, &a, guess(1), dir), + (v11, guess(0)), + "a is at 0" + ); + assert_eq!( + confirm(&obj, &b, guess(0), dir), + (v22, guess(1)), + "b is at 1" + ); + assert_eq!( + confirm(&obj, &a, guess(9), dir), + (v11, guess(0)), + "past the bound" + ); + // A word whose low half is a matchable stamp is never re-aimed. + let stamped = (1u64 << 32) | 0x8000_0001; + assert_eq!( + confirm(&obj, &a, stamped, dir), + (v11, stamped), + "stamp kept" + ); + // The empty directory confirms nothing: declined, the slow entry + // decides. + let empty = std::ptr::addr_of!(crate::object::shapes::PERRY_EMPTY_SHAPE_DIR) as *const u8; + assert_eq!( + confirm(&obj, &a, guess(0), empty), + (hole, guess(0)), + "empty dir" + ); + let copy_text = b"d3_confirm_a"; + let copy = scope.root_string_ptr(crate::string::js_string_from_bytes( + copy_text.as_ptr(), + copy_text.len() as u32, + )); + assert_ne!( + copy.with_const_ptr(|p: *const crate::StringHeader| p as usize), + a.with_const_ptr(|p: *const crate::StringHeader| p as usize), + "premise: the copy is another string object" + ); + assert_eq!( + confirm(&obj, ©, guess(0), dir), + (hole, guess(0)), + "same text, not the atom: declined, the slow entry decides" + ); + let unrecorded = a.with_const_ptr(|k: *const crate::StringHeader| { + obj.with_mut_ptr(|o: *mut ObjectHeader| { + js_object_read_confirm( + dir, + o as i64, + 7, + &AtomicU64::new(0xFFFF_FFFF), + crate::value::js_nanbox_string(k as i64).to_bits(), + ) + .to_bits() + }) + }); + assert_eq!(unrecorded, hole, "an id below the ShapeId band"); + } + + /// The front as a site calls it: `(answer bits, word after)`. + fn front( + obj: *mut ObjectHeader, + key_bits: u64, + cache_slot: *mut crate::object::PicCacheSlot, + word: u64, + ) -> (u64, u64) { + let packed = AtomicU64::new(word); + let dir = crate::object::shapes::ordinary_dir_addr(); + let bits = unsafe { + super::js_object_get_field_ic_front(dir, obj as i64, key_bits, cache_slot, &packed) + .to_bits() + }; + (bits, packed.load(Ordering::Relaxed)) + } + + /// #7753 in the front: a polymorphic site's way — `(PIC_ID_TOKEN_BIT | + /// ShapeId, slot)` in the full cache — is answered before anything else, + /// from the receiver's own inline slot; a way naming another shape, an + /// unprimed site (null cache: never dereferenced) and a primed site whose + /// ways hold nothing for this shape all decline to the slow call. + #[test] + fn the_front_answers_a_way_and_declines_a_null_cache() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let a = scope.root_string_ptr(atom(b"d3_front_way_a")); + let b = scope.root_string_ptr(atom(b"d3_front_way_b")); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + for (k, v) in [(&a, 11.0), (&b, 22.0)] { + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + let key_bits = b.with_const_ptr(|k: *const crate::StringHeader| { + crate::value::js_nanbox_string(k as i64).to_bits() + }); + let hole = crate::value::TAG_HOLE; + obj.with_mut_ptr(|o: *mut ObjectHeader| { + let shape_id = unsafe { (*o).parent_class_id }; + assert!(super::is_shape_id(shape_id), "premise: a shaped receiver"); + let mut cache: crate::object::PicCache = [0; crate::object::PIC_CACHE_WORDS]; + cache[super::PIC_WAY_STATE] = 1; + cache[super::PIC_WAY_BASE + 2] = (shape_id as u64 | super::PIC_ID_TOKEN_BIT) as i64; + cache[super::PIC_WAY_BASE + 3] = 1; + let mut slot: crate::object::PicCacheSlot = &mut cache; + assert_eq!( + front(o, key_bits, &mut slot, 0xFFFF_FFFF), + (22.0f64.to_bits(), 0xFFFF_FFFF), + "the way names this shape: its slot answers" + ); + cache[super::PIC_WAY_BASE + 2] = + ((shape_id + 1) as u64 | super::PIC_ID_TOKEN_BIT) as i64; + let mut slot: crate::object::PicCacheSlot = &mut cache; + assert_eq!( + front(o, key_bits, &mut slot, 0xFFFF_FFFF).0, + hole, + "a way for another shape" + ); + let mut null_slot: crate::object::PicCacheSlot = std::ptr::null_mut(); + assert_eq!( + front(o, key_bits, &mut null_slot, 0xFFFF_FFFF).0, + hole, + "never primed" + ); + }); + } + + /// S5 in the front: a SPILL entry — the compact word holding the + /// receiver's ShapeId flipped by `PACKED_SPILL_FLIP` — is served from the + /// spill buffer at the word's index; and the un-flipped id must be a REAL + /// ShapeId before it proves anything. A zeroed word un-flips to + /// `PACKED_SPILL_FLIP` itself, the synthetic-class floor, which an + /// unstamped receiver's `+4` word can hold: that receiver is not an + /// ordinary shaped object and its spill buffer answers nothing. + #[test] + fn the_front_serves_a_spill_entry_only_for_a_real_shape_id() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let names: [&[u8]; 4] = [b"d3_sp_a", b"d3_sp_b", b"d3_sp_c", b"d3_sp_d"]; + let keys: Vec<_> = names + .iter() + .map(|n| scope.root_string_ptr(atom(n))) + .collect(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0)); + for (i, k) in keys.iter().enumerate() { + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| { + crate::object::js_object_set_field_by_name(o, kp, 10.0 + i as f64) + }) + }); + } + let last = &keys[3]; + let key_bits = last.with_const_ptr(|k: *const crate::StringHeader| { + crate::value::js_nanbox_string(k as i64).to_bits() + }); + let packed = AtomicU64::new(0xFFFF_FFFF); + let mut slot: crate::object::PicCacheSlot = std::ptr::null_mut(); + let primed = obj.with_mut_ptr(|o: *mut ObjectHeader| { + last.with_const_ptr(|k| { + super::super::js_object_get_field_ic_slow(o as i64, k, &mut slot, &packed) + }) + }); + assert_eq!(primed, 13.0, "the priming read"); + let word = packed.load(Ordering::Relaxed); + obj.with_mut_ptr(|o: *mut ObjectHeader| { + let shape_id = unsafe { (*o).parent_class_id }; + assert_eq!( + (word as u32) ^ super::PACKED_SPILL_FLIP, + shape_id, + "premise: the site published a spill entry (word {word:#x})" + ); + let mut none: crate::object::PicCacheSlot = std::ptr::null_mut(); + assert_eq!( + front(o, key_bits, &mut none, word), + (13.0f64.to_bits(), word), + "a spill entry for this shape is served by the front" + ); + // The same receiver under a `+4` word that is no ShapeId, read at a + // site whose word's low half is 0 (it un-flips to exactly that + // word) and whose index names the live spill value, so a front + // that skipped the id check would answer it. + assert!(!super::is_shape_id(super::PACKED_SPILL_FLIP), "premise"); + unsafe { (*o).parent_class_id = super::PACKED_SPILL_FLIP }; + let unstamped = front(o, key_bits, &mut none, word & !0xFFFF_FFFF).0; + unsafe { (*o).parent_class_id = shape_id }; + assert_eq!( + unstamped, + crate::value::TAG_HOLE, + "an un-flipped word that is no ShapeId proves nothing" + ); + }); + } + + /// S6: a `length` site is never confirmed from the receiver's shape — its + /// front is handed the EMPTY directory (codegen: + /// `array_length::a_length_read_serves_a_live_plain_array_off_the_shape_compare` + /// pins the emitted operand; `test_site_miss_read` mirrors it). An + /// Array-subclass receiver serves `length` from its elements store, so a + /// latched `length` site gains nothing from a key-list confirm; every such + /// read goes to the slow entry, which answers `length` for every receiver + /// kind. Pinned on the receiver where the confirm WOULD answer — a plain + /// object with an own `length` data key — by the shape-answered counter: + /// the read is right either way, so the value alone cannot see a `length` + /// site that started confirming. + #[test] + fn a_length_site_is_never_confirmed_from_the_shape() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let len = scope.root_string_ptr(atom(b"length")); + let other = scope.root_string_ptr(atom(b"d3_len_other")); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + for (k, v) in [(&len, 3.0), (&other, 4.0)] { + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + let answered = || crate::object::shapes::SHAPE_ANSWERED_READS.load(Ordering::Relaxed); + let mut cache: crate::object::PicCache = [0; crate::object::PIC_CACHE_WORDS]; + cache[super::PIC_WAY_STATE] = -1_000_000; + let mut read = |key: &RuntimeHandle<'_>, word: u64| { + let mut slot: crate::object::PicCacheSlot = &mut cache; + let packed = AtomicU64::new(word); + obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| unsafe { + super::test_site_miss_read(o as i64, k, &mut slot, &packed) + }) + }) + }; + // Premise: the same latched site confirms an ordinary key from the + // shape, so the counter can move. + let before = answered(); + assert_eq!(read(&other, guess(1)), 4.0); + assert_eq!(answered(), before + 1, "premise: a latched confirm counts"); + for word in [guess(0), guess(1), 0xFFFF_FFFF] { + let before = answered(); + assert_eq!(read(&len, word), 3.0, "a latched `length` read ({word:#x})"); + assert_eq!( + answered(), + before, + "a `length` read was confirmed from the shape ({word:#x})" + ); + } + } +} diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 02edd45432..8b54114da6 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -50,6 +50,7 @@ pub(crate) use shapes_slot_list::{ shape_index_migrate_after_delete, shape_index_shift_in_place, try_update_stable_tombstone_shape, try_update_stable_tombstone_shape_cached, SlotIndex, }; +pub(crate) use shapes_store::PERRY_EMPTY_SHAPE_DIR; use shapes_store::{ IdList, ShapeRecord, ShapeSlab, RECORD_FLAG_BIRTH_OWNER, RECORD_FLAG_CACHE_CARRIER, RECORD_FLAG_CARRIED_SEEN, RECORD_FLAG_EXTERNAL_CARRIER, RECORD_FLAG_FACTS_INDEXED, @@ -413,67 +414,91 @@ pub(crate) fn test_positional_of_id(id: u32) -> Option<(u32, u32)> { }) } +/// The address of this thread's ordinary shape-directory mirror, which +/// [`positional_key_words`] reads through (`agent_ptrs` slot +/// `AGENT_PTR_SHAPE_DIR`). +#[inline] +pub(crate) fn ordinary_dir_addr() -> *const u8 { + ShapeSlab::ordinary_dir_addr() +} + /// The position bound of shape `id` (S3c), or `None` when it names no record. #[cfg(test)] pub(crate) fn test_position_bound_of_id(id: u32) -> Option { shape_record_by_id(id).map(|r| unsafe { (*r.0.as_ptr()).position_bound() }) } -/// The megamorphic read's slot-guess confirm: when `obj` carries an ordinary -/// ShapeId of this agent whose record says key position `guess` is inline slot -/// `guess` (`position_bound`), and the key AT that position is `key` itself -/// (one pointer compare: canonical lists hold their text's atom), the value in -/// the receiver's slot `guess`. `None` for anything else — a wrong or stale -/// guess, another text, a dictionary/class/descriptor/tombstoned shape, an id -/// that names no record — and the caller takes its ordinary path. +/// Shape `shape_id`'s positional key words, for the megamorphic read confirm +/// (`ic_miss::read_confirm::js_object_get_field_ic_front`): `(the keys +/// array, POSBOUND)` when the record answers by position — key position +/// `i < POSBOUND` IS inline slot `i` of every receiver carrying the shape — +/// and `None` otherwise (no ordinary record under that id in this agent, or +/// POSBOUND 0: a dictionary, class, descriptor/prototype-generation, +/// tombstoned or accessor-keyed shape). /// -/// The guess decides nothing: the receiver's own shape confirms it or it is -/// ignored. Allocation-free, no user code. +/// A canonical list holds its text's ATOM, boxed exactly as a site's pool +/// entry holds it, so the caller compares key WORDS: equality is identity, +/// and a mismatch proves nothing (a list written before its atom existed, an +/// SSO slot), which the caller answers by declining to the slow entry. +/// +/// `dir` is this thread's ordinary directory mirror ([`ordinary_dir_addr`]), +/// as the agent's pointer block holds it. +/// +/// Allocation-free, no user code, no formatting path: it is part of a +/// GC-leaf stub. /// /// # Safety -/// `obj` is a heap pointer above the handle band (its `+4` word is read); -/// `key` is a heap `StringHeader`. -#[inline] -pub(crate) unsafe fn confirm_slot_guess( - obj: *const crate::object::ObjectHeader, - key: *const crate::StringHeader, - guess: usize, -) -> Option { - if !slot_guess_confirmed((*obj).parent_class_id, key, guess) { - return None; +/// `dir` is this thread's [`ordinary_dir_addr`] or `PERRY_EMPTY_SHAPE_DIR` (never +/// null); any `shape_id`. +/// The words are valid until the next safepoint. +#[inline(always)] +pub(crate) unsafe fn positional_key_words( + dir: *const u8, + shape_id: u32, +) -> Option<(PositionalKeys, usize)> { + let r = ShapeSlab::ordinary_record_in(dir, shape_id)?; + Some(( + PositionalKeys(r.keys as usize as *const ArrayHeader), + r.position_bound_raw() as usize, + )) +} + +/// A record's canonical keys array, for [`positional_key_words`]: its words +/// are asked only once POSBOUND is known to be nonzero, so the front-offset +/// arithmetic runs only on the path that reads a key. +pub(crate) struct PositionalKeys(*const ArrayHeader); + +impl PositionalKeys { + /// The first logical key word. + /// + /// # Safety + /// Only when the record's POSBOUND is nonzero: then `keys` names a live + /// keys array (the collector marks through and rewrites `keys`) holding + /// at least POSBOUND logical keys. Logical element `i` sits past the + /// array's FRONT OFFSET, which a canonical list can carry without ever + /// being shifted (a size-class round-up alone makes the physical capacity + /// exceed the logical one: `keys_front_offset_tests`), so the accessor is + /// asked, never `+8`. + #[inline(always)] + pub(crate) unsafe fn words(&self) -> *const u64 { + crate::array::array_elements_ptr(self.0) as *const u64 } - Some( - *((obj as *const u8).add(std::mem::size_of::() + guess * 8) - as *const f64), - ) } -/// Does shape `shape_id` of this agent store `key` at inline slot `guess`, -/// by position? See [`confirm_slot_guess`]. +/// Does shape `shape_id` store the key whose NaN-boxed bits are `key_bits` +/// at inline slot `guess`, by position? See [`positional_key_words`]. /// /// # Safety -/// `key` is a heap `StringHeader`. -#[inline] +/// As [`positional_key_words`]. +#[cfg(test)] pub(crate) unsafe fn slot_guess_confirmed( + dir: *const u8, shape_id: u32, - key: *const crate::StringHeader, + key_bits: u64, guess: usize, ) -> bool { - // SAFETY: this thread's own mirror. - let record = unsafe { ShapeSlab::ordinary_record_in(ShapeSlab::ordinary_dir_addr(), shape_id) }; - if record.is_null() { - return false; - } - let r = &*record; - if guess >= r.position_bound_raw() as usize { - return false; - } - // A bound > 0 means the record names a live keys array (the collector - // marks through and rewrites `keys`) holding at least `bound` logical - // keys; logical element `i` sits past the array's front offset. - let arr = r.keys as usize as *const ArrayHeader; - let slots = crate::array::array_elements_ptr(arr) as *const u64; - *slots.add(guess) == crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits() + positional_key_words(dir, shape_id) + .is_some_and(|(keys, bound)| guess < bound && *keys.words().add(guess) == key_bits) } /// Byte equality without a libc call for the short keys property names are. diff --git a/crates/perry-runtime/src/object/shapes_store.rs b/crates/perry-runtime/src/object/shapes_store.rs index 0997be562f..e3b3bac941 100644 --- a/crates/perry-runtime/src/object/shapes_store.rs +++ b/crates/perry-runtime/src/object/shapes_store.rs @@ -540,30 +540,151 @@ const PAGE_MASK: usize = PAGE_LEN - 1; /// the table interior mutability through a shared slab reference: the /// collector writes liveness bits and the `keys` word through raw record /// pointers while other code holds only copies (`ShapeDescriptor`). -type Chunk = Box<[UnsafeCell; CHUNK_LEN]>; +type ChunkCells = [UnsafeCell; CHUNK_LEN]; /// One directory page: `PAGE_LEN` chunk slots. -type Page = Box<[Option; PAGE_LEN]>; - -fn new_chunk() -> Chunk { - let mut v: Vec> = Vec::with_capacity(CHUNK_LEN); - v.resize_with(CHUNK_LEN, || UnsafeCell::new(ShapeRecord::EMPTY)); - // Exact length by construction; the conversion moves the allocation. - v.into_boxed_slice() - .try_into() - .unwrap_or_else(|_| unreachable!("chunk vector has CHUNK_LEN cells")) +type PageSlots = [Slot; PAGE_LEN]; + +/// A directory or page entry: an allocation this slab owns, or the SHARED +/// all-empty one of its level ([`EMPTY_CHUNK`], [`EMPTY_PAGE`]) — never null. +/// An absent run therefore reads exactly like a present run of absent +/// records (`ShapeRecord::EMPTY`: not present, position bound 0), so a +/// reader walks page → chunk → record with no null test at either level +/// (the megamorphic read confirm, `ordinary_record_in`). Nothing is ever +/// written through a shared empty: every writer asks [`Slot::is_shared`] +/// first and allocates. The slab frees what it owns ([`ShapeSlab::free_dir`]). +#[repr(transparent)] +struct Slot(std::ptr::NonNull); + +impl Clone for Slot { + fn clone(&self) -> Self { + *self + } } +impl Copy for Slot {} + +/// A shared all-empty allocation. Never written (see [`Slot`]). +#[repr(transparent)] +pub struct SharedEmpty(T); +// SAFETY: nothing ever writes a shared empty; every reader only loads. +unsafe impl Sync for SharedEmpty {} + +static EMPTY_CHUNK: SharedEmpty = + SharedEmpty([const { UnsafeCell::new(ShapeRecord::EMPTY) }; CHUNK_LEN]); +static EMPTY_PAGE: SharedEmpty = SharedEmpty( + // SAFETY: the address of a static is never null. + [Slot(unsafe { + std::ptr::NonNull::new_unchecked(std::ptr::addr_of!(EMPTY_CHUNK.0) as *mut ChunkCells) + }); PAGE_LEN], +); -fn new_page() -> Page { - let mut v: Vec> = Vec::with_capacity(PAGE_LEN); - v.resize_with(PAGE_LEN, || None); - v.into_boxed_slice() - .try_into() - .unwrap_or_else(|_| unreachable!("page vector has PAGE_LEN slots")) +trait Level: Sized + 'static { + fn shared() -> std::ptr::NonNull; + fn fresh() -> Box; } +impl Level for ChunkCells { + fn shared() -> std::ptr::NonNull { + std::ptr::NonNull::from(&EMPTY_CHUNK.0) + } + fn fresh() -> Box { + let mut v: Vec> = Vec::with_capacity(CHUNK_LEN); + v.resize_with(CHUNK_LEN, || UnsafeCell::new(ShapeRecord::EMPTY)); + // Exact length by construction; the conversion moves the allocation. + v.into_boxed_slice() + .try_into() + .unwrap_or_else(|_| unreachable!("chunk vector has CHUNK_LEN cells")) + } +} + +impl Level for PageSlots { + fn shared() -> std::ptr::NonNull { + std::ptr::NonNull::from(&EMPTY_PAGE.0) + } + fn fresh() -> Box { + let mut v: Vec> = Vec::with_capacity(PAGE_LEN); + v.resize_with(PAGE_LEN, Slot::empty); + v.into_boxed_slice() + .try_into() + .unwrap_or_else(|_| unreachable!("page vector has PAGE_LEN slots")) + } +} + +impl Slot { + #[inline] + fn empty() -> Self { + Slot(T::shared()) + } + #[inline] + fn is_shared(self) -> bool { + self.0 == T::shared() + } + /// The owned allocation, or `None` for the shared empty. + #[inline] + fn owned(&self) -> Option<&T> { + // SAFETY: an owned slot points at a live allocation of this slab. + (!self.is_shared()).then(|| unsafe { self.0.as_ref() }) + } + #[inline] + fn owned_mut(&mut self) -> Option<&mut T> { + // SAFETY: as `owned`; `&mut self` is the slab's exclusive borrow. + (!self.is_shared()).then(|| unsafe { self.0.as_mut() }) + } + /// The owned allocation, allocating it first if this is the shared empty. + #[inline] + fn owned_or_alloc(&mut self) -> &mut T { + if self.is_shared() { + self.0 = std::ptr::NonNull::from(Box::leak(T::fresh())); + } + // SAFETY: owned now. + unsafe { self.0.as_mut() } + } + /// Free an owned allocation (not its children) and become the shared + /// empty. + fn release(&mut self) { + if !self.is_shared() { + // SAFETY: allocated by `owned_or_alloc`, freed once: the slot is + // the shared empty afterwards. + drop(unsafe { Box::from_raw(self.0.as_ptr()) }); + self.0 = T::shared(); + } + } +} + +type Page = Slot; + /// The ordinary directory mirror's type: `(page pointers, page count)`. -type OrdinaryDir = std::cell::Cell<(*const Option, usize)>; +#[repr(C)] +pub(crate) struct OrdinaryDir { + pages: std::cell::Cell<*const Page>, + len: std::cell::Cell, +} + +impl OrdinaryDir { + const fn empty() -> Self { + OrdinaryDir { + pages: std::cell::Cell::new(std::ptr::null()), + len: std::cell::Cell::new(0), + } + } + #[inline(always)] + fn get(&self) -> (*const Page, usize) { + (self.pages.get(), self.len.get()) + } + #[inline] + fn set(&self, (pages, len): (*const Page, usize)) { + self.pages.set(pages); + self.len.set(len); + } +} + +/// A directory of no pages, never written: the value of the agent's +/// shape-directory pointer slot until the agent publishes its own mirror +/// (`agent_ptrs::PERRY_AGENT_PTRS`), and what a `length` read site passes +/// (`perry-codegen` `generic_dispatch.rs`). Every id indexes past its length, +/// so a reader never needs a null test for the directory itself. +#[no_mangle] +pub static PERRY_EMPTY_SHAPE_DIR: SharedEmpty = SharedEmpty(OrdinaryDir::empty()); /// This thread's ordinary page directory as `(page pointers, page count)`: /// `ShapeSlab::pages`' element pointer and length, republished after every @@ -578,7 +699,7 @@ type OrdinaryDir = std::cell::Cell<(*const Option, usize)>; /// than `thread_local!`: the address is stable for the thread's life, and a /// late read during thread teardown sees the cleared pair. #[thread_local] -static ORDINARY_DIR: OrdinaryDir = std::cell::Cell::new((std::ptr::null(), 0)); +static ORDINARY_DIR: OrdinaryDir = OrdinaryDir::empty(); /// The by-id descriptor store. See the module docs. /// Two page directories: ordinary ShapeIds index from `SHAPE_ID_BASE`, and the @@ -593,14 +714,17 @@ impl Drop for ShapeSlab { if ORDINARY_DIR.get().0 == self.pages.as_ptr() { ORDINARY_DIR.set((std::ptr::null(), 0)); } + for band in [0u8, 1, 2] { + Self::free_dir(self.dir_mut(band)); + } } } pub(crate) struct ShapeSlab { - pages: Vec>, - dict_pages: Vec>, + pages: Vec, + dict_pages: Vec, /// The exotic-receiver band (`shapes::EXOTIC_SHAPE_ID_BASE`). - exotic_pages: Vec>, + exotic_pages: Vec, /// Present records. len: usize, } @@ -641,7 +765,7 @@ impl ShapeSlab { } #[inline] - fn dir(&self, band: u8) -> &Vec> { + fn dir(&self, band: u8) -> &Vec { match band { 0 => &self.pages, 1 => &self.dict_pages, @@ -650,7 +774,7 @@ impl ShapeSlab { } #[inline] - fn dir_mut(&mut self, band: u8) -> &mut Vec> { + fn dir_mut(&mut self, band: u8) -> &mut Vec { match band { 0 => &mut self.pages, 1 => &mut self.dict_pages, @@ -681,7 +805,7 @@ impl ShapeSlab { pub(super) fn record_ptr(&self, id: u32) -> Option<*mut ShapeRecord> { let (dict, index) = Self::index_of(id)?; let (page, chunk, slot) = Self::split(index); - let chunk = self.dir(dict).get(page)?.as_ref()?[chunk].as_ref()?; + let chunk = self.dir(dict).get(page)?.owned()?[chunk].owned()?; let cell = chunk[slot].get(); // SAFETY: the cell belongs to a live chunk owned by this slab; reads // and writes are serialized by the single-threaded agent discipline @@ -716,15 +840,15 @@ impl ShapeSlab { let (page, chunk, slot) = Self::split(index); let dir = self.dir_mut(band); if page >= dir.len() { - dir.resize_with(page + 1, || None); + dir.resize_with(page + 1, Slot::empty); // Only the ordinary band is mirrored (`ORDINARY_DIR`). if band == 0 { self.publish_dir(); } } let dir = self.dir_mut(band); - let page = dir[page].get_or_insert_with(new_page); - let chunk = page[chunk].get_or_insert_with(new_chunk); + let page = dir[page].owned_or_alloc(); + let chunk = page[chunk].owned_or_alloc(); let cell = chunk[slot].get_mut(); let previous = cell.present().then_some(*cell); // A retire-and-reinsert edits facts on a removed copy: the positional @@ -741,7 +865,7 @@ impl ShapeSlab { pub(super) fn remove(&mut self, id: u32) -> Option { let (dict, index) = Self::index_of(id)?; let (page, chunk, slot) = Self::split(index); - let chunk = self.dir_mut(dict).get_mut(page)?.as_mut()?[chunk].as_mut()?; + let chunk = self.dir_mut(dict).get_mut(page)?.owned_mut()?[chunk].owned_mut()?; let cell = chunk[slot].get_mut(); if !cell.present() { return None; @@ -757,11 +881,11 @@ impl ShapeSlab { pub(super) fn for_each(&self, mut f: impl FnMut(u32, *mut ShapeRecord)) { for dict in [0u8, 1, 2] { for (page_index, page) in self.dir(dict).iter().enumerate() { - let Some(page) = page else { + let Some(page) = page.owned() else { continue; }; for (chunk_index, chunk) in page.iter().enumerate() { - let Some(chunk) = chunk else { + let Some(chunk) = chunk.owned() else { continue; }; let base = ((page_index << PAGE_SHIFT) | chunk_index) << CHUNK_SHIFT; @@ -793,26 +917,26 @@ impl ShapeSlab { for dict in [0u8, 1, 2] { let dir = self.dir_mut(dict); for page in dir.iter_mut() { - let Some(chunks) = page.as_mut() else { + let Some(chunks) = page.owned_mut() else { continue; }; let mut live_chunks = 0usize; for chunk in chunks.iter_mut() { let empty = chunk - .as_ref() + .owned() .is_some_and(|c| c.iter().all(|cell| !unsafe { (*cell.get()).present() })); if empty { - *chunk = None; + chunk.release(); } - if chunk.is_some() { + if !chunk.is_shared() { live_chunks += 1; } } if live_chunks == 0 { - *page = None; + page.release(); } } - while dir.last().is_some_and(Option::is_none) { + while dir.last().is_some_and(|p| p.is_shared()) { dir.pop(); } dir.shrink_to_fit(); @@ -823,6 +947,12 @@ impl ShapeSlab { /// Publish `pages` for [`Self::ordinary_record_in`] (see [`ORDINARY_DIR`]). fn publish_dir(&self) { ORDINARY_DIR.set((self.pages.as_ptr(), self.pages.len())); + // Emitted read sites hand the mirror's address to the miss front + // from the agent's pointer block; publish it with the directory. + crate::agent_ptrs::publish( + crate::agent_ptrs::AGENT_PTR_SHAPE_DIR, + Self::ordinary_dir_addr(), + ); } /// The address of THIS thread's [`ORDINARY_DIR`] mirror, as an opaque @@ -838,41 +968,40 @@ impl ShapeSlab { /// The record of ordinary ShapeId `id` in the slab whose [`ORDINARY_DIR`] /// mirror is at `dir` (an [`Self::ordinary_dir_addr`] of this thread, or - /// null), or null — the fast twin of [`Self::record_ptr`] for the + /// `PERRY_EMPTY_SHAPE_DIR`), or `None` — the fast twin of [`Self::record_ptr`] for the /// megamorphic read: two dependent directory loads, no `state()`, no /// thread-local access. A dictionary- or exotic-band id indexes past the /// ordinary directory's length. The record may be absent (`EMPTY`): its /// position bound is 0. /// /// # Safety - /// `dir` is null or this thread's [`Self::ordinary_dir_addr`]. + /// `dir` is this thread's [`Self::ordinary_dir_addr`] or + /// `PERRY_EMPTY_SHAPE_DIR`; never null. #[inline(always)] - pub(super) unsafe fn ordinary_record_in(dir: *const u8, id: u32) -> *const ShapeRecord { - if dir.is_null() { - return std::ptr::null(); - } + pub(super) unsafe fn ordinary_record_in<'a>( + dir: *const u8, + id: u32, + ) -> Option<&'a ShapeRecord> { let (pages, len) = (*(dir as *const OrdinaryDir)).get(); let index = id.wrapping_sub(SHAPE_ID_BASE) as usize; let (page, chunk, slot) = Self::split(index); if page >= len { - return std::ptr::null(); + return None; } // SAFETY: `pages` holds `len` entries of this thread's slab, current - // as of the last change to it; nothing here can change it. - let Some(page) = (*pages.add(page)).as_ref() else { - return std::ptr::null(); - }; - match page[chunk].as_ref() { - Some(chunk) => chunk[slot].get(), - None => std::ptr::null(), - } + // as of the last change to it; nothing here can change it. An absent + // page or chunk is the shared empty one (`Slot`), never null, so + // both levels are plain loads and the record is never null. + let page = (*pages.add(page)).0.as_ref(); + let chunk = page[chunk].0.as_ref(); + Some(&*chunk[slot].get()) } #[cfg(test)] pub(super) fn clear(&mut self) { - self.pages.clear(); - self.dict_pages.clear(); - self.exotic_pages.clear(); + for band in [0u8, 1, 2] { + Self::free_dir(self.dir_mut(band)); + } self.publish_dir(); self.len = 0; } @@ -887,14 +1016,14 @@ impl ShapeSlab { .iter() .chain(self.dict_pages.iter()) .chain(self.exotic_pages.iter()) - .flatten() + .filter_map(Slot::owned) { pages += 1; - chunks += page.iter().filter(|c| c.is_some()).count(); + chunks += page.iter().filter(|c| !c.is_shared()).count(); } (self.pages.capacity() + self.dict_pages.capacity() + self.exotic_pages.capacity()) - * std::mem::size_of::>() - + pages * PAGE_LEN * std::mem::size_of::>() + * std::mem::size_of::() + + pages * PAGE_LEN * std::mem::size_of::>() + chunks * CHUNK_LEN * std::mem::size_of::() } @@ -905,10 +1034,23 @@ impl ShapeSlab { .iter() .chain(self.dict_pages.iter()) .chain(self.exotic_pages.iter()) - .flatten() - .map(|page| page.iter().filter(|c| c.is_some()).count()) + .filter_map(Slot::owned) + .map(|page| page.iter().filter(|c| !c.is_shared()).count()) .sum() } + + /// Free every page and chunk a directory owns, and empty it. + fn free_dir(dir: &mut Vec) { + for page in dir.iter_mut() { + if let Some(chunks) = page.owned_mut() { + for chunk in chunks.iter_mut() { + chunk.release(); + } + } + page.release(); + } + dir.clear(); + } } /// MEASUREMENT that this structure is judged on, and the test's instrument. From 70c6e2fed344c61fddccf727343b902b09bb5068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 03:43:59 +0200 Subject: [PATCH 11/16] changelog: name the fragment after #11657 --- ...ic-read-miss-front.md => 11657-megamorphic-read-miss-front.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{megamorphic-read-miss-front.md => 11657-megamorphic-read-miss-front.md} (100%) diff --git a/changelog.d/megamorphic-read-miss-front.md b/changelog.d/11657-megamorphic-read-miss-front.md similarity index 100% rename from changelog.d/megamorphic-read-miss-front.md rename to changelog.d/11657-megamorphic-read-miss-front.md From 269c1a00f002a3196b9aa39bab8ed9b9f168476c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 07:29:17 +0000 Subject: [PATCH 12/16] merge fixups: stack guard knows WindowsTeb; census reads the Slot slab main's stack guard (#10812) matches AgentPtrAccess, which this branch extended with WindowsTeb: the runtime publishes no stack limit on Windows, so no check is emitted there, as before. The census authority surfaces and the reallocating-chunk sabotage now name the Slot-based slab (ChunkCells, PageSlots, Page = Slot) this branch introduced. --- crates/perry-codegen/src/expr/agent_ptr.rs | 2 +- crates/perry-codegen/src/expr/stack_guard.rs | 13 ++++++++----- scripts/shape_descriptor_census.py | 19 +++++++++++-------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/crates/perry-codegen/src/expr/agent_ptr.rs b/crates/perry-codegen/src/expr/agent_ptr.rs index fcb5db0b45..172c958334 100644 --- a/crates/perry-codegen/src/expr/agent_ptr.rs +++ b/crates/perry-codegen/src/expr/agent_ptr.rs @@ -140,7 +140,7 @@ pub(crate) fn emit_agent_ptr_or(ctx: &mut FnCtx<'_>, slot: usize, absent: &str) /// The address of the slot `slot_off` bytes into this thread's block, for the /// two forms that name the block through the thread pointer (module docs). -fn emit_slot_addr(ctx: &mut FnCtx<'_>, access: AgentPtrAccess, slot_off: &str) -> String { +pub(crate) fn emit_slot_addr(ctx: &mut FnCtx<'_>, access: AgentPtrAccess, slot_off: &str) -> String { let blk = ctx.block(); let block = match access { AgentPtrAccess::InitialExec => format!("@{AGENT_PTRS_SYMBOL}"), diff --git a/crates/perry-codegen/src/expr/stack_guard.rs b/crates/perry-codegen/src/expr/stack_guard.rs index c8ac09be0a..855e065a3a 100644 --- a/crates/perry-codegen/src/expr/stack_guard.rs +++ b/crates/perry-codegen/src/expr/stack_guard.rs @@ -21,7 +21,7 @@ //! `llvm.frameaddress`, which would force a frame pointer and stop a //! frameless (shrink-wrapped) entry from staying frameless. -use super::agent_ptr::{agent_ptr_access, AgentPtrAccess, AGENT_PTRS_SYMBOL}; +use super::agent_ptr::{agent_ptr_access, AgentPtrAccess}; use super::FnCtx; use crate::types::{I64, I8, PTR}; @@ -65,11 +65,14 @@ fn emit_check(ctx: &mut FnCtx<'_>) { let ok_idx; let limit = match agent_ptr_access(ctx) { AgentPtrAccess::Call => return, - AgentPtrAccess::InitialExec => { + // The runtime publishes no stack limit on Windows (`stack_bounds` is + // `None` there), so the slot stays 0 and a check could never fire: + // emit none, as before the block was reachable inline on Windows. + AgentPtrAccess::WindowsTeb => return, + access @ AgentPtrAccess::InitialExec => { ok_idx = ctx.new_block("stack_guard.ok"); - let blk = ctx.block(); - let at = blk.gep(I8, &format!("@{AGENT_PTRS_SYMBOL}"), &[(I64, &slot_off)]); - blk.load(PTR, &at) + let at = super::agent_ptr::emit_slot_addr(ctx, access, &slot_off); + ctx.block().load(PTR, &at) } AgentPtrAccess::AppleTsd => { let lookup = super::hot_tls::emit_hot_tls_lookup(ctx, "stack_guard"); diff --git a/scripts/shape_descriptor_census.py b/scripts/shape_descriptor_census.py index cfd62fc399..271c204a1a 100644 --- a/scripts/shape_descriptor_census.py +++ b/scripts/shape_descriptor_census.py @@ -407,13 +407,16 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: # ordinary GC slot, and a budgeted dirty scan can hold that address # across mutator resumptions that insert descriptors — so a record's # address must never move for its lifetime. Chunks are individually - # boxed and never reallocated; only the directory of chunk pointers - # grows. Putting records into one flat `Vec` (or back into a rehashing - # bucket) moves them under the collector's feet. + # allocated (a `Slot`: an owned pointer, or the shared all-empty chunk + # that is never written) and never reallocated; only the directory of + # chunk pointers grows. Putting records into one flat `Vec` (or back + # into a rehashing bucket) moves them under the collector's feet. (r"slab\s*:\s*(?:std::cell::)?UnsafeCell\s*<\s*ShapeSlab\s*>", "by-id descriptor slab with stable record addresses"), - (r"type\s+Chunk\s*=\s*Box\s*<\s*\[\s*UnsafeCell\s*<\s*ShapeRecord\s*>\s*;\s*CHUNK_LEN\s*\]\s*>", "slab chunks individually boxed, never reallocated"), - (r"type\s+Page\s*=\s*Box\s*<\s*\[\s*Option\s*<\s*Chunk\s*>\s*;\s*PAGE_LEN\s*\]\s*>", "slab directory pages hold chunk pointers, not records"), - (r"pages\s*:\s*Vec\s*<\s*Option\s*<\s*Page\s*>\s*>", "slab directory is a vector of page pointers"), + (r"struct\s+Slot\s*<\s*T\s*>\s*\(\s*std::ptr::NonNull\s*<\s*T\s*>\s*\)", "a slab slot is one pointer to its own allocation"), + (r"type\s+ChunkCells\s*=\s*\[\s*UnsafeCell\s*<\s*ShapeRecord\s*>\s*;\s*CHUNK_LEN\s*\]", "slab chunks individually allocated, never reallocated"), + (r"type\s+PageSlots\s*=\s*\[\s*Slot\s*<\s*ChunkCells\s*>\s*;\s*PAGE_LEN\s*\]", "slab pages hold chunk pointers, not records"), + (r"type\s+Page\s*=\s*Slot\s*<\s*PageSlots\s*>", "slab directory entries are page pointers"), + (r"pages\s*:\s*Vec\s*<\s*Page\s*>", "slab directory is a vector of page pointers"), # `keys` must stay the FIRST field of the `#[repr(C)]` record: the # record address IS the rewritable keys slot (`keys_slot`). (r"#\[repr\(C\)\]\s*(?:#\[[^\]]*\]\s*)*pub\(crate\)\s+struct\s+ShapeRecord\s*\{\s*(?://[^\n]*\n\s*)*pub\(super\)\s+keys\s*:\s*u64", "slab record is repr(C) with the keys word first"), @@ -1124,8 +1127,8 @@ def run_sabotage_selftests(sources: dict[str, str], baseline: dict[str, object]) store_path = "crates/perry-runtime/src/object/shapes_store.rs" flat_slab = dict(sources) flat_slab[store_path] = flat_slab[store_path].replace( - "type Chunk = Box<[UnsafeCell; CHUNK_LEN]>;", - "type Chunk = Vec>;", + "type ChunkCells = [UnsafeCell; CHUNK_LEN];", + "type ChunkCells = Vec>;", 1, ) expect_rejected( From 617b834d9b02431b00d34fd4d372a36d0a5a35cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 09:37:52 +0000 Subject: [PATCH 13/16] rustfmt; say that an in-place rep deprecation leaves POSBOUND as it is --- crates/perry-codegen/src/expr/agent_ptr.rs | 6 +++++- crates/perry-runtime/src/object/shapes_store.rs | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/perry-codegen/src/expr/agent_ptr.rs b/crates/perry-codegen/src/expr/agent_ptr.rs index 172c958334..9cf58298bf 100644 --- a/crates/perry-codegen/src/expr/agent_ptr.rs +++ b/crates/perry-codegen/src/expr/agent_ptr.rs @@ -140,7 +140,11 @@ pub(crate) fn emit_agent_ptr_or(ctx: &mut FnCtx<'_>, slot: usize, absent: &str) /// The address of the slot `slot_off` bytes into this thread's block, for the /// two forms that name the block through the thread pointer (module docs). -pub(crate) fn emit_slot_addr(ctx: &mut FnCtx<'_>, access: AgentPtrAccess, slot_off: &str) -> String { +pub(crate) fn emit_slot_addr( + ctx: &mut FnCtx<'_>, + access: AgentPtrAccess, + slot_off: &str, +) -> String { let blk = ctx.block(); let block = match access { AgentPtrAccess::InitialExec => format!("@{AGENT_PTRS_SYMBOL}"), diff --git a/crates/perry-runtime/src/object/shapes_store.rs b/crates/perry-runtime/src/object/shapes_store.rs index 6ddde930d6..a119cc1058 100644 --- a/crates/perry-runtime/src/object/shapes_store.rs +++ b/crates/perry-runtime/src/object/shapes_store.rs @@ -340,7 +340,9 @@ impl ShapeRecord { /// which is itself a valid NaN-boxed value, so key position `i` is inline /// slot `i` whatever the slot's representation. A shape minted with a /// non-`Any` rep is its own record and gets its own bound from these - /// facts at construction and slab insert, like every other shape. + /// facts at construction and slab insert, like every other shape, and + /// deprecating a lane in place (`deprecate_rep_slot`) leaves the bound + /// as it is, correctly. #[inline] pub(super) fn positional_by_facts(&self) -> bool { self.object_kind() == ShapeObjectKind::Ordinary From f51aea97b2e798574467b8201bd3ab4c8e9e1c3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 12:08:29 +0000 Subject: [PATCH 14/16] shapes tests: the position-bound rep test passes no static id request --- crates/perry-runtime/src/object/shapes_tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/perry-runtime/src/object/shapes_tests.rs b/crates/perry-runtime/src/object/shapes_tests.rs index 8ff65032ff..ae31e69bfc 100644 --- a/crates/perry-runtime/src/object/shapes_tests.rs +++ b/crates/perry-runtime/src/object/shapes_tests.rs @@ -1494,6 +1494,7 @@ mod field_rep_identity_tests { PROTO, 0, rep, + None, )) }; let any = mint_keys(REP_ANY); From 159470f72a69a6b2b38619e1083ff0c00e74833c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 15:20:20 +0000 Subject: [PATCH 15/16] lint: thread-exit verdicts for the shared-empty shape statics; drop a now-safe unsafe in the posbound test --- crates/perry-runtime/src/object/shapes_tests.rs | 2 +- scripts/thread_exit_address_globals.json | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/object/shapes_tests.rs b/crates/perry-runtime/src/object/shapes_tests.rs index 3228dc8fe0..aed11794ba 100644 --- a/crates/perry-runtime/src/object/shapes_tests.rs +++ b/crates/perry-runtime/src/object/shapes_tests.rs @@ -1488,7 +1488,7 @@ mod field_rep_identity_tests { #[test] fn a_rep_typed_shape_carries_its_own_position_bound() { let _lock = crate::gc::global_side_table_test_lock(); - let keys = unsafe { crate::array::js_array_alloc_with_length(3) }; + let keys = crate::array::js_array_alloc_with_length(3); let mint_keys = |rep: u64| { publish_shape_result(shape_descriptor_ensure_with_rep( keys, diff --git a/scripts/thread_exit_address_globals.json b/scripts/thread_exit_address_globals.json index ad737206f1..476d10ec10 100644 --- a/scripts/thread_exit_address_globals.json +++ b/scripts/thread_exit_address_globals.json @@ -4083,6 +4083,16 @@ "verdict": "per_thread", "why": "`#[thread_local]` static (not `thread_local!`, so the megamorphic read reaches it with one thread-pointer-relative load): each thread mirrors ITS OWN ShapeSlab page directory (a Rust-heap Vec pointer and length, never an arena address), republished on every change to `pages` and cleared by ShapeSlab::drop before the Vec is freed; const-initialised to (null, 0) with no drop glue." }, + { + "file": "crates/perry-runtime/src/object/shapes_store.rs", + "names": [ + "EMPTY_CHUNK", + "EMPTY_PAGE", + "PERRY_EMPTY_SHAPE_DIR" + ], + "verdict": "no_heap_address", + "why": "Shared all-empty shape-slab structure (a chunk of EMPTY records, a page of pointers to that chunk, an empty page directory). Every pointer inside is the address of another immutable static in the program image, never an arena or heap address; nothing ever writes them, so an exited thread's Arena::drop cannot leave them dangling." + }, { "file": "crates/perry-runtime/src/object/static_shapes.rs", "names": [ From 776412c4e0e14575f7c21cdf30ec944a0d2dca84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Tue, 29 Sep 2026 16:41:02 +0000 Subject: [PATCH 16/16] shapes tests: the seeded-literal confirm test follows the dir-passing confirm and asserts POSBOUND --- crates/perry-runtime/src/agent_ptrs.rs | 1 - .../src/object/static_shapes_tests.rs | 28 ++++++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/crates/perry-runtime/src/agent_ptrs.rs b/crates/perry-runtime/src/agent_ptrs.rs index 25f80a1c9f..98be52a105 100644 --- a/crates/perry-runtime/src/agent_ptrs.rs +++ b/crates/perry-runtime/src/agent_ptrs.rs @@ -114,4 +114,3 @@ pub extern "C" fn perry_shape_dir_cell() -> *const u8 { publish(AGENT_PTR_SHAPE_DIR, dir); dir } - diff --git a/crates/perry-runtime/src/object/static_shapes_tests.rs b/crates/perry-runtime/src/object/static_shapes_tests.rs index 1f76a671df..8be10eef27 100644 --- a/crates/perry-runtime/src/object/static_shapes_tests.rs +++ b/crates/perry-runtime/src/object/static_shapes_tests.rs @@ -151,6 +151,17 @@ fn pool_atom(text: &str) -> *const crate::StringHeader { crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) } +/// The megamorphic confirm as the miss entry asks it: this agent's directory, +/// the site's key as NaN-boxed bits. +unsafe fn confirmed(id: u32, key: *const crate::StringHeader, guess: usize) -> bool { + shapes::slot_guess_confirmed( + shapes::ordinary_dir_addr(), + id, + crate::JSValue::string_ptr(key as *mut _).bits(), + guess, + ) +} + /// A seeded literal shape answers the megamorphic read's slot-guess confirm /// (`shapes::slot_guess_confirmed`: the position bound, then ONE pointer /// compare of the listed key against the site's key atom) exactly as a shape @@ -181,17 +192,20 @@ fn a_seeded_literal_shape_answers_the_megamorphic_confirm_like_a_minted_one() { (minted, a, 0, "minted `a` at 0"), (minted, k1, 1, "minted `k1` at 1"), ] { - assert_eq!( - shapes::test_positional_of_id(id), - Some((true, true)), - "{what}: the record must answer by position" + // POSBOUND is a fact of the record: a seeded shape (built by the + // slab insert, like every other) carries it, nonzero, equal to its + // definition. + let (stored, by_facts) = shapes::test_positional_of_id(id).expect("a record"); + assert!( + stored > 0 && stored == by_facts, + "{what}: the record must answer by position (POSBOUND {stored}, by facts {by_facts})" ); assert!( - unsafe { shapes::slot_guess_confirmed(id, key, guess) }, + unsafe { confirmed(id, key, guess) }, "{what}: the megamorphic confirm must accept the key atom" ); } // And refutes a wrong guess or another key. - assert!(!unsafe { shapes::slot_guess_confirmed(seeded, a, 1) }); - assert!(!unsafe { shapes::slot_guess_confirmed(seeded, k1, 1) }); + assert!(!unsafe { confirmed(seeded, a, 1) }); + assert!(!unsafe { confirmed(seeded, k1, 1) }); }