From 056abd7a751819d7f42bb3e07a4a1a50c80db558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 23 Sep 2026 18:13:42 +0000 Subject: [PATCH 1/8] perf(runtime): one string per property-key text, so a key confirm is a pointer compare A canonical key list stored whichever string its first grower passed, and a read site holds its module's pooled literal: two objects with the same bytes. Every key match against a shape's list therefore fell through to a byte compare, including the megamorphic read's confirm of its slot guess. Pool literals of at most 64 bytes are now minted as ATOMS at module init (js_string_pool_atom): the one string object for that text in the agent, shared by every module's pool. The intern cache's miss paths hand out the atom for its text, and canonical lists write the atom of every key they store (Appended::atomized on extend_slot's write paths and canonicalize's copy). The trie still validates edges by bytes, so which object a list holds never changes which node a probe reaches. The atom table is per agent, bounded by program text, strong (every atom is also a registered pool handle's value) and rewritten on move by the intern table root scanner. A pointer match proves equal text; a mismatch proves nothing (a list written before its atom existed), so every consumer keeps its byte fallback. The megamorphic shape answer now scans for identity before it compares any bytes. --- .../perry-codegen/src/codegen/string_pool.rs | 33 ++- crates/perry-codegen/src/runtime_decls/mod.rs | 4 + .../src/gc/tests/canonical_keys_holders.rs | 136 +++++++++ .../src/object/canonical_keys.rs | 47 ++++ .../object/field_get_set/ic_miss/ic_slow.rs | 166 +++++++++++ crates/perry-runtime/src/object/shapes.rs | 30 +- crates/perry-runtime/src/string/intern.rs | 257 ++++++++++++++++++ crates/perry-runtime/src/string/mod.rs | 4 + 8 files changed, 661 insertions(+), 16 deletions(-) diff --git a/crates/perry-codegen/src/codegen/string_pool.rs b/crates/perry-codegen/src/codegen/string_pool.rs index 079c4ce0ec..5c82c68741 100644 --- a/crates/perry-codegen/src/codegen/string_pool.rs +++ b/crates/perry-codegen/src/codegen/string_pool.rs @@ -96,6 +96,13 @@ impl<'a> InitChunker<'a> { } } +/// Pool literals at most this long are minted as ATOMS. **Must equal +/// `INTERN_MAX_BYTE_LEN` in `perry-runtime/src/string/intern.rs`**, the +/// longest key the runtime interns; `js_string_pool_atom` falls back to a +/// plain allocation past it, so a mismatch costs only the atom, never +/// correctness. +pub(crate) const POOL_ATOM_MAX_BYTE_LEN: usize = 64; + /// Emit the string pool into the module: byte-array constants, handle /// globals, and the `__perry_init_strings_` function that /// allocates + NaN-boxes + GC-roots each handle exactly once at startup. @@ -446,12 +453,30 @@ pub(super) fn emit_string_pool( let bytes_ref = format!("@{}", entry.bytes_global); let handle_ref = format!("@{}", entry.handle_global); let len_str = entry.byte_len.to_string(); - let from_bytes_fn = if entry.is_wtf8 { - "js_string_from_wtf8_bytes" + // A literal short enough to be a property key becomes its text's ATOM + // (`js_string_pool_atom`): one string object per key text for the + // whole agent, shared by every module's pool, every canonical shape + // key list and every intern hit — so a read site's key and the + // receiver's shape key compare by pointer (S3b). Longer literals, and + // WTF-8 ones (lone surrogates: never an identifier key), keep the + // plain allocation. + let atomize = + !entry.is_wtf8 && entry.byte_len > 0 && entry.byte_len <= POOL_ATOM_MAX_BYTE_LEN; + let handle = if atomize { + let hash = crate::nanbox::i64_literal(entry.dispatch_hash); + blk.call( + I64, + "js_string_pool_atom", + &[(PTR, &bytes_ref), (I32, &len_str), (I64, &hash), (I32, "0")], + ) } else { - "js_string_from_bytes" + let from_bytes_fn = if entry.is_wtf8 { + "js_string_from_wtf8_bytes" + } else { + "js_string_from_bytes" + }; + blk.call(I64, from_bytes_fn, &[(PTR, &bytes_ref), (I32, &len_str)]) }; - let handle = blk.call(I64, from_bytes_fn, &[(PTR, &bytes_ref), (I32, &len_str)]); let nanboxed = blk.call(DOUBLE, "js_nanbox_string", &[(I64, &handle)]); // Plain store, no remembered-set write barrier: the handle slot is // registered as a permanent global root on the very next line (always diff --git a/crates/perry-codegen/src/runtime_decls/mod.rs b/crates/perry-codegen/src/runtime_decls/mod.rs index 72d544d6fc..9d9de7474a 100644 --- a/crates/perry-codegen/src/runtime_decls/mod.rs +++ b/crates/perry-codegen/src/runtime_decls/mod.rs @@ -159,6 +159,10 @@ pub fn declare_phase1(module: &mut LlModule) { // Strings (enough to produce string literals for later phases). module.declare_function("js_string_from_bytes", I64, &[PTR, I32]); module.declare_function("js_string_from_wtf8_bytes", I64, &[PTR, I32]); + // S3b: a pooled literal short enough to be a key is minted as the atom of + // its text (`string/intern.rs::js_string_pool_atom`): bytes, len, FNV-1a + // hash, is_wtf8. + module.declare_function("js_string_pool_atom", I64, &[PTR, I32, I64, I32]); // Type checks. module.declare_function("js_is_truthy", I32, &[DOUBLE]); diff --git a/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs b/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs index e1a3e51b23..c70c672600 100644 --- a/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs +++ b/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs @@ -657,3 +657,139 @@ fn a_dead_weak_keys_entry_is_dropped_before_its_storage_is_reused() { .remove(&DEAD_MEMO_CLASS_ID); canonical_keys::reset_for_test(); } + +// ------------------------------------------------------- S3b: key atoms + +fn atom_hash(text: &[u8]) -> u64 { + crate::object::key_bytes_hash(text.as_ptr(), text.len()) +} + +fn atom_bits(atom: usize) -> u64 { + crate::value::js_nanbox_string(atom as i64).to_bits() +} + +/// S3b: a key text has ONE string object in an agent — its atom — and every +/// canonical list written after the atom exists holds it. The atom table holds +/// its strings strongly and REWRITES them on a move (the intern-table root +/// scanner), so after a moving minor the table, the list, a fresh intern and a +/// fresh pool mint all name the atom at its NEW address, and none names the +/// address it moved away from. +#[test] +fn an_atom_and_the_lists_holding_it_follow_a_moving_minor() { + let _guard = CopyingNurseryTestGuard::new(0); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + register_object_model_scanners(); + gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut); + canonical_keys::reset_for_test(); + let scope = RuntimeHandleScope::new(); + let text = b"atom_mv_kind"; + let hash = atom_hash(text); + unsafe { + let atom = + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize; + assert!( + crate::arena::pointer_in_nursery(atom), + "premise: the atom is young, so a minor can move it" + ); + // A receiver grows the key through a DIFFERENT string with the text. + let copy = nursery_key("atom_mv_kind"); + assert_ne!(copy as usize, atom, "premise: two string objects, one text"); + let o = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0)); + set(o, nursery_key("atom_mv_a"), 1.0); + set(o, copy, 2.0); + let list = keys_of(o); + assert_eq!( + crate::array::js_array_get(list, 1).bits(), + atom_bits(atom), + "INVARIANT: the written list holds the atom, not the grower's copy" + ); + + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert!( + trace.copying_nursery.copied_objects > 0, + "premise: the minor copied" + ); + let moved = crate::string::atom_lookup(text, hash).expect("the atom survives") as usize; + assert_ne!(moved, atom, "premise: the minor moved the atom"); + let header = crate::value::addr_class::try_read_tracked_gc_header(moved) + .expect("the moved atom is a tracked cell"); + assert_eq!( + (*header.as_ptr()).gc_flags & GC_FLAG_FORWARDED, + 0, + "INVARIANT: the table names the live copy, not a forwarding header" + ); + assert_eq!( + crate::array::js_array_get(keys_of(o), 1).bits(), + atom_bits(moved), + "INVARIANT: the list follows its atom through the move" + ); + // After the move: interning another copy, and minting again, both + // return the moved atom — never a third string. + assert_eq!( + crate::string::js_string_intern(nursery_key("atom_mv_kind"), hash) as usize, + moved, + "a copy interns to the moved atom" + ); + assert_eq!( + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize, + moved, + "a second mint finds the moved atom" + ); + // A list written after the move holds the moved atom. + let o2 = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0)); + set(o2, nursery_key("atom_mv_b"), 1.0); + set(o2, nursery_key("atom_mv_kind"), 2.0); + assert_eq!( + crate::array::js_array_get(keys_of(o2), 1).bits(), + atom_bits(moved), + "INVARIANT: a list written after the move holds the moved atom" + ); + } +} + +/// S3b, a collection DURING the write: the canonical backing allocation that +/// publishes a list holding an atom is itself the collection point, and moves +/// the atom. The published list must hold the atom's LIVE address. +#[test] +fn a_list_written_while_its_atom_moves_holds_the_live_atom() { + let _guard = CopyingNurseryTestGuard::new(0); + let _pacing = crate::gc::policy::force_alloc_point_minor_pacing(); + let _scan = NoConservativeScan::new(); + let trigger = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + register_object_model_scanners(); + gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut); + canonical_keys::reset_for_test(); + let text = b"atom_during_kind"; + let hash = atom_hash(text); + unsafe { + let atom = + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize; + assert!( + crate::arena::pointer_in_nursery(atom), + "premise: the atom is young" + ); + let scope = RuntimeHandleScope::new(); + let copy = scope.root_string_ptr(nursery_key("atom_during_kind")); + // The empty list plus this key is a fresh backing: its allocation is + // the collection point. + arm_collection_on_next_block(&trigger); + let list = copy.with_const_ptr(|k: *const crate::StringHeader| { + canonical_keys::extend_key( + &SharedLayout::shape_cache_entry(), + canonical_keys::CanonicalKeys::EMPTY, + k, + ) + }); + let live = crate::string::atom_lookup(text, hash).expect("the atom survives") as usize; + assert_ne!( + live, atom, + "premise: the backing allocation did not move the atom, so this run proved \ + nothing. Check the trigger arming." + ); + assert_eq!( + crate::array::js_array_get(list.as_ptr(), 0).bits(), + atom_bits(live), + "INVARIANT: the list holds the atom's live address" + ); + } +} diff --git a/crates/perry-runtime/src/object/canonical_keys.rs b/crates/perry-runtime/src/object/canonical_keys.rs index c3f8a6c164..00704f8880 100644 --- a/crates/perry-runtime/src/object/canonical_keys.rs +++ b/crates/perry-runtime/src/object/canonical_keys.rs @@ -676,6 +676,37 @@ impl Appended { } } + /// The same slot, spelled with its text's ATOM when one exists — the one + /// string a read site's pooled key also is (`string::intern::AtomTable`). + /// + /// Applied to every key this module WRITES into a list, and nowhere else: + /// the trie validates edges by bytes, so which string object a list holds + /// never changes which node a probe reaches, only whether a later key + /// compare against the list can stop at pointer equality. Heap strings + /// only — an SSO slot stays an SSO slot, so `is_pointer` (and with it the + /// backing's all-pointer layout) is unchanged by the substitution. + /// + /// `h` is this slot's `edge_hash`, which for a string IS the FNV-1a hash of + /// its bytes — the atom table's hash. + /// + /// # Safety + /// The operand is live. + unsafe fn atomized(self, h: u64) -> Self { + match self { + Appended::Key(key) if !key.is_null() => match crate::string::atom_for_key(key, h) { + Some(atom) => Appended::Key(atom), + None => self, + }, + Appended::Slot(v) if v.is_string() => { + match crate::string::atom_for_key(v.as_string_ptr(), h) { + Some(atom) => Appended::Slot(JSValue::string_ptr(atom as *mut StringHeader)), + None => self, + } + } + _ => self, + } + } + /// Is the appended slot a heap string POINTER? An SSO short string and a /// tombstone are not, and either one costs the child its all-pointer /// layout — see `Node::all_ptr`. @@ -789,6 +820,8 @@ pub(crate) unsafe fn extend_slot( if let Some(hit) = probe(parent, parent_len, appended, entry, h) { return hit; } + // A new list is about to be WRITTEN: it holds the atom of this key's text. + let appended = appended.atomized(appended.slot_hash()); // Whether the child's slots are all heap string pointers is the parent's // answer AND this slot's, so it is read before the allocation and never @@ -1180,6 +1213,20 @@ pub(crate) unsafe fn canonicalize( // GC_STORE_AUDIT(INIT): fresh is unpublished; publish length only after // all elements are initialized, with no intervening GC allocation. std::ptr::copy_nonoverlapping(slots, dst, len as usize); + // Every heap key of the new list is its text's atom where one exists (see + // `Appended::atomized`). Heap string to heap string, so `all_ptr` holds. + for i in 0..len as usize { + let slot = Appended::Slot(JSValue::from_bits((*dst.add(i)).to_bits())); + if let Appended::Slot(v) = slot { + if v.is_string() { + if let Appended::Slot(atom) = slot.atomized(slot.slot_hash()) { + // GC_STORE_AUDIT(INIT): `fresh` is unpublished; its length + // is set on the next line, after the last slot is written. + *dst.add(i) = f64::from_bits(atom.bits()); + } + } + } + } if with_attrs { let attrs = crate::object::key_attrs::keys_attrs(fresh); crate::object::key_attrs::copy_entries(keys, 0, attrs, len); diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 50e821cb2d..09f0fc5b8a 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -525,6 +525,172 @@ mod tests { } } + /// `megamorphic_receivers` with a chosen text for the read key (slot 2), + /// each receiver growing that key through its OWN freshly allocated string + /// — never an atom — so which string a shape's list ends up holding is + /// decided by the list writer, not by the test. + fn megamorphic_receivers_keyed<'s>( + scope: &'s crate::gc::RuntimeHandleScope, + n: usize, + read_key: &[u8], + extra_prefix: &str, + ) -> Vec> { + let mut out = Vec::new(); + for i in 0..n { + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + for (k, v) in [ + (&b"pos"[..], 1.0), + (&b"end"[..], 2.0), + (read_key, 100.0 + i as f64), + ] { + let key = scope.root_string_ptr(key_of(k)); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + let extra = format!("{extra_prefix}{i}"); + let key = scope.root_string_ptr(key_of(extra.as_bytes())); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, 7.0)) + }); + out.push(obj); + } + out + } + + fn atom_of(text: &[u8]) -> *mut crate::StringHeader { + let hash = crate::object::key_bytes_hash(text.as_ptr(), text.len()); + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) + } + + fn answered() -> u64 { + crate::object::shapes::SHAPE_ANSWERED_READS.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Read `key` at one latched site over `objs` until it latches, then once + /// more; every read must return `expect(i)`. Returns the reads the + /// receivers' SHAPES answered on the latched pass. + fn latched_pass( + objs: &[crate::gc::RuntimeHandle<'_>], + key: &crate::gc::RuntimeHandle<'_>, + expect: impl Fn(usize) -> u64, + ) -> u64 { + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let read = |o: &crate::gc::RuntimeHandle<'_>, slot: &mut PicCacheSlot| { + o.with_mut_ptr(|p: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(p), k, slot, &packed)) + }) + }; + for round in 0..4 { + for (i, o) in objs.iter().enumerate() { + assert_eq!( + read(o, &mut slot).to_bits(), + expect(i), + "round {round} receiver {i}" + ); + } + } + assert!( + cache[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] < 0, + "premise: the site latched megamorphic" + ); + let before = answered(); + for (i, o) in objs.iter().enumerate() { + assert_eq!( + read(o, &mut slot).to_bits(), + expect(i), + "latched read, receiver {i}" + ); + } + answered() - before + } + + /// S3b: one key text reaching the runtime as TWO string objects — each + /// receiver grows it through its own fresh copy — is still ONE string in + /// every shape's key list: the text's atom, the same object a read site's + /// pooled key is. So the site matches by pointer; and a site holding yet + /// another copy (not the atom) still gets the right answer by bytes. + #[test] + fn a_key_text_in_two_string_objects_is_one_atom_in_every_shape() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_two_objects_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let copy = scope.root_string_ptr(key_of(text)); + let addr = |h: &crate::gc::RuntimeHandle<'_>| { + h.with_const_ptr(|p: *const crate::StringHeader| p as usize) + }; + assert_ne!( + addr(&atom), + addr(©), + "premise: two string objects, one text" + ); + assert!( + unsafe { crate::string::is_atom_for_test(atom.with_const_ptr(|p| p)) }, + "premise: the pool mint made an atom" + ); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_two_x"); + let atom_bits = crate::value::js_nanbox_string(addr(&atom) as i64).to_bits(); + for (i, o) in objs.iter().enumerate() { + let stored = o.with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::object_keys(p).get(2).bits() + }); + assert_eq!( + stored, atom_bits, + "INVARIANT: receiver {i}'s shape holds the atom, not the copy it grew with" + ); + } + // A computed key with the text interns TO the atom. + let interned = copy.with_const_ptr(|p: *const crate::StringHeader| { + crate::string::js_string_intern( + p, + crate::object::key_bytes_hash(text.as_ptr(), text.len()), + ) as usize + }); + assert_eq!(interned, addr(&atom), "a copy interns to the atom"); + // The site holds the atom: every latched read is the receiver's own + // value, answered by its shape. + let by_atom = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + by_atom >= 47, + "the shape answers a pointer-equal key: {by_atom}" + ); + // A site holding a different string object with the same text (not + // the atom): the answer is the same, found by bytes. + let fresh = scope.root_string_ptr(key_of(text)); + let by_bytes = latched_pass(&objs, &fresh, |i| (100.0 + i as f64).to_bits()); + assert!( + by_bytes >= 47, + "a non-atom key text still matches by bytes: {by_bytes}" + ); + } + + /// A list written BEFORE its key's atom existed holds another string. A + /// pointer mismatch is not an answer: the shape still answers by bytes. + #[test] + fn a_list_written_before_the_atom_existed_still_answers() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_pre_atom_kind"; + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_pre_x"); + // The lists hold the string the grow path interned; a collision + // evicts it from the cache before the atom is minted, so the atom is + // a different object. + crate::string::test_evict_interned(text); + let atom = scope.root_string_ptr(atom_of(text)); + let atom_bits = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let stored = objs[0].with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::object_keys(p).get(2).bits() + }); + assert_ne!(stored, atom_bits, "premise: the list predates the atom"); + let n = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!(n >= 47, "the shape answers by bytes: {n}"); + } + /// A plain own data read that has never primed: the entry must fall all the /// way through to the miss handler, answer the field, and leave the site /// primed exactly as the old `js_object_get_field_ic_miss_packed` edge did. diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 2a62a4a1e9..4f87d4d02e 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -257,11 +257,12 @@ impl ShapeRecordRef { /// SHAPE's key count, never the backing's length (#10969: one backing per /// growth chain). /// - /// A stored key matches the site's key by identity, or else by (byte - /// length, bytes): a canonical list holds the string its first grower - /// passed, which is usually NOT the read site's pooled literal. The - /// site's slot guess is tried first. Allocation-free, never calls user - /// code. + /// Key compares go identity first: canonical lists hold their text's ATOM + /// (`string::intern::AtomTable`), which is also what a read site's pooled + /// key is, so the site's guess, and then any position, matches by one + /// pointer compare. A byte pass remains for a list written before its + /// atom existed (and for SSO slots) — a pointer MISmatch proves nothing. + /// Allocation-free, never calls user code. #[inline] pub(crate) unsafe fn inline_slot_of_key( self, @@ -284,11 +285,16 @@ impl ShapeRecordRef { let bound = len .min(r.logical_key_count as usize) .min(r.live_inline_slot_count as usize); + let heap_bits = crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits(); // The site's slot guess first: the receiver's shape confirms it. - if hint < bound && stored_key_matches(key, (*slots.add(hint)).to_bits()) { + if hint < bound && (*slots.add(hint)).to_bits() == heap_bits { return Some(hint); } - (0..bound).find(|&i| i != hint && stored_key_matches(key, (*slots.add(i)).to_bits())) + // Identity over the whole list before any byte is compared. + if let Some(i) = (0..bound).find(|&i| (*slots.add(i)).to_bits() == heap_bits) { + return Some(i); + } + (0..bound).find(|&i| stored_key_matches(key, (*slots.add(i)).to_bits())) } /// The SPILL position at which this shape stores `key` as an own DATA @@ -333,11 +339,11 @@ impl ShapeRecordRef { } } -/// Does the key-list entry `bits` name `key`? A canonical list holds heap -/// strings of its own (NOT the site's pooled key — measured: every stored key -/// of a literal-born shape is a distinct heap string) or SSO immediates, so a -/// stored key matches by identity, by SSO identity, or by (byte length, -/// bytes). +/// Does the key-list entry `bits` name `key`? A canonical list holds its +/// text's ATOM where one exists (the site's pooled key), but must not be +/// assumed to: a list written before its atom existed holds another heap +/// string, and a slot may be an SSO immediate. So a stored key matches by +/// identity, by SSO identity, or by (byte length, bytes). #[inline] unsafe fn stored_key_matches(key: *const crate::StringHeader, bits: u64) -> bool { if bits == crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits() { diff --git a/crates/perry-runtime/src/string/intern.rs b/crates/perry-runtime/src/string/intern.rs index da9d2ef971..5083c2e22c 100644 --- a/crates/perry-runtime/src/string/intern.rs +++ b/crates/perry-runtime/src/string/intern.rs @@ -36,6 +36,229 @@ crate::perry_thread_local! { std::cell::UnsafeCell::new(crate::zeroed_cache::new_zeroed_cache(INTERN_TABLE_SIZE)); } +/// Property-key ATOMS: exactly one string object per key text among the +/// program's pooled literals. +/// +/// The direct-mapped table above is a CACHE — a collision evicts — so two +/// strings with one text can both be "interned" at different moments, and a +/// key compare can never conclude anything from pointer equality alone. That +/// is why every runtime key match, and the megamorphic read's confirm against +/// the receiver's key list, paid a byte compare: a canonical key list held the +/// string its first grower happened to pass, and a read site holds its +/// module's pooled literal, and the two were different objects with the same +/// bytes. +/// +/// An atom is the one string for its text in this agent, for the agent's +/// lifetime. Atoms are minted only from the compiled program's string pools +/// (`js_string_pool_atom`, at module init), so the table is bounded by the +/// program TEXT, never by runtime data, and it can hold its strings strongly +/// without a death prune: every atom is also the value of a registered pool +/// handle, so the table keeps nothing alive that was not already live. The +/// collector rewrites the entries on move through the intern-table root +/// scanner (`scan_intern_table_roots_mut`), exactly like the cache's. +/// +/// Two funnels consult it: the intern cache's miss paths (so a computed key +/// with an atom's text interns TO the atom), and canonical key lists when they +/// write a key (`canonical_keys::Appended::atomized`), so a read site's pooled +/// key and the receiver's shape key are one pointer. +/// +/// The table never decides an answer. A pointer match proves equal text; a +/// pointer MISmatch proves nothing (a list written before its atom existed +/// holds another string), so every consumer still falls back to bytes on a +/// mismatch. +pub(crate) struct AtomTable { + /// Open addressing, linear probe, power-of-two capacity; `string_ptr == 0` + /// is an empty slot. Entries are never removed. + slots: Vec, + len: usize, +} + +impl AtomTable { + const fn new() -> Self { + AtomTable { + slots: Vec::new(), + len: 0, + } + } + + /// The atom for `bytes` (whose FNV-1a hash is `hash`), if any. + /// + /// # Safety + /// Every tabled pointer is a live string (the collector keeps them so). + unsafe fn lookup(&self, bytes: &[u8], hash: u64) -> Option<*const StringHeader> { + self.lookup_from(0, bytes, hash) + } + + /// [`Self::lookup`] for a string that may itself BE the atom (`from`, or + /// 0): a tabled pointer equal to it answers without comparing bytes, which + /// is the common case for a key a list already holds. + /// + /// # Safety + /// As [`Self::lookup`]. + unsafe fn lookup_from( + &self, + from: usize, + bytes: &[u8], + hash: u64, + ) -> Option<*const StringHeader> { + if self.slots.is_empty() { + return None; + } + let mask = self.slots.len() - 1; + let mut i = (hash as usize) & mask; + loop { + let entry = &self.slots[i]; + if entry.string_ptr == 0 { + return None; + } + if entry.hash == hash { + let existing = entry.string_ptr as *const StringHeader; + if entry.string_ptr == from { + return Some(existing); + } + if (*existing).byte_len as usize == bytes.len() + && std::slice::from_raw_parts(string_data(existing), bytes.len()) == bytes + { + return Some(existing); + } + } + i = (i + 1) & mask; + } + } + + /// Table `atom` under `hash`. The caller has proved no atom exists for + /// its text. Rust-heap only: never allocates on the GC heap. + fn insert(&mut self, hash: u64, atom: *const StringHeader) { + if (self.len + 1) * 4 > self.slots.len() * 3 { + let cap = (self.slots.len() * 2).max(256); + let old = std::mem::replace( + &mut self.slots, + vec![ + InternEntry { + hash: 0, + string_ptr: 0, + }; + cap + ], + ); + for entry in old.into_iter().filter(|e| e.string_ptr != 0) { + self.place(entry); + } + } + self.place(InternEntry { + hash, + string_ptr: atom as usize, + }); + self.len += 1; + } + + fn place(&mut self, entry: InternEntry) { + let mask = self.slots.len() - 1; + let mut i = (entry.hash as usize) & mask; + while self.slots[i].string_ptr != 0 { + i = (i + 1) & mask; + } + self.slots[i] = entry; + } +} + +crate::perry_thread_local! { + /// Per agent, like the cache: an atom is a string in THIS agent's heap. + pub(crate) static ATOMS: std::cell::UnsafeCell = + std::cell::UnsafeCell::new(AtomTable::new()); +} + +/// The atom for `bytes`, without allocating. `None` when no atom exists — or +/// when the agent's table is already torn down. +#[inline] +pub(crate) fn atom_lookup(bytes: &[u8], hash: u64) -> Option<*const StringHeader> { + ATOMS + .try_with(|t| unsafe { (*t.get()).lookup(bytes, hash) }) + .ok() + .flatten() +} + +/// The atom with `key`'s text, if one exists. `hash` is the FNV-1a hash of +/// `key`'s bytes (`key_bytes_hash`). Allocation-free, GC-free. +/// +/// # Safety +/// `key` is a live heap `StringHeader`. +#[inline] +pub(crate) unsafe fn atom_for_key( + key: *const StringHeader, + hash: u64, +) -> Option<*const StringHeader> { + if key.is_null() || (*key).byte_len > INTERN_MAX_BYTE_LEN { + return None; + } + let bytes = std::slice::from_raw_parts(string_data(key), (*key).byte_len as usize); + ATOMS + .try_with(|t| (*t.get()).lookup_from(key as usize, bytes, hash)) + .ok() + .flatten() +} + +/// Mint (or find) the atom for a pooled literal: the one string object this +/// agent uses for that text from now on. Called from `__perry_init_strings_*` +/// in place of `js_string_from_bytes` for pool entries that can be property +/// keys (at most `INTERN_MAX_BYTE_LEN` bytes). `hash` is the pool's +/// precomputed FNV-1a hash of the bytes — the same function as every other +/// key hash here. +/// +/// Adopts the intern cache's string when it already holds this text, so the +/// keys runtime code interned before this module initialised keep matching. +/// Longer literals are not keys worth an atom and take the plain allocation. +#[no_mangle] +pub extern "C" fn js_string_pool_atom( + bytes: *const u8, + len: u32, + hash: u64, + is_wtf8: i32, +) -> *mut StringHeader { + if len == 0 || len > INTERN_MAX_BYTE_LEN || bytes.is_null() { + return if is_wtf8 != 0 { + js_string_from_wtf8_bytes(bytes, len) + } else { + js_string_from_bytes(bytes, len) + }; + } + let input = unsafe { std::slice::from_raw_parts(bytes, len as usize) }; + if let Some(atom) = atom_lookup(input, hash) { + return atom as *mut StringHeader; + } + // Finds the cache's string for this text or allocates one, marking it + // interned and immutable (`refcount = 0`). Nothing is held across the + // allocation: `bytes` is read-only data in the compiled image. + let atom = intern_dispatch_bytes(0, bytes, len as usize, 0, is_wtf8 != 0); + if atom.is_null() { + return js_string_from_bytes(bytes, len); + } + let _ = ATOMS.try_with(|t| unsafe { (*t.get()).insert(hash, atom) }); + atom as *mut StringHeader +} + +/// Test hook: evict `bytes` from the intern CACHE (a collision would), so the +/// next atom mint for that text allocates a new string instead of adopting +/// the cached one. +#[cfg(test)] +pub(crate) fn test_evict_interned(bytes: &[u8]) { + let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); + with_intern_table(|table| unsafe { + (*table)[(hash as usize) & INTERN_TABLE_MASK] = InternEntry { + hash: 0, + string_ptr: 0, + }; + }); +} + +/// Test view: is `p` the atom of its text? +#[cfg(test)] +pub(crate) unsafe fn is_atom_for_test(p: *const StringHeader) -> bool { + let bytes = std::slice::from_raw_parts(string_data(p), (*p).byte_len as usize); + let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); + atom_lookup(bytes, hash) == Some(p) +} + #[inline] pub(crate) fn with_intern_table( f: impl FnOnce(*mut [InternEntry; INTERN_TABLE_SIZE]) -> R, @@ -77,6 +300,20 @@ pub extern "C" fn js_string_intern(key: *const StringHeader, hash: u64) -> *cons return existing; } + // An atom owns this text: the cache slot takes the ATOM, so a computed + // key interns to the same string a read site and a shape key list hold + // (see `AtomTable`), never to a second copy. + let bytes = std::slice::from_raw_parts(string_data(key), byte_len as usize); + if let Some(atom) = atom_lookup(bytes, hash) { + with_intern_table(|table| { + (*table)[slot] = InternEntry { + hash, + string_ptr: atom as usize, + }; + }); + return atom; + } + // Miss or collision — insert (evict on collision) with_intern_table(|table| { (*table)[slot] = InternEntry { @@ -149,6 +386,16 @@ pub(crate) fn intern_dispatch_bytes( if let Some(existing) = hit { return existing; } + // An atom owns this text (see `AtomTable`): hand it out rather than a copy. + if let Some(atom) = atom_lookup(input, hash) { + with_intern_table(|table| unsafe { + (*table)[slot] = InternEntry { + hash, + string_ptr: atom as usize, + }; + }); + return atom; + } let key = if is_wtf8 { js_string_from_wtf8_bytes(bytes, byte_len as u32) @@ -267,6 +514,16 @@ pub fn scan_intern_table_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<' visitor.visit_tagged_usize_slot(&mut entry.string_ptr, crate::value::STRING_TAG); } }); + // The atoms: strong like the cache (every atom is also a registered pool + // handle's value, so this keeps nothing alive that was not), rewritten on + // move. A key's hash is its content's, so a move never rehashes. + let _ = ATOMS.try_with(|t| unsafe { + for entry in (*t.get()).slots.iter_mut() { + if entry.string_ptr != 0 { + visitor.visit_tagged_usize_slot(&mut entry.string_ptr, crate::value::STRING_TAG); + } + } + }); } /// #11507: the table is zero-allocated rather than filled, so a thread's first diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index 40de0c9758..d49320bf79 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -231,6 +231,10 @@ pub use html::{ js_string_fontcolor, js_string_fontsize, js_string_italics, js_string_link, js_string_small, js_string_strike, js_string_sub, js_string_sup, }; +pub(crate) use intern::atom_for_key; +pub use intern::js_string_pool_atom; +#[cfg(test)] +pub(crate) use intern::{atom_lookup, is_atom_for_test, test_evict_interned}; pub use intern::{js_string_intern, scan_intern_table_roots, scan_intern_table_roots_mut}; pub use io::{js_string_error, js_string_print, js_string_warn}; pub(crate) use iter_object::dispatch_string_iterator_method_builtin; From ee4b31a696ee6db42c598caeec2d74db291e8fcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 26 Sep 2026 08:43:31 +0000 Subject: [PATCH 2/8] perf(runtime): confirm a megamorphic site's slot guess against the receiver's key list first A site latched megamorphic sends every read that misses its compact word to js_object_get_field_ic_slow, which answered it from the receiver's shape only after decoding the word, classifying the receiver and scanning the key list. The site may hold one thing: a slot guess (the compact word's high half, the slot the receiver's shape answered last), which the receiver's own shape confirms or refutes. The slow entry now asks that first, and only at a latched site, so a site that can still be primed is primed as before: the receiver's ShapeId names its record; the record's POSITION BOUND says logical key position `guess` is inline slot `guess`; the key at that position must be this key (one pointer compare, S3b atoms); then the receiver's slot is the answer. Anything else continues down the unchanged path. Nothing is emitted at the site, so code size is unchanged. Whether a shape can answer by position is a FACT OF THE RECORD, stored in bit 15 of flags_and_kind (RECORD_POSITIONAL, in the pairwise-disjointness assert): an Ordinary, generation-0, hole-free shape with a keys array and no ACCESSOR key in its attribute summary. It is written by refresh_positional wherever an input can change (construction, with_summary, slab insert, the in-place stable-tombstone update), read with one load on the megamorphic path, and debug builds assert it against its definition on every read. The bound is then min(key count, live inline slots). A test walks every minted record of the agent and fails if the bit and its definition disagree (sabotage: dropping the slab-insert refresh fails it, 8 of 68 records). The in-place updaters only accept a private-epoch record (nonzero generation, never positional), so their refreshes cannot flip the bit today; a second test drives both updaters to zero holes and asserts that premise, so it is where those refreshes start to matter if it ever changes. Logical position i is read past the keys array's front offset (array_elements_ptr), so a shifted keys array is answered correctly. The confirm reads the record through a thread-local mirror of the ordinary page directory (pointer and length, republished whenever the slab's `pages` change, cleared before the slab is dropped): one thread-pointer-relative load and two directory loads, no runtime-state resolution. The step runs in the slow entry's frameless head; the rest of the entry moved out of line. The mirror has a per_thread verdict in thread_exit_address_globals.json. --- .../object/field_get_set/ic_miss/ic_slow.rs | 483 ++++++++++++++++++ crates/perry-runtime/src/object/shapes.rs | 132 ++++- .../src/object/shapes_slot_list.rs | 2 + .../perry-runtime/src/object/shapes_store.rs | 147 +++++- .../src/object/tombstone_tests.rs | 62 +++ scripts/thread_exit_address_globals.json | 8 + test-files/test_gap_megamorphic_slot_guess.ts | 101 ++++ 7 files changed, 915 insertions(+), 20 deletions(-) create mode 100644 test-files/test_gap_megamorphic_slot_guess.ts diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 09f0fc5b8a..f8b7be174e 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -237,6 +237,68 @@ pub extern "C-unwind" fn js_object_get_field_ic_slow( key: *const crate::StringHeader, cache_slot: *mut PicCacheSlot, packed: *const AtomicU64, +) -> f64 { + // --- 0. a MEGAMORPHIC site's slot guess, confirmed by the receiver ------ + // + // A site whose way state is latched negative sees more shapes than any + // per-site entry can name, and every read that misses its compact word + // lands here. The site may still hold one thing: a slot GUESS (the compact + // word's high half — the slot the receiver's shape answered last, step 2b), + // which the RECEIVER'S own shape confirms or refutes + // (`shapes::confirm_slot_guess`: the position bound says key position + // `guess` is inline slot `guess`, and the key there IS this key, one + // pointer compare). Everything the guess cannot answer continues in + // `ic_slow_body` unchanged. Asked first, and only at a latched site, so a + // site that can still be primed is primed exactly as before. `length` is + // excluded as in step 2b: an Array-subclass receiver serves it from its + // elements store. Kept in this frameless entry, with the body out of line, + // so the confirmed read pays no prologue for the arms below. + if let Some(value) = unsafe { megamorphic_slot_guess(obj_handle, key, cache_slot, packed) } { + return value; + } + ic_slow_body(obj_handle, key, cache_slot, packed) +} + +/// Step 0 of [`js_object_get_field_ic_slow`]: the latched site's slot guess. +/// +/// # Safety +/// The entry's contract: a POINTER receiver handle, this site's cache slot +/// and packed word. +#[inline(always)] +unsafe fn megamorphic_slot_guess( + obj_handle: i64, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, +) -> Option { + let obj = obj_handle as usize as *const ObjectHeader; + if packed.is_null() + || key.is_null() + || !crate::value::addr_class::is_above_handle_band(obj as usize) + { + return None; + } + let cache = crate::object::pic_slot_peek(cache_slot); + if cache.is_null() + || (*cache)[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] >= 0 + || key_is_length(key) + { + return None; + } + let guess = ((*packed).load(Ordering::Relaxed) >> 32) as usize; + let value = crate::object::shapes::confirm_slot_guess(obj, key, guess)?; + #[cfg(test)] + crate::object::shapes::SHAPE_ANSWERED_READS.fetch_add(1, Ordering::Relaxed); + Some(value) +} + +/// Everything after step 0 of [`js_object_get_field_ic_slow`]. +#[inline(never)] +fn ic_slow_body( + obj_handle: i64, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, ) -> f64 { let obj = obj_handle as usize as *const ObjectHeader; let addr = obj as usize; @@ -691,6 +753,427 @@ mod tests { assert!(n >= 47, "the shape answers by bytes: {n}"); } + /// Dictionary receivers keep their ShapeId across layout changes, so the + /// shape can never answer for one by position: a latched site reading + /// half-dictionary receivers answers every read correctly, and not one + /// dictionary read is counted as shape-answered. + #[test] + fn a_dictionary_receiver_is_never_answered_by_position() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_dict_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_dict_x"); + let ordinary = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + ordinary >= 47, + "premise: ordinary receivers are shape-answered ({ordinary})" + ); + let mut dict = 0; + for o in objs.iter().step_by(2) { + if o.with_mut_ptr(|p: *mut ObjectHeader| unsafe { + crate::object::dictionary::latch_object_to_dictionary(p) + }) { + dict += 1; + let bound = o.with_const_ptr(|p: *const ObjectHeader| { + crate::object::shapes::test_position_bound_of(p) + }); + assert_eq!(bound, Some(0), "a dictionary shape has no position bound"); + } + } + assert_eq!( + dict, 24, + "premise: every other receiver latched to dictionary" + ); + let n = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + n <= 24, + "INVARIANT: at most the 24 ordinary receivers are shape-answered, got {n}" + ); + } + + /// Tombstones: a receiver whose list carries a HOLE (a tombstone delete) + /// cannot be answered by position — its shape reports no position bound — + /// and every read still returns the receiver's own value; the deleted key + /// reads undefined. + #[test] + fn a_receiver_with_a_tombstoned_key_is_never_answered_by_position() { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + crate::object::delete_rest::test_set_tombstone_deletes(None); + } + } + crate::object::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = Restore; + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_tomb_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_tomb_x"); + let _ = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + let end = scope.root_string_ptr(key_of(b"end")); + let mut tombstoned = 0; + for (i, o) in objs.iter().enumerate().step_by(3) { + // The first delete of a shared list compacts (it takes ownership); + // the second tombstones in place (`tombstone_tests.rs`). + let extra = format!("s3b_tomb_x{i}"); + let extra = scope.root_string_ptr(key_of(extra.as_bytes())); + for k in [&extra, &end] { + o.with_mut_ptr(|p: *mut ObjectHeader| { + k.with_const_ptr(|kp| crate::object::js_object_delete_field(p, kp)) + }); + } + let (holes, bound) = o.with_const_ptr(|p: *const ObjectHeader| unsafe { + ( + crate::object::shapes::object_shape_hole_count(p), + crate::object::shapes::test_position_bound_of(p), + ) + }); + if holes > 0 { + tombstoned += 1; + assert_eq!( + bound, + Some(0), + "INVARIANT: a tombstoned shape has no position bound" + ); + } + } + assert_eq!( + tombstoned, 16, + "premise: every third receiver carries a tombstone" + ); + let _ = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + let _ = latched_pass(&objs, &end, |i| { + if i % 3 == 0 { + crate::value::TAG_UNDEFINED + } else { + 2.0f64.to_bits() + } + }); + } + + /// The slot-guess confirm for shape `shape_id`, as the megamorphic entry + /// asks it (`shapes::slot_guess_confirmed`): `Some(guess)` when the shape + /// says key position `guess` is inline slot `guess` and holds exactly the + /// key `site_key_bits` names, `None` for a decline. + unsafe fn guess_walk(shape_id: u32, guess: u64, site_key_bits: u64) -> Option { + let key = (site_key_bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader; + crate::object::shapes::slot_guess_confirmed(shape_id, key, guess as usize) + .then_some(guess as usize) + } + + fn stamp_of(o: &crate::gc::RuntimeHandle<'_>) -> u32 { + o.with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::shapes::object_shape_stamp(p) + }) + } + + /// For every ordinary receiver the slot-guess confirm reaches the + /// receiver's own record, confirms the site's guess against the + /// receiver's own key, and names the slot holding the receiver's own + /// value. A guess whose position holds a DIFFERENT key, a guess past the + /// shape's bound, an id outside the ShapeId range and an id whose page + /// was never allocated all decline. + #[test] + fn the_slot_guess_is_confirmed_only_against_the_receivers_own_key() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3c_walk_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let site = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3c_walk_x"); + for (i, o) in objs.iter().enumerate() { + let id = stamp_of(o); + let slot = unsafe { guess_walk(id, 2, site) }; + assert_eq!(slot, Some(2), "receiver {i}: the right guess is confirmed"); + let value = o.with_const_ptr(|p: *const ObjectHeader| unsafe { + *((p as *const u8).add(std::mem::size_of::() + 2 * 8) as *const f64) + }); + assert_eq!( + value, + 100.0 + i as f64, + "receiver {i}: the confirmed slot is its own" + ); + for wrong in [0u64, 1, 3] { + assert_eq!( + unsafe { guess_walk(id, wrong, site) }, + None, + "receiver {i}: guess {wrong} holds another key and must decline" + ); + } + assert_eq!(unsafe { guess_walk(id, 4, site) }, None, "past the bound"); + assert_eq!(unsafe { guess_walk(id, u64::from(u32::MAX), site) }, None); + } + assert_eq!( + unsafe { guess_walk(5, 2, site) }, + None, + "a class id is no ShapeId" + ); + assert_eq!( + unsafe { guess_walk(0xBFFF_FFFF, 2, site) }, + None, + "a page never minted" + ); + } + + /// The slot-guess confirm on dictionary and tombstoned receivers: declines, + /// whatever the guess. + #[test] + fn the_slot_guess_never_matches_a_dictionary_or_tombstoned_receiver() { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + crate::object::delete_rest::test_set_tombstone_deletes(None); + } + } + crate::object::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = Restore; + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3c_walk2_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let site = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let objs = megamorphic_receivers_keyed(&scope, 8, text, "s3c_walk2_x"); + let dict = &objs[0]; + assert!( + dict.with_mut_ptr(|p: *mut ObjectHeader| unsafe { + crate::object::dictionary::latch_object_to_dictionary(p) + }), + "premise: the receiver latched to dictionary" + ); + for guess in 0..4 { + assert_eq!( + unsafe { guess_walk(stamp_of(dict), guess, site) }, + None, + "dictionary" + ); + } + let tomb = &objs[1]; + let end = scope.root_string_ptr(key_of(b"end")); + let extra = scope.root_string_ptr(key_of(b"s3c_walk2_x1")); + for k in [&extra, &end] { + tomb.with_mut_ptr(|p: *mut ObjectHeader| { + k.with_const_ptr(|kp| crate::object::js_object_delete_field(p, kp)) + }); + } + let holes = tomb.with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::shapes::object_shape_hole_count(p) + }); + assert!(holes > 0, "premise: the receiver carries a tombstone"); + for guess in 0..4 { + assert_eq!( + unsafe { guess_walk(stamp_of(tomb), guess, site) }, + None, + "tombstoned" + ); + } + } + + fn boxed(p: *mut ObjectHeader) -> f64 { + f64::from_bits(0x7FFD_0000_0000_0000 | (p as u64 & 0x0000_FFFF_FFFF_FFFF)) + } + + /// A [[Prototype]] change is a shape transition (the prototype is part of + /// shape identity) that moves no own key and no own slot: receivers built + /// by `new F()` / `setPrototypeOf` before their fields are assigned — every + /// tsc AST node — get a NEW shape with the SAME own-key layout as a + /// prototype-less twin, and the megamorphic read answers them from that + /// shape, by the key-list scan and by the slot-guess confirm. + #[test] + fn a_prototype_change_is_a_new_shape_with_the_same_layout_and_is_answered() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3_proto_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let site = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let proto = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 2)); + let build = |with_proto: bool, i: usize| { + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + if with_proto { + let (o, p) = ( + obj.with_mut_ptr(|o: *mut ObjectHeader| o), + proto.with_mut_ptr(|p: *mut ObjectHeader| p), + ); + crate::object::object_ops::js_object_set_prototype_of(boxed(o), boxed(p)); + } + let extra = format!("s3_proto_x{i}"); + for (k, v) in [ + (&b"pos"[..], 1.0), + (&b"end"[..], 2.0), + (&text[..], 100.0 + i as f64), + (extra.as_bytes(), 7.0), + ] { + let key = scope.root_string_ptr(key_of(k)); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + obj + }; + let objs: Vec<_> = (0..48).map(|i| build(true, i)).collect(); + let twin = build(false, 0); + let (shape, twin_shape) = (stamp_of(&objs[0]), stamp_of(&twin)); + assert_ne!( + shape, twin_shape, + "premise: the prototype is part of shape identity" + ); + let d = |id| crate::object::shapes::shape_descriptor_by_id(id).expect("live shape"); + let (d, t) = (d(shape), d(twin_shape)); + assert_eq!( + ( + d.keys, + d.logical_key_count, + d.live_inline_slot_count, + d.semantic_generation + ), + ( + t.keys, + t.logical_key_count, + t.live_inline_slot_count, + t.semantic_generation + ), + "the prototype change moved no own key and bumped no generation" + ); + for (i, o) in objs.iter().enumerate() { + assert_eq!( + unsafe { guess_walk(stamp_of(o), 2, site) }, + Some(2), + "INVARIANT: the guess confirm answers prototype-linked receiver {i}" + ); + } + let n = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + n >= 47, + "INVARIANT: prototype-linked receivers are shape-answered: {n}" + ); + } + + /// A shape over a SHIFTED keys array (a front offset): the slot-guess + /// confirm reads LOGICAL key position `i`, past the front offset, so the + /// guess for the key now at logical 0 is confirmed at 0, and the physical + /// position it used to occupy (1) is not. + #[test] + fn a_shape_over_a_shifted_keys_array_is_answered_by_logical_position() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let arr = scope.root_raw_mut_ptr(crate::array::js_array_alloc(4)); + let mut keys = Vec::new(); + for name in [&b"sh_a"[..], b"sh_b", b"sh_c"] { + let k = scope.root_string_ptr(key_of(name)); + let bits = k.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + arr.with_mut_ptr(|a| crate::array::js_array_push(a, crate::JSValue::from_bits(bits))); + keys.push(k); + } + let a = arr.with_mut_ptr(|a: *mut crate::array::ArrayHeader| a); + let bound = + |id: u32| crate::object::shapes::test_position_bound_of_id(id).expect("a live shape"); + let flat = crate::object::shapes::shape_descriptor_ensure(a, 3, 3).expect("mints"); + assert_eq!(bound(flat), 3, "premise: an unshifted keys array"); + crate::array::js_array_shift_f64(a); + assert_ne!( + unsafe { crate::array::array_front_offset(a) }, + 0, + "premise: shift left a front offset" + ); + let shifted = crate::object::shapes::shape_descriptor_ensure(a, 2, 2).expect("mints"); + assert_eq!(bound(shifted), 2, "premise: the shifted shape has two keys"); + let sh_b = keys[1].with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + assert_eq!( + unsafe { guess_walk(shifted, 0, sh_b) }, + Some(0), + "INVARIANT: the confirm reads the logical position" + ); + assert_eq!( + unsafe { guess_walk(shifted, 1, sh_b) }, + None, + "INVARIANT: the physical position is not a key position" + ); + } + + /// The positional bit is a stored FACT of the shape record, read on every + /// latched miss; it must equal its definition for every record the agent + /// has minted. Mints ordinary, dictionary, tombstoned, accessor and + /// class-keyed shapes, then walks the whole slab. + #[test] + fn every_minted_records_positional_bit_matches_its_facts() { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + crate::object::delete_rest::test_set_tombstone_deletes(None); + } + } + crate::object::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = Restore; + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let objs = megamorphic_receivers_keyed(&scope, 24, b"pos_fact_kind", "pos_fact_x"); + // Tombstones: the in-place stable-tombstone update rewrites the hole + // count (two deletes: the first compacts, the second tombstones). + let end = scope.root_string_ptr(key_of(b"end")); + for (i, o) in objs.iter().enumerate().step_by(3) { + let extra = format!("pos_fact_x{i}"); + let extra = scope.root_string_ptr(key_of(extra.as_bytes())); + for k in [&extra, &end] { + o.with_mut_ptr(|p: *mut ObjectHeader| { + k.with_const_ptr(|kp| crate::object::js_object_delete_field(p, kp)) + }); + } + } + // Dictionaries. + for o in objs.iter().skip(1).step_by(3) { + o.with_mut_ptr(|p: *mut ObjectHeader| unsafe { + crate::object::dictionary::latch_object_to_dictionary(p) + }); + } + // Keep the tombstoned receivers growing: re-adds take the cached + // stable-tombstone update. + let again = scope.root_string_ptr(key_of(b"pos_fact_again")); + for o in objs.iter().step_by(3) { + o.with_mut_ptr(|p| { + again.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(p, kp, 3.0)) + }); + } + // Accessor keys: the summary is an input of the bit. + let acc = scope.root_string_ptr(key_of(b"pos_fact_acc")); + for o in objs.iter().skip(2).step_by(3) { + let (ov, kv) = ( + o.with_const_ptr(|p: *const ObjectHeader| { + crate::value::js_nanbox_pointer(p as i64) + }), + acc.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64) + }), + ); + let undef = f64::from_bits(crate::value::TAG_UNDEFINED); + crate::object::js_object_define_accessor(ov, kv, undef, undef); + } + let (n, positional, accessor, bad) = crate::object::shapes::test_positional_census(); + assert!( + positional > 0 && positional < n, + "premise: the slab holds both answerable and unanswerable shapes ({positional} of {n})" + ); + assert!( + accessor > 0, + "premise: an ordinary shape with an accessor key was minted" + ); + assert!( + bad.is_empty(), + "INVARIANT: the stored positional bit equals its definition; {} of {n} records disagree: {bad:x?}", + bad.len() + ); + } + /// A plain own data read that has never primed: the entry must fall all the /// way through to the miss handler, answer the field, and leave the site /// primed exactly as the old `js_object_get_field_ic_miss_packed` edge did. diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 4f87d4d02e..c21a47a509 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -249,13 +249,11 @@ impl ShapeRecordRef { /// shape's own canonical key list — or `None` when the shape cannot answer /// by position alone. /// - /// The position of a key in the keys list is its slot exactly when the - /// shape is `Ordinary`, generation 0 (no descriptor/prototype mutation - /// minted it) and hole-free; a position below `live_inline_slot_count` is - /// an inline slot of every receiver carrying the shape (the invariant the - /// read cache's prime already relies on). The list is bounded by the - /// SHAPE's key count, never the backing's length (#10969: one backing per - /// growth chain). + /// The shape's [`ShapeRecord::position_bound`] says how many leading key + /// positions ARE inline slots of every receiver carrying it (0 for a + /// dictionary, class, descriptor/prototype-generation or tombstoned + /// shape). The list is bounded by that — never by the backing's length + /// (#10969: one backing per growth chain). /// /// Key compares go identity first: canonical lists hold their text's ATOM /// (`string::intern::AtomTable`), which is also what a read site's pooled @@ -270,11 +268,8 @@ impl ShapeRecordRef { hint: usize, ) -> Option { let r = &*self.0.as_ptr(); - if r.object_kind() != ShapeObjectKind::Ordinary - || r.semantic_generation != 0 - || r.hole_count != 0 - || r.keys == 0 - { + let bound = r.position_bound() as usize; + if bound == 0 { return None; } let (slots, len) = @@ -282,11 +277,9 @@ impl ShapeRecordRef { if slots.is_null() { return None; } - let bound = len - .min(r.logical_key_count as usize) - .min(r.live_inline_slot_count as usize); + let bound = bound.min(len); let heap_bits = crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits(); - // The site's slot guess first: the receiver's shape confirms it. + // The site's slot guess, confirmed by the receiver's own key. if hint < bound && (*slots.add(hint)).to_bits() == heap_bits { return Some(hint); } @@ -374,6 +367,113 @@ unsafe fn stored_key_matches(key: *const crate::StringHeader, bits: u64) -> bool } } +/// The position bound of `obj`'s shape (S3c), or `None` when its word names +/// no record in this agent. +#[cfg(test)] +pub(crate) fn test_position_bound_of(obj: *const crate::object::ObjectHeader) -> Option { + let id = unsafe { object_shape_stamp(obj) }; + shape_record_by_id(id).map(|r| unsafe { (*r.0.as_ptr()).position_bound() }) +} + +/// Walk every present record of this agent's slab: `(records, positional, +/// ordinary records with an accessor key, disagreements)`, where a disagreement is a record whose stored positional +/// bit differs from [`ShapeRecord::positional_by_facts`]. +#[cfg(test)] +pub(crate) fn test_positional_census() -> (usize, usize, usize, Vec) { + let table = &crate::state::state().shapes; + let (mut n, mut positional, mut accessor, mut bad) = (0usize, 0usize, 0usize, Vec::new()); + table.slab().for_each(|id, p| { + let r = unsafe { &*p }; + if !r.present() { + return; + } + n += 1; + if r.positional_bit() { + positional += 1; + } + if r.object_kind() == ShapeObjectKind::Ordinary + && r.summary() & crate::object::key_attrs::SUMMARY_ACCESSOR != 0 + { + accessor += 1; + } + if r.positional_bit() != r.positional_by_facts() { + bad.push(id); + } + }); + (n, positional, accessor, bad) +} + +/// `(stored positional bit, its definition)` for shape `id`. +#[cfg(test)] +pub(crate) fn test_positional_of_id(id: u32) -> Option<(bool, bool)> { + shape_record_by_id(id).map(|r| unsafe { + let r = &*r.0.as_ptr(); + (r.positional_bit(), r.positional_by_facts()) + }) +} + +/// The position bound of shape `id` (S3c), or `None` when it names no record. +#[cfg(test)] +pub(crate) fn test_position_bound_of_id(id: u32) -> Option { + shape_record_by_id(id).map(|r| unsafe { (*r.0.as_ptr()).position_bound() }) +} + +/// The megamorphic read's slot-guess confirm: when `obj` carries an ordinary +/// ShapeId of this agent whose record says key position `guess` is inline slot +/// `guess` (`position_bound`), and the key AT that position is `key` itself +/// (one pointer compare: canonical lists hold their text's atom), the value in +/// the receiver's slot `guess`. `None` for anything else — a wrong or stale +/// guess, another text, a dictionary/class/descriptor/tombstoned shape, an id +/// that names no record — and the caller takes its ordinary path. +/// +/// The guess decides nothing: the receiver's own shape confirms it or it is +/// ignored. Allocation-free, no user code. +/// +/// # Safety +/// `obj` is a heap pointer above the handle band (its `+4` word is read); +/// `key` is a heap `StringHeader`. +#[inline] +pub(crate) unsafe fn confirm_slot_guess( + obj: *const crate::object::ObjectHeader, + key: *const crate::StringHeader, + guess: usize, +) -> Option { + if !slot_guess_confirmed((*obj).parent_class_id, key, guess) { + return None; + } + Some( + *((obj as *const u8).add(std::mem::size_of::() + guess * 8) + as *const f64), + ) +} + +/// Does shape `shape_id` of this agent store `key` at inline slot `guess`, +/// by position? See [`confirm_slot_guess`]. +/// +/// # Safety +/// `key` is a heap `StringHeader`. +#[inline] +pub(crate) unsafe fn slot_guess_confirmed( + shape_id: u32, + key: *const crate::StringHeader, + guess: usize, +) -> bool { + let record = ShapeSlab::ordinary_record(shape_id); + if record.is_null() { + return false; + } + let r = &*record; + if guess >= r.position_bound() as usize { + return false; + } + // A bound > 0 means the record names a live keys array (the collector + // marks through and rewrites `keys`) holding at least `bound` logical + // keys; logical element `i` sits past the array's front offset. + let arr = r.keys as usize as *const ArrayHeader; + let slots = crate::array::array_elements_ptr(arr) as *const u64; + *slots.add(guess) == crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits() +} + /// Byte equality without a libc call for the short keys property names are. #[inline] unsafe fn bytes_eq(a: *const u8, b: *const u8, n: usize) -> bool { diff --git a/crates/perry-runtime/src/object/shapes_slot_list.rs b/crates/perry-runtime/src/object/shapes_slot_list.rs index f90ab264d6..37651aa907 100644 --- a/crates/perry-runtime/src/object/shapes_slot_list.rs +++ b/crates/perry-runtime/src/object/shapes_slot_list.rs @@ -578,6 +578,8 @@ pub(crate) unsafe fn try_update_stable_tombstone_shape_cached( record.logical_key_count = logical_key_count; record.live_inline_slot_count = live_inline_slot_count; record.hole_count = hole_count; + // The hole count is an input of the positional bit. + record.refresh_positional(); super::debug_assert_object_shape_parity(obj); Some(id) } diff --git a/crates/perry-runtime/src/object/shapes_store.rs b/crates/perry-runtime/src/object/shapes_store.rs index c0bafd1766..08fbb37016 100644 --- a/crates/perry-runtime/src/object/shapes_store.rs +++ b/crates/perry-runtime/src/object/shapes_store.rs @@ -72,7 +72,9 @@ pub(crate) struct ShapeRecord { pub(super) hole_count: u32, /// Low 8 bits: the `RECORD_FLAG_*` set. Bits 8-10: the `ShapeObjectKind` /// discriminant. Bits 11-14: the births a keyless birth shape served while - /// tracking its width (#10905). Bit 15: reserved. Bits 16-23: the + /// tracking its width (#10905). Bit 15: ANSWERABLE BY POSITION (see + /// [`ShapeRecord::position_bound`]), derived from the record's own facts. + /// Bits 16-23: the /// attribute SUMMARY byte (`key_attrs::SUMMARY_*`), an identity fact. /// Bits 24-31: the inline width a keyless birth shape's descendants grow /// to (#10905). The two #10905 fields are learned facts of the record, @@ -107,6 +109,10 @@ const RECORD_BIRTHS_MASK: u32 = 0xF << RECORD_BIRTHS_SHIFT; /// #10905 (`shapes_birth_width`): the learned descendant width, bits 24-31. const RECORD_WIDTH_SHIFT: u32 = 24; const RECORD_WIDTH_MASK: u32 = 0xFF << RECORD_WIDTH_SHIFT; +/// Bit 15: the shape answers by position ([`ShapeRecord::position_bound`]). +/// A function of the record's facts, rewritten by +/// [`ShapeRecord::refresh_positional`] wherever an input changes. +const RECORD_POSITIONAL: u32 = 1 << 15; // The fields of `flags_and_kind` are pairwise disjoint. const _: () = { let fields = [ @@ -115,6 +121,7 @@ const _: () = { RECORD_BIRTHS_MASK, RECORD_SUMMARY_MASK, RECORD_WIDTH_MASK, + RECORD_POSITIONAL, ]; let mut i = 0; while i < fields.len() { @@ -187,6 +194,8 @@ impl ShapeRecord { pub(super) fn with_summary(mut self, summary: u8) -> ShapeRecord { self.flags_and_kind = (self.flags_and_kind & !RECORD_SUMMARY_MASK) | (u32::from(summary) << RECORD_SUMMARY_SHIFT); + // The summary is an input of the positional bit (an accessor key). + self.refresh_positional(); self } @@ -247,7 +256,7 @@ impl ShapeRecord { // because `facts_match` compares the full enum. let kind_bits = (object_kind.code() as u32) << RECORD_KIND_SHIFT; debug_assert!(kind_bits & !RECORD_KIND_MASK == 0, "kind does not fit"); - ShapeRecord { + let mut record = ShapeRecord { keys, semantic_generation, proto_id: 0, @@ -255,7 +264,74 @@ impl ShapeRecord { live_inline_slot_count, hole_count, flags_and_kind: u32::from(flags) | kind_bits, + }; + record.refresh_positional(); + record + } + + /// How many leading keys of this shape's list sit AT their own inline + /// slot: logical key position `i < bound` IS inline slot `i` of every + /// receiver carrying the shape. 0 when the shape cannot answer by position + /// at all. + /// + /// Whether it can is a FACT OF THE RECORD, stored in bit 15 + /// (`RECORD_POSITIONAL`) and read here with one load: the megamorphic + /// read asks it on every latched miss. [`Self::positional_by_facts`] is + /// its definition; every write of one of its inputs is followed by + /// [`Self::refresh_positional`] (construction, `with_summary`, slab + /// insert, the in-place stable-tombstone update), and debug builds assert + /// the stored bit against the definition on every read. + /// + /// The bound is `min(logical_key_count, live_inline_slot_count)`: a key + /// past the key count is another list's (canonical backings are shared by + /// a growth chain), and one past the live inline count is spilled. + #[inline] + pub(crate) fn position_bound(&self) -> u32 { + debug_assert_eq!( + self.flags_and_kind & RECORD_POSITIONAL != 0, + self.positional_by_facts(), + "the positional bit disagrees with the record's facts: {self:?}" + ); + if self.flags_and_kind & RECORD_POSITIONAL == 0 { + return 0; } + self.logical_key_count.min(self.live_inline_slot_count) + } + + /// The definition of the positional bit. The conjuncts are + /// `js_shape_ordinary_inline_slot_for_key`'s: + /// + /// * `Ordinary` — a class shape's slots are its class layout, and a + /// DICTIONARY shape keeps its id across layout changes, so a dictionary + /// receiver must never be matched by position; + /// * generation 0 — a descriptor/prototype mutation minted this layout; + /// * no tombstones — the answer is only claimed for hole-free lists; + /// * no ACCESSOR key in the attribute summary — an accessor key's slot + /// holds its accessor pair, not a value; + /// * a keys array at all. + #[inline] + pub(super) fn positional_by_facts(&self) -> bool { + self.object_kind() == ShapeObjectKind::Ordinary + && self.semantic_generation == 0 + && self.hole_count == 0 + && self.keys != 0 + && self.summary() & crate::object::key_attrs::SUMMARY_ACCESSOR == 0 + } + + /// Rewrite the positional bit from the record's facts. + #[inline] + pub(super) fn refresh_positional(&mut self) { + if self.positional_by_facts() { + self.flags_and_kind |= RECORD_POSITIONAL; + } else { + self.flags_and_kind &= !RECORD_POSITIONAL; + } + } + + /// The stored positional bit, for the agreement test. + #[cfg(test)] + pub(super) fn positional_bit(&self) -> bool { + self.flags_and_kind & RECORD_POSITIONAL != 0 } /// The same record for a receiver whose [[Prototype]] identity is @@ -465,6 +541,19 @@ fn new_page() -> Page { .unwrap_or_else(|_| unreachable!("page vector has PAGE_LEN slots")) } +/// This thread's ordinary page directory as `(page pointers, page count)`: +/// `ShapeSlab::pages`' element pointer and length, republished after every +/// change to `pages` (`ShapeSlab::publish_dir`) and cleared before the slab +/// is dropped. The megamorphic read's slot-guess confirm +/// ([`ShapeSlab::ordinary_record`]) reads it with one thread-local load +/// instead of resolving the runtime state and walking `record_ptr`. +/// `#[thread_local]` (const, no destructor) rather than `thread_local!`: a +/// late read during thread teardown sees the cleared pair, and the access +/// compiles to one thread-pointer-relative load. +#[thread_local] +static ORDINARY_DIR: std::cell::Cell<(*const Option, usize)> = + std::cell::Cell::new((std::ptr::null(), 0)); + /// The by-id descriptor store. See the module docs. /// Two page directories: ordinary ShapeIds index from `SHAPE_ID_BASE`, and the /// dictionary band (`shapes::DICTIONARY_SHAPE_ID_BASE`) from its own base. One @@ -473,6 +562,14 @@ fn new_page() -> Page { /// that one ~196 KB allocation moved the GC arena's pages relative to the /// page-class table window and cost +2.3% instructions (1.65 M vs 0.20 M /// registered-page misses in `classify_heap_generation`). +impl Drop for ShapeSlab { + fn drop(&mut self) { + if ORDINARY_DIR.get().0 == self.pages.as_ptr() { + ORDINARY_DIR.set((std::ptr::null(), 0)); + } + } +} + pub(crate) struct ShapeSlab { pages: Vec>, dict_pages: Vec>, @@ -587,18 +684,26 @@ impl ShapeSlab { /// Install `record` under `id`, allocating the page and chunk on first /// touch. Returns the record it replaced, if the id was already present. pub(super) fn insert(&mut self, id: u32, mut record: ShapeRecord) -> Option { - let (dict, index) = + let (band, index) = Self::index_of(id).expect("ShapeSlab::insert: id outside the ShapeId range"); record.set(RECORD_FLAG_PRESENT, true); let (page, chunk, slot) = Self::split(index); - let dir = self.dir_mut(dict); + let dir = self.dir_mut(band); if page >= dir.len() { dir.resize_with(page + 1, || None); + // Only the ordinary band is mirrored (`ORDINARY_DIR`). + if band == 0 { + self.publish_dir(); + } } + let dir = self.dir_mut(band); let page = dir[page].get_or_insert_with(new_page); let chunk = page[chunk].get_or_insert_with(new_chunk); let cell = chunk[slot].get_mut(); let previous = cell.present().then_some(*cell); + // A retire-and-reinsert edits facts on a removed copy: the positional + // bit follows them. + record.refresh_positional(); *cell = record; if previous.is_none() { self.len += 1; @@ -686,6 +791,39 @@ impl ShapeSlab { } dir.shrink_to_fit(); } + self.publish_dir(); + } + + /// Publish `pages` for [`Self::ordinary_record`] (see [`ORDINARY_DIR`]). + fn publish_dir(&self) { + ORDINARY_DIR.set((self.pages.as_ptr(), self.pages.len())); + } + + /// The record of ordinary ShapeId `id` in THIS thread's slab, or null — + /// the fast twin of [`Self::record_ptr`] for the megamorphic read: one + /// thread-local load, two dependent directory loads, no `state()`. A + /// dictionary- or exotic-band id indexes past the ordinary directory's + /// length. The + /// record may be absent (`EMPTY`): its position bound is 0. + #[inline(always)] + pub(super) fn ordinary_record(id: u32) -> *const ShapeRecord { + let (pages, len) = ORDINARY_DIR.get(); + let index = id.wrapping_sub(SHAPE_ID_BASE) as usize; + let (page, chunk, slot) = Self::split(index); + if page >= len { + return std::ptr::null(); + } + // SAFETY: `pages` holds `len` entries of this thread's slab, current + // as of the last change to it; nothing here can change it. + unsafe { + let Some(page) = (*pages.add(page)).as_ref() else { + return std::ptr::null(); + }; + match page[chunk].as_ref() { + Some(chunk) => chunk[slot].get(), + None => std::ptr::null(), + } + } } #[cfg(test)] @@ -693,6 +831,7 @@ impl ShapeSlab { self.pages.clear(); self.dict_pages.clear(); self.exotic_pages.clear(); + self.publish_dir(); self.len = 0; } diff --git a/crates/perry-runtime/src/object/tombstone_tests.rs b/crates/perry-runtime/src/object/tombstone_tests.rs index d156c14d22..9e18f364d1 100644 --- a/crates/perry-runtime/src/object/tombstone_tests.rs +++ b/crates/perry-runtime/src/object/tombstone_tests.rs @@ -1118,3 +1118,65 @@ fn stable_tombstone_bound_move_mints_a_new_shape_id() { assert_eq!(after.hole_count, shape.hole_count); } } + +/// The two in-place stable-tombstone updaters rewrite the hole count (and, in +/// the uncached one, the summary) of a live record without reinserting it, +/// and both are inputs of the positional bit (`ShapeRecord::position_bound`). +/// Both refresh the bit, but today no update can FLIP it: the updaters only +/// accept a stable-tombstone receiver's detached record, which carries a +/// private-epoch (nonzero) semantic generation, so it is never positional, +/// hole or not. Driven to zero holes directly, each updater must leave the bit +/// equal to its definition, and the premise that pins the bit false is +/// asserted, so the day an updater admits a generation-0 record this test +/// is where the refresh starts to matter. +#[test] +fn in_place_tombstone_updates_keep_the_positional_bit_equal_to_its_facts() { + super::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = scopeguard_tombstone_flag(); + let _global = crate::gc::global_side_table_test_lock(); + unsafe { + for cached in [true, false] { + let name: &[u8] = if cached { + b"posbit_cached" + } else { + b"posbit_plain" + }; + let (obj, shape) = stable_receiver_one_hole(0, name); + let id = super::shapes::object_shape_stamp(obj); + let keys = shape.keys as usize as *mut crate::ArrayHeader; + let updated = if cached { + super::shapes::try_update_stable_tombstone_shape_cached( + obj, + shape, + shape.logical_key_count, + shape.live_inline_slot_count, + 0, + ) + } else { + super::shapes::try_update_stable_tombstone_shape( + obj, + keys, + shape.logical_key_count, + shape.live_inline_slot_count, + 0, + ) + }; + assert_eq!( + updated, + Some(id), + "premise: the updater (cached={cached}) ran in place" + ); + let after = super::shapes::object_shape_descriptor(obj).unwrap(); + assert_eq!(after.hole_count, 0, "premise: the update wrote zero holes"); + assert_ne!( + after.semantic_generation, 0, + "premise: an in-place-updatable record is a private epoch (nonzero generation)" + ); + let (bit, facts) = super::shapes::test_positional_of_id(id).unwrap(); + assert_eq!( + bit, facts, + "INVARIANT: the in-place update (cached={cached}) left the positional bit stale" + ); + } + } +} diff --git a/scripts/thread_exit_address_globals.json b/scripts/thread_exit_address_globals.json index db97d8f5b0..a9bc9c2681 100644 --- a/scripts/thread_exit_address_globals.json +++ b/scripts/thread_exit_address_globals.json @@ -4086,6 +4086,14 @@ ], "verdict": "no_heap_address", "why": "Holds (Handle, Parked) rows: a Handle is a native-registry id in [1, 0x40000) minted by the process-global shared pool (perry_ffi::shared_handle_id_pool), never an arena address, and Parked is {kind, NativeRegistrationIdentity{domain, serial, numeric_id}, epoch}. Payloads live in the process-global common HANDLES map, not in any thread's arena. A row cannot name a different object after reuse: release() retires only via begin_retirement_of(identity), which rejects any registration whose serial differs, and ids are reissued only after a full trace proves them unreferenced (#11453)." + }, + { + "file": "crates/perry-runtime/src/object/shapes_store.rs", + "names": [ + "ORDINARY_DIR" + ], + "verdict": "per_thread", + "why": "`#[thread_local]` static (not `thread_local!`, so the megamorphic read reaches it with one thread-pointer-relative load): each thread mirrors ITS OWN ShapeSlab page directory (a Rust-heap Vec pointer and length, never an arena address), republished on every change to `pages` and cleared by ShapeSlab::drop before the Vec is freed; const-initialised to (null, 0) with no drop glue." } ] } diff --git a/test-files/test_gap_megamorphic_slot_guess.ts b/test-files/test_gap_megamorphic_slot_guess.ts new file mode 100644 index 0000000000..a8af38d056 --- /dev/null +++ b/test-files/test_gap_megamorphic_slot_guess.ts @@ -0,0 +1,101 @@ +// S3c: the inline shape-confirmed slot guess at a megamorphic read site. +// +// One read site (`readKind`) sees 60+ shapes, so it latches megamorphic and +// its reads take the inline guess: the site's slot guess is confirmed against +// the RECEIVER's own shape key list before any slot is loaded. Every receiver +// family below is built so that a guess which is NOT confirmed by the +// receiver's own shape would return a wrong value: +// * `kind` at slot 2 in most shapes and at other slots in some (the guess +// is right for some receivers, wrong for others); +// * a DIFFERENT key sitting at the guessed slot; +// * dictionary-mode receivers (many keys added one by one); +// * receivers that deleted a key (tombstones / compaction); +// * an accessor installed with defineProperty (descriptor shape); +// * `kind` inherited from a prototype, and `kind` absent; +// * class instances. +// The output is a per-family checksum and must equal node's. + +const EXTRA = []; +for (let i = 0; i < 48; i++) EXTRA.push("x" + i); + +function plain(i: number): any { + const o: any = { pos: i, end: i + 1, kind: i % 7 }; + o[EXTRA[i % 48]] = i; + return o; +} +function shifted(i: number): any { + // `kind` at slot 4: the site's guess (2) names `flags` here. + const o: any = { pos: i, end: i + 1, flags: 1000 + i, parent: null, kind: 50 + (i % 5) }; + o[EXTRA[(i + 7) % 48]] = i; + return o; +} +function dictionary(i: number): any { + const o: any = { pos: i, end: i + 1, kind: 200 + (i % 3) }; + for (let k = 0; k < 200; k++) o["d" + i + "_" + k] = k; + return o; +} +function deleted(i: number): any { + const o: any = { pos: i, end: i + 1, kind: 300 + (i % 4), gone: 1, also: 2 }; + o[EXTRA[(i + 3) % 48]] = i; + delete o.also; + delete o.end; + return o; +} +function accessor(i: number): any { + const o: any = { pos: i, end: i + 1, kind: -1 }; + Object.defineProperty(o, "kind", { get() { return 400 + (i % 6); }, enumerable: true }); + return o; +} +const proto = { kind: 500 }; +function inherited(i: number): any { + const o: any = Object.create(proto); + o.pos = i; + o.end = i + 1; + o.flags = i; + return o; +} +function absent(i: number): any { + const o: any = { pos: i, end: i + 1, flags: 600 + i }; + o[EXTRA[(i + 11) % 48]] = i; + return o; +} +class Node3 { + pos: number; end: number; kind: number; + constructor(i: number) { this.pos = i; this.end = i + 1; this.kind = 700 + (i % 9); } +} + +function readKind(node: any): any { + return node.kind; +} + +const families: [string, (i: number) => any][] = [ + ["plain", plain], ["shifted", shifted], ["dictionary", dictionary], ["deleted", deleted], + ["accessor", accessor], ["inherited", inherited], ["absent", absent], + ["class", (i: number) => new Node3(i)], +]; +const pool: [number, any][] = []; +for (let f = 0; f < families.length; f++) { + for (let i = 0; i < 64; i++) pool.push([f, families[f][1](i)]); +} +// Interleave so the site sees every family while latched. +const order: number[] = []; +for (let i = 0; i < pool.length; i++) order.push((i * 37) % pool.length); + +const sums: number[] = families.map(() => 0); +let undef = 0; +for (let round = 0; round < 50; round++) { + for (const j of order) { + const [f, o] = pool[j]; + const v = readKind(o); + if (v === undefined) undef++; + else sums[f] += v; + } + // Mutate some receivers between rounds: a moved key, a re-added key. + if (round === 20) { + for (const [f, o] of pool) { + if (f === 0 && o.pos % 5 === 0) { delete o.kind; o.kind = 900; } + } + } +} +for (let f = 0; f < families.length; f++) console.log(families[f][0] + " " + sums[f]); +console.log("undefined " + undef); From e8a6c403210c02b3b10f92ac109cd60e607177cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 27 Sep 2026 09:12:50 +0000 Subject: [PATCH 3/8] fix(runtime): an atom is key identity, never interned-key eligibility Minting atoms through the intern cache flagged every pool literal GC_FLAG_INTERNED, which silently admitted literal keys to the interned-only own-property lanes (read lane, set fast paths, chain store, proxy put). On Zod the widened read lane misses for inherited keys: keys_find_slot_by_key_ptr 5014 -> 8022 calls, +0.3%. Atoms are now plain allocations, and the intern cache neither adopts nor hands them out. --- .../src/gc/tests/canonical_keys_holders.rs | 15 +++-- .../object/field_get_set/ic_miss/ic_slow.rs | 20 +++++- crates/perry-runtime/src/string/intern.rs | 66 +++++++------------ 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs b/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs index c70c672600..0656c379f6 100644 --- a/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs +++ b/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs @@ -671,9 +671,9 @@ fn atom_bits(atom: usize) -> u64 { /// S3b: a key text has ONE string object in an agent — its atom — and every /// canonical list written after the atom exists holds it. The atom table holds /// its strings strongly and REWRITES them on a move (the intern-table root -/// scanner), so after a moving minor the table, the list, a fresh intern and a -/// fresh pool mint all name the atom at its NEW address, and none names the -/// address it moved away from. +/// scanner), so after a moving minor the table, the list and a fresh pool mint +/// all name the atom at its NEW address, and none names the address it moved +/// away from. Interning is separate: the intern cache never hands out an atom. #[test] fn an_atom_and_the_lists_holding_it_follow_a_moving_minor() { let _guard = CopyingNurseryTestGuard::new(0); @@ -723,12 +723,13 @@ fn an_atom_and_the_lists_holding_it_follow_a_moving_minor() { atom_bits(moved), "INVARIANT: the list follows its atom through the move" ); - // After the move: interning another copy, and minting again, both - // return the moved atom — never a third string. - assert_eq!( + // After the move: minting again returns the moved atom — never a third + // string. Interning a copy does NOT: an atom is identity, not + // eligibility (`string::intern::AtomTable`). + assert_ne!( crate::string::js_string_intern(nursery_key("atom_mv_kind"), hash) as usize, moved, - "a copy interns to the moved atom" + "INVARIANT: the intern cache never hands out an atom" ); assert_eq!( crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize, diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index f8b7be174e..78c7b67b99 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -704,14 +704,30 @@ mod tests { "INVARIANT: receiver {i}'s shape holds the atom, not the copy it grew with" ); } - // A computed key with the text interns TO the atom. + // Identity is not eligibility: the atom is not an interned string, and + // a computed key with the text interns to its own string, never to the + // atom (`GC_FLAG_INTERNED` admits keys to the own-property lanes; an + // atom must not widen them). + let atom_flags = atom.with_const_ptr(|p: *const crate::StringHeader| unsafe { + (*((p as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader)) + .gc_flags + }); + assert_eq!( + atom_flags & crate::gc::GC_FLAG_INTERNED, + 0, + "INVARIANT: an atom is never flagged interned" + ); let interned = copy.with_const_ptr(|p: *const crate::StringHeader| { crate::string::js_string_intern( p, crate::object::key_bytes_hash(text.as_ptr(), text.len()), ) as usize }); - assert_eq!(interned, addr(&atom), "a copy interns to the atom"); + assert_ne!( + interned, + addr(&atom), + "INVARIANT: interning never returns the atom" + ); // The site holds the atom: every latched read is the receiver's own // value, answered by its shape. let by_atom = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); diff --git a/crates/perry-runtime/src/string/intern.rs b/crates/perry-runtime/src/string/intern.rs index 5083c2e22c..792ab0e34c 100644 --- a/crates/perry-runtime/src/string/intern.rs +++ b/crates/perry-runtime/src/string/intern.rs @@ -57,10 +57,19 @@ crate::perry_thread_local! { /// collector rewrites the entries on move through the intern-table root /// scanner (`scan_intern_table_roots_mut`), exactly like the cache's. /// -/// Two funnels consult it: the intern cache's miss paths (so a computed key -/// with an atom's text interns TO the atom), and canonical key lists when they -/// write a key (`canonical_keys::Appended::atomized`), so a read site's pooled -/// key and the receiver's shape key are one pointer. +/// One funnel consults it: canonical key lists when they write a key +/// (`canonical_keys::Appended::atomized`), so a read site's pooled key and the +/// receiver's shape key are one pointer. +/// +/// An atom is NOT an interned string. It is minted by a plain allocation and +/// never carries `GC_FLAG_INTERNED`, and the intern cache neither adopts nor +/// hands out atoms. `GC_FLAG_INTERNED` is an ELIGIBILITY bit: the own-property +/// read lane, the set fast paths, the chain store and the proxy put paths +/// admit only interned keys. Minting atoms as interned strings silently widened +/// every one of those lanes to every pool-literal key, and on Zod the widened +/// read lane MISSES (the key is inherited, not own) at ~330 instructions each: +/// +0.3% instructions, measured, with the atom table itself inert. Identity is +/// the atom's job; eligibility stays exactly what it was. /// /// The table never decides an answer. A pointer match proves equal text; a /// pointer MISmatch proves nothing (a list written before its atom existed @@ -205,9 +214,10 @@ pub(crate) unsafe fn atom_for_key( /// precomputed FNV-1a hash of the bytes — the same function as every other /// key hash here. /// -/// Adopts the intern cache's string when it already holds this text, so the -/// keys runtime code interned before this module initialised keep matching. -/// Longer literals are not keys worth an atom and take the plain allocation. +/// A plain allocation, exactly what the pool minted before atoms existed: the +/// atom is not interned and does not adopt the intern cache's string (see +/// `AtomTable`: identity, never eligibility). Longer literals are not keys +/// worth an atom and take the plain allocation without a table entry. #[no_mangle] pub extern "C" fn js_string_pool_atom( bytes: *const u8, @@ -226,20 +236,18 @@ pub extern "C" fn js_string_pool_atom( if let Some(atom) = atom_lookup(input, hash) { return atom as *mut StringHeader; } - // Finds the cache's string for this text or allocates one, marking it - // interned and immutable (`refcount = 0`). Nothing is held across the - // allocation: `bytes` is read-only data in the compiled image. - let atom = intern_dispatch_bytes(0, bytes, len as usize, 0, is_wtf8 != 0); - if atom.is_null() { - return js_string_from_bytes(bytes, len); - } + // Nothing is held across the allocation: `bytes` is read-only data in the + // compiled image. Shared (`refcount = 0`) like every pool literal. + let atom: *const StringHeader = if is_wtf8 != 0 { + js_string_from_wtf8_bytes(bytes, len) + } else { + js_string_from_bytes(bytes, len) + }; let _ = ATOMS.try_with(|t| unsafe { (*t.get()).insert(hash, atom) }); atom as *mut StringHeader } -/// Test hook: evict `bytes` from the intern CACHE (a collision would), so the -/// next atom mint for that text allocates a new string instead of adopting -/// the cached one. +/// Test hook: evict `bytes` from the intern CACHE (a collision would). #[cfg(test)] pub(crate) fn test_evict_interned(bytes: &[u8]) { let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); @@ -300,20 +308,6 @@ pub extern "C" fn js_string_intern(key: *const StringHeader, hash: u64) -> *cons return existing; } - // An atom owns this text: the cache slot takes the ATOM, so a computed - // key interns to the same string a read site and a shape key list hold - // (see `AtomTable`), never to a second copy. - let bytes = std::slice::from_raw_parts(string_data(key), byte_len as usize); - if let Some(atom) = atom_lookup(bytes, hash) { - with_intern_table(|table| { - (*table)[slot] = InternEntry { - hash, - string_ptr: atom as usize, - }; - }); - return atom; - } - // Miss or collision — insert (evict on collision) with_intern_table(|table| { (*table)[slot] = InternEntry { @@ -386,16 +380,6 @@ pub(crate) fn intern_dispatch_bytes( if let Some(existing) = hit { return existing; } - // An atom owns this text (see `AtomTable`): hand it out rather than a copy. - if let Some(atom) = atom_lookup(input, hash) { - with_intern_table(|table| unsafe { - (*table)[slot] = InternEntry { - hash, - string_ptr: atom as usize, - }; - }); - return atom; - } let key = if is_wtf8 { js_string_from_wtf8_bytes(bytes, byte_len as u32) From e8e53077919317c6aecabc20d2cf01b29de242b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 01:43:57 +0000 Subject: [PATCH 4/8] docs(changelog): megamorphic reads confirm the slot guess by key atom --- changelog.d/megamorphic-read-key-atoms.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog.d/megamorphic-read-key-atoms.md diff --git a/changelog.d/megamorphic-read-key-atoms.md b/changelog.d/megamorphic-read-key-atoms.md new file mode 100644 index 0000000000..26ae826a5c --- /dev/null +++ b/changelog.d/megamorphic-read-key-atoms.md @@ -0,0 +1,5 @@ +Megamorphic property reads confirm the site's last slot against the receiver's +shape with one pointer compare: property-key literals are now one string per +key text (key atoms), and shape key lists hold that string. A 40-shape +`o.kind` read drops from ~300 to ~140 instructions. Atoms do not change which +keys count as interned. From d98497fd1c1b45b8b3c26b20b0d7a925132cc1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 15:36:36 +0200 Subject: [PATCH 5/8] changelog: name the fragment after PR #11633 --- ...phic-read-key-atoms.md => 11633-megamorphic-read-key-atoms.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{megamorphic-read-key-atoms.md => 11633-megamorphic-read-key-atoms.md} (100%) diff --git a/changelog.d/megamorphic-read-key-atoms.md b/changelog.d/11633-megamorphic-read-key-atoms.md similarity index 100% rename from changelog.d/megamorphic-read-key-atoms.md rename to changelog.d/11633-megamorphic-read-key-atoms.md From 71406b4953362ac546db7793f6aa211aefb0a356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 16:24:11 +0200 Subject: [PATCH 6/8] fix(runtime): an SSO key slot is its own atom; say so in code for the SSO unbox inventory atomized() replaced heap-string key slots with their atom and left every other slot alone. That was correct for short (SSO) strings, whose bits are their identity, but only implicitly, so the SSO unbox inventory (#11627) counted it as a new heap-only string reader. The SSO arm is now explicit. --- crates/perry-runtime/src/object/canonical_keys.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/perry-runtime/src/object/canonical_keys.rs b/crates/perry-runtime/src/object/canonical_keys.rs index 00704f8880..1a826ae6ec 100644 --- a/crates/perry-runtime/src/object/canonical_keys.rs +++ b/crates/perry-runtime/src/object/canonical_keys.rs @@ -697,6 +697,10 @@ impl Appended { Some(atom) => Appended::Key(atom), None => self, }, + // An SSO short string carries its bytes in the value itself, so + // its bits ARE its identity: equal texts are already equal words + // and there is no heap string to replace with an atom. + Appended::Slot(v) if v.is_short_string() => self, Appended::Slot(v) if v.is_string() => { match crate::string::atom_for_key(v.as_string_ptr(), h) { Some(atom) => Appended::Slot(JSValue::string_ptr(atom as *mut StringHeader)), From 805b222678243498e3e72905c50a31f6f8e7b6fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 21:07:35 +0000 Subject: [PATCH 7/8] regen: js_string_pool_atom in the wasm ABI table and the linux gc-call-effects table --- crates/perry-codegen/src/gc_effects/linux-x86_64.tsv | 1 + crates/perry-codegen/src/wasm32/runtime_abi.tsv | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv index b71c7ad644..6625c52a2a 100644 --- a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv @@ -3128,6 +3128,7 @@ js_string_normalize Reenters js_string_pad_end Reenters js_string_pad_fill Reenters js_string_pad_start Reenters +js_string_pool_atom Reenters js_string_position_to_index Leaf js_string_print Leaf js_string_raw Reenters diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index 059313f6b0..c3e2693778 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -3616,6 +3616,7 @@ js_string_normalize ptr ptr,f64 js_string_pad_end ptr ptr,f64,ptr js_string_pad_fill ptr f64 js_string_pad_start ptr ptr,f64,ptr +js_string_pool_atom ptr ptr,i32u,i64,i32s js_string_position_to_index i32s f64 js_string_print void ptr js_string_raw ptr f64,f64 From 5bfb01c7839842241f1412250c01a0f44385f0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 28 Sep 2026 21:14:37 +0000 Subject: [PATCH 8/8] test(runtime): atoms survive a moving minor via the atom young log --- .../src/gc/tests/minor_fixed_cost.rs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs b/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs index 008f666f7b..b62f32743e 100644 --- a/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs +++ b/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs @@ -178,3 +178,44 @@ fn small_int_cache_writer_publishing_a_young_string_is_caught() { crate::string::test_write_small_int_cache_slot(255, young); crate::string::debug_assert_small_string_caches_not_minor_relevant(); } + +/// An atom minted from a young string and reachable ONLY through the atom +/// table survives a moving minor: the table names the forwarded, live copy +/// (found again by text and by `atom_for_key`), not from-space. The atom +/// young log is what makes the minor visit that slot. +#[test] +fn young_atom_is_rewritten_through_the_atom_young_log() { + let _guard = CopyingNurseryTestGuard::new(0); + let _clear = ClearTablesOnDrop; + gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut); + crate::string::test_clear_intern_table(); + + let bytes = b"minor-fixed-cost-young-atom"; + let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); + let young = crate::string::js_string_pool_atom(bytes.as_ptr(), bytes.len() as u32, hash, 0); + assert!(crate::arena::pointer_in_nursery(young as usize)); + assert_eq!( + crate::string::atom_lookup(bytes, hash), + Some(young as *const _) + ); + + let _ = gc_collect_minor(); + + let tabled = crate::string::atom_lookup(bytes, hash).expect("the atom must stay tabled"); + assert_ne!( + tabled as usize, young as usize, + "the table must name the evacuated copy, not from-space" + ); + unsafe { + assert_string_bytes(tabled, bytes); + let fresh = crate::string::js_string_pool_atom(bytes.as_ptr(), bytes.len() as u32, hash, 0); + assert_eq!( + fresh as usize, tabled as usize, + "the pool must reuse the live atom" + ); + assert!(crate::string::is_atom_for_test(tabled)); + // A second string with the same text resolves to the same atom. + let other = crate::string::js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32); + assert_eq!(crate::string::atom_for_key(other, hash), Some(tabled)); + } +}