diff --git a/changelog.d/11633-megamorphic-read-key-atoms.md b/changelog.d/11633-megamorphic-read-key-atoms.md new file mode 100644 index 0000000000..26ae826a5c --- /dev/null +++ b/changelog.d/11633-megamorphic-read-key-atoms.md @@ -0,0 +1,5 @@ +Megamorphic property reads confirm the site's last slot against the receiver's +shape with one pointer compare: property-key literals are now one string per +key text (key atoms), and shape key lists hold that string. A 40-shape +`o.kind` read drops from ~300 to ~140 instructions. Atoms do not change which +keys count as interned. diff --git a/crates/perry-codegen/src/codegen/string_pool.rs b/crates/perry-codegen/src/codegen/string_pool.rs index 079c4ce0ec..5c82c68741 100644 --- a/crates/perry-codegen/src/codegen/string_pool.rs +++ b/crates/perry-codegen/src/codegen/string_pool.rs @@ -96,6 +96,13 @@ impl<'a> InitChunker<'a> { } } +/// Pool literals at most this long are minted as ATOMS. **Must equal +/// `INTERN_MAX_BYTE_LEN` in `perry-runtime/src/string/intern.rs`**, the +/// longest key the runtime interns; `js_string_pool_atom` falls back to a +/// plain allocation past it, so a mismatch costs only the atom, never +/// correctness. +pub(crate) const POOL_ATOM_MAX_BYTE_LEN: usize = 64; + /// Emit the string pool into the module: byte-array constants, handle /// globals, and the `__perry_init_strings_` function that /// allocates + NaN-boxes + GC-roots each handle exactly once at startup. @@ -446,12 +453,30 @@ pub(super) fn emit_string_pool( let bytes_ref = format!("@{}", entry.bytes_global); let handle_ref = format!("@{}", entry.handle_global); let len_str = entry.byte_len.to_string(); - let from_bytes_fn = if entry.is_wtf8 { - "js_string_from_wtf8_bytes" + // A literal short enough to be a property key becomes its text's ATOM + // (`js_string_pool_atom`): one string object per key text for the + // whole agent, shared by every module's pool, every canonical shape + // key list and every intern hit — so a read site's key and the + // receiver's shape key compare by pointer (S3b). Longer literals, and + // WTF-8 ones (lone surrogates: never an identifier key), keep the + // plain allocation. + let atomize = + !entry.is_wtf8 && entry.byte_len > 0 && entry.byte_len <= POOL_ATOM_MAX_BYTE_LEN; + let handle = if atomize { + let hash = crate::nanbox::i64_literal(entry.dispatch_hash); + blk.call( + I64, + "js_string_pool_atom", + &[(PTR, &bytes_ref), (I32, &len_str), (I64, &hash), (I32, "0")], + ) } else { - "js_string_from_bytes" + let from_bytes_fn = if entry.is_wtf8 { + "js_string_from_wtf8_bytes" + } else { + "js_string_from_bytes" + }; + blk.call(I64, from_bytes_fn, &[(PTR, &bytes_ref), (I32, &len_str)]) }; - let handle = blk.call(I64, from_bytes_fn, &[(PTR, &bytes_ref), (I32, &len_str)]); let nanboxed = blk.call(DOUBLE, "js_nanbox_string", &[(I64, &handle)]); // Plain store, no remembered-set write barrier: the handle slot is // registered as a permanent global root on the very next line (always diff --git a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv index b71c7ad644..0a6eeb9e53 100644 --- a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv @@ -3029,6 +3029,7 @@ js_sqlite_stmt_get Reenters js_sqlite_stmt_raw Reenters js_sqlite_stmt_run Reenters js_sqlite_transaction Reenters +js_stack_overflow Reenters js_state_get Reenters js_state_init Reenters js_state_set Reenters @@ -3128,6 +3129,7 @@ js_string_normalize Reenters js_string_pad_end Reenters js_string_pad_fill Reenters js_string_pad_start Reenters +js_string_pool_atom Reenters js_string_position_to_index Leaf js_string_print Leaf js_string_raw Reenters diff --git a/crates/perry-codegen/src/runtime_decls/mod.rs b/crates/perry-codegen/src/runtime_decls/mod.rs index 72d544d6fc..9d9de7474a 100644 --- a/crates/perry-codegen/src/runtime_decls/mod.rs +++ b/crates/perry-codegen/src/runtime_decls/mod.rs @@ -159,6 +159,10 @@ pub fn declare_phase1(module: &mut LlModule) { // Strings (enough to produce string literals for later phases). module.declare_function("js_string_from_bytes", I64, &[PTR, I32]); module.declare_function("js_string_from_wtf8_bytes", I64, &[PTR, I32]); + // S3b: a pooled literal short enough to be a key is minted as the atom of + // its text (`string/intern.rs::js_string_pool_atom`): bytes, len, FNV-1a + // hash, is_wtf8. + module.declare_function("js_string_pool_atom", I64, &[PTR, I32, I64, I32]); // Type checks. module.declare_function("js_is_truthy", I32, &[DOUBLE]); diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index 0d0eafedbf..9f204dec05 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -3617,6 +3617,7 @@ js_string_normalize ptr ptr,f64 js_string_pad_end ptr ptr,f64,ptr js_string_pad_fill ptr f64 js_string_pad_start ptr ptr,f64,ptr +js_string_pool_atom ptr ptr,i32u,i64,i32s js_string_position_to_index i32s f64 js_string_print void ptr js_string_raw ptr f64,f64 diff --git a/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs b/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs index e1a3e51b23..0656c379f6 100644 --- a/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs +++ b/crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs @@ -657,3 +657,140 @@ fn a_dead_weak_keys_entry_is_dropped_before_its_storage_is_reused() { .remove(&DEAD_MEMO_CLASS_ID); canonical_keys::reset_for_test(); } + +// ------------------------------------------------------- S3b: key atoms + +fn atom_hash(text: &[u8]) -> u64 { + crate::object::key_bytes_hash(text.as_ptr(), text.len()) +} + +fn atom_bits(atom: usize) -> u64 { + crate::value::js_nanbox_string(atom as i64).to_bits() +} + +/// S3b: a key text has ONE string object in an agent — its atom — and every +/// canonical list written after the atom exists holds it. The atom table holds +/// its strings strongly and REWRITES them on a move (the intern-table root +/// scanner), so after a moving minor the table, the list and a fresh pool mint +/// all name the atom at its NEW address, and none names the address it moved +/// away from. Interning is separate: the intern cache never hands out an atom. +#[test] +fn an_atom_and_the_lists_holding_it_follow_a_moving_minor() { + let _guard = CopyingNurseryTestGuard::new(0); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + register_object_model_scanners(); + gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut); + canonical_keys::reset_for_test(); + let scope = RuntimeHandleScope::new(); + let text = b"atom_mv_kind"; + let hash = atom_hash(text); + unsafe { + let atom = + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize; + assert!( + crate::arena::pointer_in_nursery(atom), + "premise: the atom is young, so a minor can move it" + ); + // A receiver grows the key through a DIFFERENT string with the text. + let copy = nursery_key("atom_mv_kind"); + assert_ne!(copy as usize, atom, "premise: two string objects, one text"); + let o = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0)); + set(o, nursery_key("atom_mv_a"), 1.0); + set(o, copy, 2.0); + let list = keys_of(o); + assert_eq!( + crate::array::js_array_get(list, 1).bits(), + atom_bits(atom), + "INVARIANT: the written list holds the atom, not the grower's copy" + ); + + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert!( + trace.copying_nursery.copied_objects > 0, + "premise: the minor copied" + ); + let moved = crate::string::atom_lookup(text, hash).expect("the atom survives") as usize; + assert_ne!(moved, atom, "premise: the minor moved the atom"); + let header = crate::value::addr_class::try_read_tracked_gc_header(moved) + .expect("the moved atom is a tracked cell"); + assert_eq!( + (*header.as_ptr()).gc_flags & GC_FLAG_FORWARDED, + 0, + "INVARIANT: the table names the live copy, not a forwarding header" + ); + assert_eq!( + crate::array::js_array_get(keys_of(o), 1).bits(), + atom_bits(moved), + "INVARIANT: the list follows its atom through the move" + ); + // After the move: minting again returns the moved atom — never a third + // string. Interning a copy does NOT: an atom is identity, not + // eligibility (`string::intern::AtomTable`). + assert_ne!( + crate::string::js_string_intern(nursery_key("atom_mv_kind"), hash) as usize, + moved, + "INVARIANT: the intern cache never hands out an atom" + ); + assert_eq!( + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize, + moved, + "a second mint finds the moved atom" + ); + // A list written after the move holds the moved atom. + let o2 = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0)); + set(o2, nursery_key("atom_mv_b"), 1.0); + set(o2, nursery_key("atom_mv_kind"), 2.0); + assert_eq!( + crate::array::js_array_get(keys_of(o2), 1).bits(), + atom_bits(moved), + "INVARIANT: a list written after the move holds the moved atom" + ); + } +} + +/// S3b, a collection DURING the write: the canonical backing allocation that +/// publishes a list holding an atom is itself the collection point, and moves +/// the atom. The published list must hold the atom's LIVE address. +#[test] +fn a_list_written_while_its_atom_moves_holds_the_live_atom() { + let _guard = CopyingNurseryTestGuard::new(0); + let _pacing = crate::gc::policy::force_alloc_point_minor_pacing(); + let _scan = NoConservativeScan::new(); + let trigger = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + register_object_model_scanners(); + gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut); + canonical_keys::reset_for_test(); + let text = b"atom_during_kind"; + let hash = atom_hash(text); + unsafe { + let atom = + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize; + assert!( + crate::arena::pointer_in_nursery(atom), + "premise: the atom is young" + ); + let scope = RuntimeHandleScope::new(); + let copy = scope.root_string_ptr(nursery_key("atom_during_kind")); + // The empty list plus this key is a fresh backing: its allocation is + // the collection point. + arm_collection_on_next_block(&trigger); + let list = copy.with_const_ptr(|k: *const crate::StringHeader| { + canonical_keys::extend_key( + &SharedLayout::shape_cache_entry(), + canonical_keys::CanonicalKeys::EMPTY, + k, + ) + }); + let live = crate::string::atom_lookup(text, hash).expect("the atom survives") as usize; + assert_ne!( + live, atom, + "premise: the backing allocation did not move the atom, so this run proved \ + nothing. Check the trigger arming." + ); + assert_eq!( + crate::array::js_array_get(list.as_ptr(), 0).bits(), + atom_bits(live), + "INVARIANT: the list holds the atom's live address" + ); + } +} diff --git a/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs b/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs index 008f666f7b..b62f32743e 100644 --- a/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs +++ b/crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs @@ -178,3 +178,44 @@ fn small_int_cache_writer_publishing_a_young_string_is_caught() { crate::string::test_write_small_int_cache_slot(255, young); crate::string::debug_assert_small_string_caches_not_minor_relevant(); } + +/// An atom minted from a young string and reachable ONLY through the atom +/// table survives a moving minor: the table names the forwarded, live copy +/// (found again by text and by `atom_for_key`), not from-space. The atom +/// young log is what makes the minor visit that slot. +#[test] +fn young_atom_is_rewritten_through_the_atom_young_log() { + let _guard = CopyingNurseryTestGuard::new(0); + let _clear = ClearTablesOnDrop; + gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut); + crate::string::test_clear_intern_table(); + + let bytes = b"minor-fixed-cost-young-atom"; + let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); + let young = crate::string::js_string_pool_atom(bytes.as_ptr(), bytes.len() as u32, hash, 0); + assert!(crate::arena::pointer_in_nursery(young as usize)); + assert_eq!( + crate::string::atom_lookup(bytes, hash), + Some(young as *const _) + ); + + let _ = gc_collect_minor(); + + let tabled = crate::string::atom_lookup(bytes, hash).expect("the atom must stay tabled"); + assert_ne!( + tabled as usize, young as usize, + "the table must name the evacuated copy, not from-space" + ); + unsafe { + assert_string_bytes(tabled, bytes); + let fresh = crate::string::js_string_pool_atom(bytes.as_ptr(), bytes.len() as u32, hash, 0); + assert_eq!( + fresh as usize, tabled as usize, + "the pool must reuse the live atom" + ); + assert!(crate::string::is_atom_for_test(tabled)); + // A second string with the same text resolves to the same atom. + let other = crate::string::js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32); + assert_eq!(crate::string::atom_for_key(other, hash), Some(tabled)); + } +} diff --git a/crates/perry-runtime/src/object/canonical_keys.rs b/crates/perry-runtime/src/object/canonical_keys.rs index c3f8a6c164..1a826ae6ec 100644 --- a/crates/perry-runtime/src/object/canonical_keys.rs +++ b/crates/perry-runtime/src/object/canonical_keys.rs @@ -676,6 +676,41 @@ impl Appended { } } + /// The same slot, spelled with its text's ATOM when one exists — the one + /// string a read site's pooled key also is (`string::intern::AtomTable`). + /// + /// Applied to every key this module WRITES into a list, and nowhere else: + /// the trie validates edges by bytes, so which string object a list holds + /// never changes which node a probe reaches, only whether a later key + /// compare against the list can stop at pointer equality. Heap strings + /// only — an SSO slot stays an SSO slot, so `is_pointer` (and with it the + /// backing's all-pointer layout) is unchanged by the substitution. + /// + /// `h` is this slot's `edge_hash`, which for a string IS the FNV-1a hash of + /// its bytes — the atom table's hash. + /// + /// # Safety + /// The operand is live. + unsafe fn atomized(self, h: u64) -> Self { + match self { + Appended::Key(key) if !key.is_null() => match crate::string::atom_for_key(key, h) { + Some(atom) => Appended::Key(atom), + None => self, + }, + // An SSO short string carries its bytes in the value itself, so + // its bits ARE its identity: equal texts are already equal words + // and there is no heap string to replace with an atom. + Appended::Slot(v) if v.is_short_string() => self, + Appended::Slot(v) if v.is_string() => { + match crate::string::atom_for_key(v.as_string_ptr(), h) { + Some(atom) => Appended::Slot(JSValue::string_ptr(atom as *mut StringHeader)), + None => self, + } + } + _ => self, + } + } + /// Is the appended slot a heap string POINTER? An SSO short string and a /// tombstone are not, and either one costs the child its all-pointer /// layout — see `Node::all_ptr`. @@ -789,6 +824,8 @@ pub(crate) unsafe fn extend_slot( if let Some(hit) = probe(parent, parent_len, appended, entry, h) { return hit; } + // A new list is about to be WRITTEN: it holds the atom of this key's text. + let appended = appended.atomized(appended.slot_hash()); // Whether the child's slots are all heap string pointers is the parent's // answer AND this slot's, so it is read before the allocation and never @@ -1180,6 +1217,20 @@ pub(crate) unsafe fn canonicalize( // GC_STORE_AUDIT(INIT): fresh is unpublished; publish length only after // all elements are initialized, with no intervening GC allocation. std::ptr::copy_nonoverlapping(slots, dst, len as usize); + // Every heap key of the new list is its text's atom where one exists (see + // `Appended::atomized`). Heap string to heap string, so `all_ptr` holds. + for i in 0..len as usize { + let slot = Appended::Slot(JSValue::from_bits((*dst.add(i)).to_bits())); + if let Appended::Slot(v) = slot { + if v.is_string() { + if let Appended::Slot(atom) = slot.atomized(slot.slot_hash()) { + // GC_STORE_AUDIT(INIT): `fresh` is unpublished; its length + // is set on the next line, after the last slot is written. + *dst.add(i) = f64::from_bits(atom.bits()); + } + } + } + } if with_attrs { let attrs = crate::object::key_attrs::keys_attrs(fresh); crate::object::key_attrs::copy_entries(keys, 0, attrs, len); diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 50e821cb2d..78c7b67b99 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -237,6 +237,68 @@ pub extern "C-unwind" fn js_object_get_field_ic_slow( key: *const crate::StringHeader, cache_slot: *mut PicCacheSlot, packed: *const AtomicU64, +) -> f64 { + // --- 0. a MEGAMORPHIC site's slot guess, confirmed by the receiver ------ + // + // A site whose way state is latched negative sees more shapes than any + // per-site entry can name, and every read that misses its compact word + // lands here. The site may still hold one thing: a slot GUESS (the compact + // word's high half — the slot the receiver's shape answered last, step 2b), + // which the RECEIVER'S own shape confirms or refutes + // (`shapes::confirm_slot_guess`: the position bound says key position + // `guess` is inline slot `guess`, and the key there IS this key, one + // pointer compare). Everything the guess cannot answer continues in + // `ic_slow_body` unchanged. Asked first, and only at a latched site, so a + // site that can still be primed is primed exactly as before. `length` is + // excluded as in step 2b: an Array-subclass receiver serves it from its + // elements store. Kept in this frameless entry, with the body out of line, + // so the confirmed read pays no prologue for the arms below. + if let Some(value) = unsafe { megamorphic_slot_guess(obj_handle, key, cache_slot, packed) } { + return value; + } + ic_slow_body(obj_handle, key, cache_slot, packed) +} + +/// Step 0 of [`js_object_get_field_ic_slow`]: the latched site's slot guess. +/// +/// # Safety +/// The entry's contract: a POINTER receiver handle, this site's cache slot +/// and packed word. +#[inline(always)] +unsafe fn megamorphic_slot_guess( + obj_handle: i64, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, +) -> Option { + let obj = obj_handle as usize as *const ObjectHeader; + if packed.is_null() + || key.is_null() + || !crate::value::addr_class::is_above_handle_band(obj as usize) + { + return None; + } + let cache = crate::object::pic_slot_peek(cache_slot); + if cache.is_null() + || (*cache)[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] >= 0 + || key_is_length(key) + { + return None; + } + let guess = ((*packed).load(Ordering::Relaxed) >> 32) as usize; + let value = crate::object::shapes::confirm_slot_guess(obj, key, guess)?; + #[cfg(test)] + crate::object::shapes::SHAPE_ANSWERED_READS.fetch_add(1, Ordering::Relaxed); + Some(value) +} + +/// Everything after step 0 of [`js_object_get_field_ic_slow`]. +#[inline(never)] +fn ic_slow_body( + obj_handle: i64, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, ) -> f64 { let obj = obj_handle as usize as *const ObjectHeader; let addr = obj as usize; @@ -525,6 +587,609 @@ mod tests { } } + /// `megamorphic_receivers` with a chosen text for the read key (slot 2), + /// each receiver growing that key through its OWN freshly allocated string + /// — never an atom — so which string a shape's list ends up holding is + /// decided by the list writer, not by the test. + fn megamorphic_receivers_keyed<'s>( + scope: &'s crate::gc::RuntimeHandleScope, + n: usize, + read_key: &[u8], + extra_prefix: &str, + ) -> Vec> { + let mut out = Vec::new(); + for i in 0..n { + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + for (k, v) in [ + (&b"pos"[..], 1.0), + (&b"end"[..], 2.0), + (read_key, 100.0 + i as f64), + ] { + let key = scope.root_string_ptr(key_of(k)); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + let extra = format!("{extra_prefix}{i}"); + let key = scope.root_string_ptr(key_of(extra.as_bytes())); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, 7.0)) + }); + out.push(obj); + } + out + } + + fn atom_of(text: &[u8]) -> *mut crate::StringHeader { + let hash = crate::object::key_bytes_hash(text.as_ptr(), text.len()); + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) + } + + fn answered() -> u64 { + crate::object::shapes::SHAPE_ANSWERED_READS.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Read `key` at one latched site over `objs` until it latches, then once + /// more; every read must return `expect(i)`. Returns the reads the + /// receivers' SHAPES answered on the latched pass. + fn latched_pass( + objs: &[crate::gc::RuntimeHandle<'_>], + key: &crate::gc::RuntimeHandle<'_>, + expect: impl Fn(usize) -> u64, + ) -> u64 { + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let read = |o: &crate::gc::RuntimeHandle<'_>, slot: &mut PicCacheSlot| { + o.with_mut_ptr(|p: *mut ObjectHeader| { + key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(p), k, slot, &packed)) + }) + }; + for round in 0..4 { + for (i, o) in objs.iter().enumerate() { + assert_eq!( + read(o, &mut slot).to_bits(), + expect(i), + "round {round} receiver {i}" + ); + } + } + assert!( + cache[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] < 0, + "premise: the site latched megamorphic" + ); + let before = answered(); + for (i, o) in objs.iter().enumerate() { + assert_eq!( + read(o, &mut slot).to_bits(), + expect(i), + "latched read, receiver {i}" + ); + } + answered() - before + } + + /// S3b: one key text reaching the runtime as TWO string objects — each + /// receiver grows it through its own fresh copy — is still ONE string in + /// every shape's key list: the text's atom, the same object a read site's + /// pooled key is. So the site matches by pointer; and a site holding yet + /// another copy (not the atom) still gets the right answer by bytes. + #[test] + fn a_key_text_in_two_string_objects_is_one_atom_in_every_shape() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_two_objects_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let copy = scope.root_string_ptr(key_of(text)); + let addr = |h: &crate::gc::RuntimeHandle<'_>| { + h.with_const_ptr(|p: *const crate::StringHeader| p as usize) + }; + assert_ne!( + addr(&atom), + addr(©), + "premise: two string objects, one text" + ); + assert!( + unsafe { crate::string::is_atom_for_test(atom.with_const_ptr(|p| p)) }, + "premise: the pool mint made an atom" + ); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_two_x"); + let atom_bits = crate::value::js_nanbox_string(addr(&atom) as i64).to_bits(); + for (i, o) in objs.iter().enumerate() { + let stored = o.with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::object_keys(p).get(2).bits() + }); + assert_eq!( + stored, atom_bits, + "INVARIANT: receiver {i}'s shape holds the atom, not the copy it grew with" + ); + } + // Identity is not eligibility: the atom is not an interned string, and + // a computed key with the text interns to its own string, never to the + // atom (`GC_FLAG_INTERNED` admits keys to the own-property lanes; an + // atom must not widen them). + let atom_flags = atom.with_const_ptr(|p: *const crate::StringHeader| unsafe { + (*((p as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader)) + .gc_flags + }); + assert_eq!( + atom_flags & crate::gc::GC_FLAG_INTERNED, + 0, + "INVARIANT: an atom is never flagged interned" + ); + let interned = copy.with_const_ptr(|p: *const crate::StringHeader| { + crate::string::js_string_intern( + p, + crate::object::key_bytes_hash(text.as_ptr(), text.len()), + ) as usize + }); + assert_ne!( + interned, + addr(&atom), + "INVARIANT: interning never returns the atom" + ); + // The site holds the atom: every latched read is the receiver's own + // value, answered by its shape. + let by_atom = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + by_atom >= 47, + "the shape answers a pointer-equal key: {by_atom}" + ); + // A site holding a different string object with the same text (not + // the atom): the answer is the same, found by bytes. + let fresh = scope.root_string_ptr(key_of(text)); + let by_bytes = latched_pass(&objs, &fresh, |i| (100.0 + i as f64).to_bits()); + assert!( + by_bytes >= 47, + "a non-atom key text still matches by bytes: {by_bytes}" + ); + } + + /// A list written BEFORE its key's atom existed holds another string. A + /// pointer mismatch is not an answer: the shape still answers by bytes. + #[test] + fn a_list_written_before_the_atom_existed_still_answers() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_pre_atom_kind"; + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_pre_x"); + // The lists hold the string the grow path interned; a collision + // evicts it from the cache before the atom is minted, so the atom is + // a different object. + crate::string::test_evict_interned(text); + let atom = scope.root_string_ptr(atom_of(text)); + let atom_bits = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let stored = objs[0].with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::object_keys(p).get(2).bits() + }); + assert_ne!(stored, atom_bits, "premise: the list predates the atom"); + let n = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!(n >= 47, "the shape answers by bytes: {n}"); + } + + /// Dictionary receivers keep their ShapeId across layout changes, so the + /// shape can never answer for one by position: a latched site reading + /// half-dictionary receivers answers every read correctly, and not one + /// dictionary read is counted as shape-answered. + #[test] + fn a_dictionary_receiver_is_never_answered_by_position() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_dict_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_dict_x"); + let ordinary = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + ordinary >= 47, + "premise: ordinary receivers are shape-answered ({ordinary})" + ); + let mut dict = 0; + for o in objs.iter().step_by(2) { + if o.with_mut_ptr(|p: *mut ObjectHeader| unsafe { + crate::object::dictionary::latch_object_to_dictionary(p) + }) { + dict += 1; + let bound = o.with_const_ptr(|p: *const ObjectHeader| { + crate::object::shapes::test_position_bound_of(p) + }); + assert_eq!(bound, Some(0), "a dictionary shape has no position bound"); + } + } + assert_eq!( + dict, 24, + "premise: every other receiver latched to dictionary" + ); + let n = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + n <= 24, + "INVARIANT: at most the 24 ordinary receivers are shape-answered, got {n}" + ); + } + + /// Tombstones: a receiver whose list carries a HOLE (a tombstone delete) + /// cannot be answered by position — its shape reports no position bound — + /// and every read still returns the receiver's own value; the deleted key + /// reads undefined. + #[test] + fn a_receiver_with_a_tombstoned_key_is_never_answered_by_position() { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + crate::object::delete_rest::test_set_tombstone_deletes(None); + } + } + crate::object::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = Restore; + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3b_tomb_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3b_tomb_x"); + let _ = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + let end = scope.root_string_ptr(key_of(b"end")); + let mut tombstoned = 0; + for (i, o) in objs.iter().enumerate().step_by(3) { + // The first delete of a shared list compacts (it takes ownership); + // the second tombstones in place (`tombstone_tests.rs`). + let extra = format!("s3b_tomb_x{i}"); + let extra = scope.root_string_ptr(key_of(extra.as_bytes())); + for k in [&extra, &end] { + o.with_mut_ptr(|p: *mut ObjectHeader| { + k.with_const_ptr(|kp| crate::object::js_object_delete_field(p, kp)) + }); + } + let (holes, bound) = o.with_const_ptr(|p: *const ObjectHeader| unsafe { + ( + crate::object::shapes::object_shape_hole_count(p), + crate::object::shapes::test_position_bound_of(p), + ) + }); + if holes > 0 { + tombstoned += 1; + assert_eq!( + bound, + Some(0), + "INVARIANT: a tombstoned shape has no position bound" + ); + } + } + assert_eq!( + tombstoned, 16, + "premise: every third receiver carries a tombstone" + ); + let _ = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + let _ = latched_pass(&objs, &end, |i| { + if i % 3 == 0 { + crate::value::TAG_UNDEFINED + } else { + 2.0f64.to_bits() + } + }); + } + + /// The slot-guess confirm for shape `shape_id`, as the megamorphic entry + /// asks it (`shapes::slot_guess_confirmed`): `Some(guess)` when the shape + /// says key position `guess` is inline slot `guess` and holds exactly the + /// key `site_key_bits` names, `None` for a decline. + unsafe fn guess_walk(shape_id: u32, guess: u64, site_key_bits: u64) -> Option { + let key = (site_key_bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader; + crate::object::shapes::slot_guess_confirmed(shape_id, key, guess as usize) + .then_some(guess as usize) + } + + fn stamp_of(o: &crate::gc::RuntimeHandle<'_>) -> u32 { + o.with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::shapes::object_shape_stamp(p) + }) + } + + /// For every ordinary receiver the slot-guess confirm reaches the + /// receiver's own record, confirms the site's guess against the + /// receiver's own key, and names the slot holding the receiver's own + /// value. A guess whose position holds a DIFFERENT key, a guess past the + /// shape's bound, an id outside the ShapeId range and an id whose page + /// was never allocated all decline. + #[test] + fn the_slot_guess_is_confirmed_only_against_the_receivers_own_key() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3c_walk_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let site = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let objs = megamorphic_receivers_keyed(&scope, 48, text, "s3c_walk_x"); + for (i, o) in objs.iter().enumerate() { + let id = stamp_of(o); + let slot = unsafe { guess_walk(id, 2, site) }; + assert_eq!(slot, Some(2), "receiver {i}: the right guess is confirmed"); + let value = o.with_const_ptr(|p: *const ObjectHeader| unsafe { + *((p as *const u8).add(std::mem::size_of::() + 2 * 8) as *const f64) + }); + assert_eq!( + value, + 100.0 + i as f64, + "receiver {i}: the confirmed slot is its own" + ); + for wrong in [0u64, 1, 3] { + assert_eq!( + unsafe { guess_walk(id, wrong, site) }, + None, + "receiver {i}: guess {wrong} holds another key and must decline" + ); + } + assert_eq!(unsafe { guess_walk(id, 4, site) }, None, "past the bound"); + assert_eq!(unsafe { guess_walk(id, u64::from(u32::MAX), site) }, None); + } + assert_eq!( + unsafe { guess_walk(5, 2, site) }, + None, + "a class id is no ShapeId" + ); + assert_eq!( + unsafe { guess_walk(0xBFFF_FFFF, 2, site) }, + None, + "a page never minted" + ); + } + + /// The slot-guess confirm on dictionary and tombstoned receivers: declines, + /// whatever the guess. + #[test] + fn the_slot_guess_never_matches_a_dictionary_or_tombstoned_receiver() { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + crate::object::delete_rest::test_set_tombstone_deletes(None); + } + } + crate::object::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = Restore; + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3c_walk2_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let site = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let objs = megamorphic_receivers_keyed(&scope, 8, text, "s3c_walk2_x"); + let dict = &objs[0]; + assert!( + dict.with_mut_ptr(|p: *mut ObjectHeader| unsafe { + crate::object::dictionary::latch_object_to_dictionary(p) + }), + "premise: the receiver latched to dictionary" + ); + for guess in 0..4 { + assert_eq!( + unsafe { guess_walk(stamp_of(dict), guess, site) }, + None, + "dictionary" + ); + } + let tomb = &objs[1]; + let end = scope.root_string_ptr(key_of(b"end")); + let extra = scope.root_string_ptr(key_of(b"s3c_walk2_x1")); + for k in [&extra, &end] { + tomb.with_mut_ptr(|p: *mut ObjectHeader| { + k.with_const_ptr(|kp| crate::object::js_object_delete_field(p, kp)) + }); + } + let holes = tomb.with_const_ptr(|p: *const ObjectHeader| unsafe { + crate::object::shapes::object_shape_hole_count(p) + }); + assert!(holes > 0, "premise: the receiver carries a tombstone"); + for guess in 0..4 { + assert_eq!( + unsafe { guess_walk(stamp_of(tomb), guess, site) }, + None, + "tombstoned" + ); + } + } + + fn boxed(p: *mut ObjectHeader) -> f64 { + f64::from_bits(0x7FFD_0000_0000_0000 | (p as u64 & 0x0000_FFFF_FFFF_FFFF)) + } + + /// A [[Prototype]] change is a shape transition (the prototype is part of + /// shape identity) that moves no own key and no own slot: receivers built + /// by `new F()` / `setPrototypeOf` before their fields are assigned — every + /// tsc AST node — get a NEW shape with the SAME own-key layout as a + /// prototype-less twin, and the megamorphic read answers them from that + /// shape, by the key-list scan and by the slot-guess confirm. + #[test] + fn a_prototype_change_is_a_new_shape_with_the_same_layout_and_is_answered() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let text = b"s3_proto_kind"; + let atom = scope.root_string_ptr(atom_of(text)); + let site = atom.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + let proto = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 2)); + let build = |with_proto: bool, i: usize| { + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + if with_proto { + let (o, p) = ( + obj.with_mut_ptr(|o: *mut ObjectHeader| o), + proto.with_mut_ptr(|p: *mut ObjectHeader| p), + ); + crate::object::object_ops::js_object_set_prototype_of(boxed(o), boxed(p)); + } + let extra = format!("s3_proto_x{i}"); + for (k, v) in [ + (&b"pos"[..], 1.0), + (&b"end"[..], 2.0), + (&text[..], 100.0 + i as f64), + (extra.as_bytes(), 7.0), + ] { + let key = scope.root_string_ptr(key_of(k)); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + obj + }; + let objs: Vec<_> = (0..48).map(|i| build(true, i)).collect(); + let twin = build(false, 0); + let (shape, twin_shape) = (stamp_of(&objs[0]), stamp_of(&twin)); + assert_ne!( + shape, twin_shape, + "premise: the prototype is part of shape identity" + ); + let d = |id| crate::object::shapes::shape_descriptor_by_id(id).expect("live shape"); + let (d, t) = (d(shape), d(twin_shape)); + assert_eq!( + ( + d.keys, + d.logical_key_count, + d.live_inline_slot_count, + d.semantic_generation + ), + ( + t.keys, + t.logical_key_count, + t.live_inline_slot_count, + t.semantic_generation + ), + "the prototype change moved no own key and bumped no generation" + ); + for (i, o) in objs.iter().enumerate() { + assert_eq!( + unsafe { guess_walk(stamp_of(o), 2, site) }, + Some(2), + "INVARIANT: the guess confirm answers prototype-linked receiver {i}" + ); + } + let n = latched_pass(&objs, &atom, |i| (100.0 + i as f64).to_bits()); + assert!( + n >= 47, + "INVARIANT: prototype-linked receivers are shape-answered: {n}" + ); + } + + /// A shape over a SHIFTED keys array (a front offset): the slot-guess + /// confirm reads LOGICAL key position `i`, past the front offset, so the + /// guess for the key now at logical 0 is confirmed at 0, and the physical + /// position it used to occupy (1) is not. + #[test] + fn a_shape_over_a_shifted_keys_array_is_answered_by_logical_position() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let arr = scope.root_raw_mut_ptr(crate::array::js_array_alloc(4)); + let mut keys = Vec::new(); + for name in [&b"sh_a"[..], b"sh_b", b"sh_c"] { + let k = scope.root_string_ptr(key_of(name)); + let bits = k.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + arr.with_mut_ptr(|a| crate::array::js_array_push(a, crate::JSValue::from_bits(bits))); + keys.push(k); + } + let a = arr.with_mut_ptr(|a: *mut crate::array::ArrayHeader| a); + let bound = + |id: u32| crate::object::shapes::test_position_bound_of_id(id).expect("a live shape"); + let flat = crate::object::shapes::shape_descriptor_ensure(a, 3, 3).expect("mints"); + assert_eq!(bound(flat), 3, "premise: an unshifted keys array"); + crate::array::js_array_shift_f64(a); + assert_ne!( + unsafe { crate::array::array_front_offset(a) }, + 0, + "premise: shift left a front offset" + ); + let shifted = crate::object::shapes::shape_descriptor_ensure(a, 2, 2).expect("mints"); + assert_eq!(bound(shifted), 2, "premise: the shifted shape has two keys"); + let sh_b = keys[1].with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64).to_bits() + }); + assert_eq!( + unsafe { guess_walk(shifted, 0, sh_b) }, + Some(0), + "INVARIANT: the confirm reads the logical position" + ); + assert_eq!( + unsafe { guess_walk(shifted, 1, sh_b) }, + None, + "INVARIANT: the physical position is not a key position" + ); + } + + /// The positional bit is a stored FACT of the shape record, read on every + /// latched miss; it must equal its definition for every record the agent + /// has minted. Mints ordinary, dictionary, tombstoned, accessor and + /// class-keyed shapes, then walks the whole slab. + #[test] + fn every_minted_records_positional_bit_matches_its_facts() { + struct Restore; + impl Drop for Restore { + fn drop(&mut self) { + crate::object::delete_rest::test_set_tombstone_deletes(None); + } + } + crate::object::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = Restore; + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let objs = megamorphic_receivers_keyed(&scope, 24, b"pos_fact_kind", "pos_fact_x"); + // Tombstones: the in-place stable-tombstone update rewrites the hole + // count (two deletes: the first compacts, the second tombstones). + let end = scope.root_string_ptr(key_of(b"end")); + for (i, o) in objs.iter().enumerate().step_by(3) { + let extra = format!("pos_fact_x{i}"); + let extra = scope.root_string_ptr(key_of(extra.as_bytes())); + for k in [&extra, &end] { + o.with_mut_ptr(|p: *mut ObjectHeader| { + k.with_const_ptr(|kp| crate::object::js_object_delete_field(p, kp)) + }); + } + } + // Dictionaries. + for o in objs.iter().skip(1).step_by(3) { + o.with_mut_ptr(|p: *mut ObjectHeader| unsafe { + crate::object::dictionary::latch_object_to_dictionary(p) + }); + } + // Keep the tombstoned receivers growing: re-adds take the cached + // stable-tombstone update. + let again = scope.root_string_ptr(key_of(b"pos_fact_again")); + for o in objs.iter().step_by(3) { + o.with_mut_ptr(|p| { + again.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(p, kp, 3.0)) + }); + } + // Accessor keys: the summary is an input of the bit. + let acc = scope.root_string_ptr(key_of(b"pos_fact_acc")); + for o in objs.iter().skip(2).step_by(3) { + let (ov, kv) = ( + o.with_const_ptr(|p: *const ObjectHeader| { + crate::value::js_nanbox_pointer(p as i64) + }), + acc.with_const_ptr(|p: *const crate::StringHeader| { + crate::value::js_nanbox_string(p as i64) + }), + ); + let undef = f64::from_bits(crate::value::TAG_UNDEFINED); + crate::object::js_object_define_accessor(ov, kv, undef, undef); + } + let (n, positional, accessor, bad) = crate::object::shapes::test_positional_census(); + assert!( + positional > 0 && positional < n, + "premise: the slab holds both answerable and unanswerable shapes ({positional} of {n})" + ); + assert!( + accessor > 0, + "premise: an ordinary shape with an accessor key was minted" + ); + assert!( + bad.is_empty(), + "INVARIANT: the stored positional bit equals its definition; {} of {n} records disagree: {bad:x?}", + bad.len() + ); + } + /// A plain own data read that has never primed: the entry must fall all the /// way through to the miss handler, answer the field, and leave the site /// primed exactly as the old `js_object_get_field_ic_miss_packed` edge did. diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 87a58d7934..fbbb2d97bb 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -252,19 +252,18 @@ impl ShapeRecordRef { /// shape's own canonical key list — or `None` when the shape cannot answer /// by position alone. /// - /// The position of a key in the keys list is its slot exactly when the - /// shape is `Ordinary`, generation 0 (no descriptor/prototype mutation - /// minted it) and hole-free; a position below `live_inline_slot_count` is - /// an inline slot of every receiver carrying the shape (the invariant the - /// read cache's prime already relies on). The list is bounded by the - /// SHAPE's key count, never the backing's length (#10969: one backing per - /// growth chain). + /// The shape's [`ShapeRecord::position_bound`] says how many leading key + /// positions ARE inline slots of every receiver carrying it (0 for a + /// dictionary, class, descriptor/prototype-generation or tombstoned + /// shape). The list is bounded by that — never by the backing's length + /// (#10969: one backing per growth chain). /// - /// A stored key matches the site's key by identity, or else by (byte - /// length, bytes): a canonical list holds the string its first grower - /// passed, which is usually NOT the read site's pooled literal. The - /// site's slot guess is tried first. Allocation-free, never calls user - /// code. + /// Key compares go identity first: canonical lists hold their text's ATOM + /// (`string::intern::AtomTable`), which is also what a read site's pooled + /// key is, so the site's guess, and then any position, matches by one + /// pointer compare. A byte pass remains for a list written before its + /// atom existed (and for SSO slots) — a pointer MISmatch proves nothing. + /// Allocation-free, never calls user code. #[inline] pub(crate) unsafe fn inline_slot_of_key( self, @@ -272,11 +271,8 @@ impl ShapeRecordRef { hint: usize, ) -> Option { let r = &*self.0.as_ptr(); - if r.object_kind() != ShapeObjectKind::Ordinary - || r.semantic_generation != 0 - || r.hole_count != 0 - || r.keys == 0 - { + let bound = r.position_bound() as usize; + if bound == 0 { return None; } let (slots, len) = @@ -284,14 +280,17 @@ impl ShapeRecordRef { if slots.is_null() { return None; } - let bound = len - .min(r.logical_key_count as usize) - .min(r.live_inline_slot_count as usize); - // The site's slot guess first: the receiver's shape confirms it. - if hint < bound && stored_key_matches(key, (*slots.add(hint)).to_bits()) { + let bound = bound.min(len); + let heap_bits = crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits(); + // The site's slot guess, confirmed by the receiver's own key. + if hint < bound && (*slots.add(hint)).to_bits() == heap_bits { return Some(hint); } - (0..bound).find(|&i| i != hint && stored_key_matches(key, (*slots.add(i)).to_bits())) + // Identity over the whole list before any byte is compared. + if let Some(i) = (0..bound).find(|&i| (*slots.add(i)).to_bits() == heap_bits) { + return Some(i); + } + (0..bound).find(|&i| stored_key_matches(key, (*slots.add(i)).to_bits())) } /// The SPILL position at which this shape stores `key` as an own DATA @@ -336,11 +335,11 @@ impl ShapeRecordRef { } } -/// Does the key-list entry `bits` name `key`? A canonical list holds heap -/// strings of its own (NOT the site's pooled key — measured: every stored key -/// of a literal-born shape is a distinct heap string) or SSO immediates, so a -/// stored key matches by identity, by SSO identity, or by (byte length, -/// bytes). +/// Does the key-list entry `bits` name `key`? A canonical list holds its +/// text's ATOM where one exists (the site's pooled key), but must not be +/// assumed to: a list written before its atom existed holds another heap +/// string, and a slot may be an SSO immediate. So a stored key matches by +/// identity, by SSO identity, or by (byte length, bytes). #[inline] unsafe fn stored_key_matches(key: *const crate::StringHeader, bits: u64) -> bool { if bits == crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits() { @@ -371,6 +370,113 @@ unsafe fn stored_key_matches(key: *const crate::StringHeader, bits: u64) -> bool } } +/// The position bound of `obj`'s shape (S3c), or `None` when its word names +/// no record in this agent. +#[cfg(test)] +pub(crate) fn test_position_bound_of(obj: *const crate::object::ObjectHeader) -> Option { + let id = unsafe { object_shape_stamp(obj) }; + shape_record_by_id(id).map(|r| unsafe { (*r.0.as_ptr()).position_bound() }) +} + +/// Walk every present record of this agent's slab: `(records, positional, +/// ordinary records with an accessor key, disagreements)`, where a disagreement is a record whose stored positional +/// bit differs from [`ShapeRecord::positional_by_facts`]. +#[cfg(test)] +pub(crate) fn test_positional_census() -> (usize, usize, usize, Vec) { + let table = &crate::state::state().shapes; + let (mut n, mut positional, mut accessor, mut bad) = (0usize, 0usize, 0usize, Vec::new()); + table.slab().for_each(|id, p| { + let r = unsafe { &*p }; + if !r.present() { + return; + } + n += 1; + if r.positional_bit() { + positional += 1; + } + if r.object_kind() == ShapeObjectKind::Ordinary + && r.summary() & crate::object::key_attrs::SUMMARY_ACCESSOR != 0 + { + accessor += 1; + } + if r.positional_bit() != r.positional_by_facts() { + bad.push(id); + } + }); + (n, positional, accessor, bad) +} + +/// `(stored positional bit, its definition)` for shape `id`. +#[cfg(test)] +pub(crate) fn test_positional_of_id(id: u32) -> Option<(bool, bool)> { + shape_record_by_id(id).map(|r| unsafe { + let r = &*r.0.as_ptr(); + (r.positional_bit(), r.positional_by_facts()) + }) +} + +/// The position bound of shape `id` (S3c), or `None` when it names no record. +#[cfg(test)] +pub(crate) fn test_position_bound_of_id(id: u32) -> Option { + shape_record_by_id(id).map(|r| unsafe { (*r.0.as_ptr()).position_bound() }) +} + +/// The megamorphic read's slot-guess confirm: when `obj` carries an ordinary +/// ShapeId of this agent whose record says key position `guess` is inline slot +/// `guess` (`position_bound`), and the key AT that position is `key` itself +/// (one pointer compare: canonical lists hold their text's atom), the value in +/// the receiver's slot `guess`. `None` for anything else — a wrong or stale +/// guess, another text, a dictionary/class/descriptor/tombstoned shape, an id +/// that names no record — and the caller takes its ordinary path. +/// +/// The guess decides nothing: the receiver's own shape confirms it or it is +/// ignored. Allocation-free, no user code. +/// +/// # Safety +/// `obj` is a heap pointer above the handle band (its `+4` word is read); +/// `key` is a heap `StringHeader`. +#[inline] +pub(crate) unsafe fn confirm_slot_guess( + obj: *const crate::object::ObjectHeader, + key: *const crate::StringHeader, + guess: usize, +) -> Option { + if !slot_guess_confirmed((*obj).parent_class_id, key, guess) { + return None; + } + Some( + *((obj as *const u8).add(std::mem::size_of::() + guess * 8) + as *const f64), + ) +} + +/// Does shape `shape_id` of this agent store `key` at inline slot `guess`, +/// by position? See [`confirm_slot_guess`]. +/// +/// # Safety +/// `key` is a heap `StringHeader`. +#[inline] +pub(crate) unsafe fn slot_guess_confirmed( + shape_id: u32, + key: *const crate::StringHeader, + guess: usize, +) -> bool { + let record = ShapeSlab::ordinary_record(shape_id); + if record.is_null() { + return false; + } + let r = &*record; + if guess >= r.position_bound() as usize { + return false; + } + // A bound > 0 means the record names a live keys array (the collector + // marks through and rewrites `keys`) holding at least `bound` logical + // keys; logical element `i` sits past the array's front offset. + let arr = r.keys as usize as *const ArrayHeader; + let slots = crate::array::array_elements_ptr(arr) as *const u64; + *slots.add(guess) == crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits() +} + /// Byte equality without a libc call for the short keys property names are. #[inline] unsafe fn bytes_eq(a: *const u8, b: *const u8, n: usize) -> bool { diff --git a/crates/perry-runtime/src/object/shapes_slot_list.rs b/crates/perry-runtime/src/object/shapes_slot_list.rs index 1986eb9763..b286fd9efb 100644 --- a/crates/perry-runtime/src/object/shapes_slot_list.rs +++ b/crates/perry-runtime/src/object/shapes_slot_list.rs @@ -578,6 +578,8 @@ pub(crate) unsafe fn try_update_stable_tombstone_shape_cached( record.logical_key_count = logical_key_count; record.live_inline_slot_count = live_inline_slot_count; record.hole_count = hole_count; + // The hole count is an input of the positional bit. + record.refresh_positional(); super::debug_assert_object_shape_parity(obj); Some(id) } diff --git a/crates/perry-runtime/src/object/shapes_store.rs b/crates/perry-runtime/src/object/shapes_store.rs index acb15198f8..7da3bb851e 100644 --- a/crates/perry-runtime/src/object/shapes_store.rs +++ b/crates/perry-runtime/src/object/shapes_store.rs @@ -72,7 +72,9 @@ pub(crate) struct ShapeRecord { pub(super) hole_count: u32, /// Low 8 bits: the `RECORD_FLAG_*` set. Bits 8-10: the `ShapeObjectKind` /// discriminant. Bits 11-14: the births a keyless birth shape served while - /// tracking its width (#10905). Bit 15: reserved. Bits 16-23: the + /// tracking its width (#10905). Bit 15: ANSWERABLE BY POSITION (see + /// [`ShapeRecord::position_bound`]), derived from the record's own facts. + /// Bits 16-23: the /// attribute SUMMARY byte (`key_attrs::SUMMARY_*`), an identity fact. /// Bits 24-31: the inline width a keyless birth shape's descendants grow /// to (#10905). The two #10905 fields are learned facts of the record, @@ -112,6 +114,10 @@ const RECORD_BIRTHS_MASK: u32 = 0xF << RECORD_BIRTHS_SHIFT; /// #10905 (`shapes_birth_width`): the learned descendant width, bits 24-31. const RECORD_WIDTH_SHIFT: u32 = 24; const RECORD_WIDTH_MASK: u32 = 0xFF << RECORD_WIDTH_SHIFT; +/// Bit 15: the shape answers by position ([`ShapeRecord::position_bound`]). +/// A function of the record's facts, rewritten by +/// [`ShapeRecord::refresh_positional`] wherever an input changes. +const RECORD_POSITIONAL: u32 = 1 << 15; // The fields of `flags_and_kind` are pairwise disjoint. const _: () = { let fields = [ @@ -120,6 +126,7 @@ const _: () = { RECORD_BIRTHS_MASK, RECORD_SUMMARY_MASK, RECORD_WIDTH_MASK, + RECORD_POSITIONAL, ]; let mut i = 0; while i < fields.len() { @@ -193,6 +200,8 @@ impl ShapeRecord { pub(super) fn with_summary(mut self, summary: u8) -> ShapeRecord { self.flags_and_kind = (self.flags_and_kind & !RECORD_SUMMARY_MASK) | (u32::from(summary) << RECORD_SUMMARY_SHIFT); + // The summary is an input of the positional bit (an accessor key). + self.refresh_positional(); self } @@ -253,7 +262,7 @@ impl ShapeRecord { // because `facts_match` compares the full enum. let kind_bits = (object_kind.code() as u32) << RECORD_KIND_SHIFT; debug_assert!(kind_bits & !RECORD_KIND_MASK == 0, "kind does not fit"); - ShapeRecord { + let mut record = ShapeRecord { keys, semantic_generation, proto_id: 0, @@ -262,7 +271,74 @@ impl ShapeRecord { hole_count, flags_and_kind: u32::from(flags) | kind_bits, rep: 0, + }; + record.refresh_positional(); + record + } + + /// How many leading keys of this shape's list sit AT their own inline + /// slot: logical key position `i < bound` IS inline slot `i` of every + /// receiver carrying the shape. 0 when the shape cannot answer by position + /// at all. + /// + /// Whether it can is a FACT OF THE RECORD, stored in bit 15 + /// (`RECORD_POSITIONAL`) and read here with one load: the megamorphic + /// read asks it on every latched miss. [`Self::positional_by_facts`] is + /// its definition; every write of one of its inputs is followed by + /// [`Self::refresh_positional`] (construction, `with_summary`, slab + /// insert, the in-place stable-tombstone update), and debug builds assert + /// the stored bit against the definition on every read. + /// + /// The bound is `min(logical_key_count, live_inline_slot_count)`: a key + /// past the key count is another list's (canonical backings are shared by + /// a growth chain), and one past the live inline count is spilled. + #[inline] + pub(crate) fn position_bound(&self) -> u32 { + debug_assert_eq!( + self.flags_and_kind & RECORD_POSITIONAL != 0, + self.positional_by_facts(), + "the positional bit disagrees with the record's facts: {self:?}" + ); + if self.flags_and_kind & RECORD_POSITIONAL == 0 { + return 0; } + self.logical_key_count.min(self.live_inline_slot_count) + } + + /// The definition of the positional bit. The conjuncts are + /// `js_shape_ordinary_inline_slot_for_key`'s: + /// + /// * `Ordinary` — a class shape's slots are its class layout, and a + /// DICTIONARY shape keeps its id across layout changes, so a dictionary + /// receiver must never be matched by position; + /// * generation 0 — a descriptor/prototype mutation minted this layout; + /// * no tombstones — the answer is only claimed for hole-free lists; + /// * no ACCESSOR key in the attribute summary — an accessor key's slot + /// holds its accessor pair, not a value; + /// * a keys array at all. + #[inline] + pub(super) fn positional_by_facts(&self) -> bool { + self.object_kind() == ShapeObjectKind::Ordinary + && self.semantic_generation == 0 + && self.hole_count == 0 + && self.keys != 0 + && self.summary() & crate::object::key_attrs::SUMMARY_ACCESSOR == 0 + } + + /// Rewrite the positional bit from the record's facts. + #[inline] + pub(super) fn refresh_positional(&mut self) { + if self.positional_by_facts() { + self.flags_and_kind |= RECORD_POSITIONAL; + } else { + self.flags_and_kind &= !RECORD_POSITIONAL; + } + } + + /// The stored positional bit, for the agreement test. + #[cfg(test)] + pub(super) fn positional_bit(&self) -> bool { + self.flags_and_kind & RECORD_POSITIONAL != 0 } /// The same record carrying field representation `rep` (`field_rep`). @@ -497,6 +573,19 @@ fn new_page() -> Page { .unwrap_or_else(|_| unreachable!("page vector has PAGE_LEN slots")) } +/// This thread's ordinary page directory as `(page pointers, page count)`: +/// `ShapeSlab::pages`' element pointer and length, republished after every +/// change to `pages` (`ShapeSlab::publish_dir`) and cleared before the slab +/// is dropped. The megamorphic read's slot-guess confirm +/// ([`ShapeSlab::ordinary_record`]) reads it with one thread-local load +/// instead of resolving the runtime state and walking `record_ptr`. +/// `#[thread_local]` (const, no destructor) rather than `thread_local!`: a +/// late read during thread teardown sees the cleared pair, and the access +/// compiles to one thread-pointer-relative load. +#[thread_local] +static ORDINARY_DIR: std::cell::Cell<(*const Option, usize)> = + std::cell::Cell::new((std::ptr::null(), 0)); + /// The by-id descriptor store. See the module docs. /// Two page directories: ordinary ShapeIds index from `SHAPE_ID_BASE`, and the /// dictionary band (`shapes::DICTIONARY_SHAPE_ID_BASE`) from its own base. One @@ -505,6 +594,14 @@ fn new_page() -> Page { /// that one ~196 KB allocation moved the GC arena's pages relative to the /// page-class table window and cost +2.3% instructions (1.65 M vs 0.20 M /// registered-page misses in `classify_heap_generation`). +impl Drop for ShapeSlab { + fn drop(&mut self) { + if ORDINARY_DIR.get().0 == self.pages.as_ptr() { + ORDINARY_DIR.set((std::ptr::null(), 0)); + } + } +} + pub(crate) struct ShapeSlab { pages: Vec>, dict_pages: Vec>, @@ -619,18 +716,26 @@ impl ShapeSlab { /// Install `record` under `id`, allocating the page and chunk on first /// touch. Returns the record it replaced, if the id was already present. pub(super) fn insert(&mut self, id: u32, mut record: ShapeRecord) -> Option { - let (dict, index) = + let (band, index) = Self::index_of(id).expect("ShapeSlab::insert: id outside the ShapeId range"); record.set(RECORD_FLAG_PRESENT, true); let (page, chunk, slot) = Self::split(index); - let dir = self.dir_mut(dict); + let dir = self.dir_mut(band); if page >= dir.len() { dir.resize_with(page + 1, || None); + // Only the ordinary band is mirrored (`ORDINARY_DIR`). + if band == 0 { + self.publish_dir(); + } } + let dir = self.dir_mut(band); let page = dir[page].get_or_insert_with(new_page); let chunk = page[chunk].get_or_insert_with(new_chunk); let cell = chunk[slot].get_mut(); let previous = cell.present().then_some(*cell); + // A retire-and-reinsert edits facts on a removed copy: the positional + // bit follows them. + record.refresh_positional(); *cell = record; if previous.is_none() { self.len += 1; @@ -718,6 +823,39 @@ impl ShapeSlab { } dir.shrink_to_fit(); } + self.publish_dir(); + } + + /// Publish `pages` for [`Self::ordinary_record`] (see [`ORDINARY_DIR`]). + fn publish_dir(&self) { + ORDINARY_DIR.set((self.pages.as_ptr(), self.pages.len())); + } + + /// The record of ordinary ShapeId `id` in THIS thread's slab, or null — + /// the fast twin of [`Self::record_ptr`] for the megamorphic read: one + /// thread-local load, two dependent directory loads, no `state()`. A + /// dictionary- or exotic-band id indexes past the ordinary directory's + /// length. The + /// record may be absent (`EMPTY`): its position bound is 0. + #[inline(always)] + pub(super) fn ordinary_record(id: u32) -> *const ShapeRecord { + let (pages, len) = ORDINARY_DIR.get(); + let index = id.wrapping_sub(SHAPE_ID_BASE) as usize; + let (page, chunk, slot) = Self::split(index); + if page >= len { + return std::ptr::null(); + } + // SAFETY: `pages` holds `len` entries of this thread's slab, current + // as of the last change to it; nothing here can change it. + unsafe { + let Some(page) = (*pages.add(page)).as_ref() else { + return std::ptr::null(); + }; + match page[chunk].as_ref() { + Some(chunk) => chunk[slot].get(), + None => std::ptr::null(), + } + } } #[cfg(test)] @@ -725,6 +863,7 @@ impl ShapeSlab { self.pages.clear(); self.dict_pages.clear(); self.exotic_pages.clear(); + self.publish_dir(); self.len = 0; } diff --git a/crates/perry-runtime/src/object/tombstone_tests.rs b/crates/perry-runtime/src/object/tombstone_tests.rs index d156c14d22..9e18f364d1 100644 --- a/crates/perry-runtime/src/object/tombstone_tests.rs +++ b/crates/perry-runtime/src/object/tombstone_tests.rs @@ -1118,3 +1118,65 @@ fn stable_tombstone_bound_move_mints_a_new_shape_id() { assert_eq!(after.hole_count, shape.hole_count); } } + +/// The two in-place stable-tombstone updaters rewrite the hole count (and, in +/// the uncached one, the summary) of a live record without reinserting it, +/// and both are inputs of the positional bit (`ShapeRecord::position_bound`). +/// Both refresh the bit, but today no update can FLIP it: the updaters only +/// accept a stable-tombstone receiver's detached record, which carries a +/// private-epoch (nonzero) semantic generation, so it is never positional, +/// hole or not. Driven to zero holes directly, each updater must leave the bit +/// equal to its definition, and the premise that pins the bit false is +/// asserted, so the day an updater admits a generation-0 record this test +/// is where the refresh starts to matter. +#[test] +fn in_place_tombstone_updates_keep_the_positional_bit_equal_to_its_facts() { + super::delete_rest::test_set_tombstone_deletes(Some(true)); + let _restore = scopeguard_tombstone_flag(); + let _global = crate::gc::global_side_table_test_lock(); + unsafe { + for cached in [true, false] { + let name: &[u8] = if cached { + b"posbit_cached" + } else { + b"posbit_plain" + }; + let (obj, shape) = stable_receiver_one_hole(0, name); + let id = super::shapes::object_shape_stamp(obj); + let keys = shape.keys as usize as *mut crate::ArrayHeader; + let updated = if cached { + super::shapes::try_update_stable_tombstone_shape_cached( + obj, + shape, + shape.logical_key_count, + shape.live_inline_slot_count, + 0, + ) + } else { + super::shapes::try_update_stable_tombstone_shape( + obj, + keys, + shape.logical_key_count, + shape.live_inline_slot_count, + 0, + ) + }; + assert_eq!( + updated, + Some(id), + "premise: the updater (cached={cached}) ran in place" + ); + let after = super::shapes::object_shape_descriptor(obj).unwrap(); + assert_eq!(after.hole_count, 0, "premise: the update wrote zero holes"); + assert_ne!( + after.semantic_generation, 0, + "premise: an in-place-updatable record is a private epoch (nonzero generation)" + ); + let (bit, facts) = super::shapes::test_positional_of_id(id).unwrap(); + assert_eq!( + bit, facts, + "INVARIANT: the in-place update (cached={cached}) left the positional bit stale" + ); + } + } +} diff --git a/crates/perry-runtime/src/string/intern.rs b/crates/perry-runtime/src/string/intern.rs index f67b50e49d..093d097cec 100644 --- a/crates/perry-runtime/src/string/intern.rs +++ b/crates/perry-runtime/src/string/intern.rs @@ -36,6 +36,315 @@ crate::perry_thread_local! { std::cell::UnsafeCell::new(crate::zeroed_cache::new_zeroed_cache(INTERN_TABLE_SIZE)); } +/// Property-key ATOMS: exactly one string object per key text among the +/// program's pooled literals. +/// +/// The direct-mapped table above is a CACHE — a collision evicts — so two +/// strings with one text can both be "interned" at different moments, and a +/// key compare can never conclude anything from pointer equality alone. That +/// is why every runtime key match, and the megamorphic read's confirm against +/// the receiver's key list, paid a byte compare: a canonical key list held the +/// string its first grower happened to pass, and a read site holds its +/// module's pooled literal, and the two were different objects with the same +/// bytes. +/// +/// An atom is the one string for its text in this agent, for the agent's +/// lifetime. Atoms are minted only from the compiled program's string pools +/// (`js_string_pool_atom`, at module init), so the table is bounded by the +/// program TEXT, never by runtime data, and it can hold its strings strongly +/// without a death prune: every atom is also the value of a registered pool +/// handle, so the table keeps nothing alive that was not already live. The +/// collector rewrites the entries on move through the intern-table root +/// scanner (`scan_intern_table_roots_mut`), exactly like the cache's. +/// +/// One funnel consults it: canonical key lists when they write a key +/// (`canonical_keys::Appended::atomized`), so a read site's pooled key and the +/// receiver's shape key are one pointer. +/// +/// An atom is NOT an interned string. It is minted by a plain allocation and +/// never carries `GC_FLAG_INTERNED`, and the intern cache neither adopts nor +/// hands out atoms. `GC_FLAG_INTERNED` is an ELIGIBILITY bit: the own-property +/// read lane, the set fast paths, the chain store and the proxy put paths +/// admit only interned keys. Minting atoms as interned strings silently widened +/// every one of those lanes to every pool-literal key, and on Zod the widened +/// read lane MISSES (the key is inherited, not own) at ~330 instructions each: +/// +0.3% instructions, measured, with the atom table itself inert. Identity is +/// the atom's job; eligibility stays exactly what it was. +/// +/// The table never decides an answer. A pointer match proves equal text; a +/// pointer MISmatch proves nothing (a list written before its atom existed +/// holds another string), so every consumer still falls back to bytes on a +/// mismatch. +pub(crate) struct AtomTable { + /// Open addressing, linear probe, power-of-two capacity; `string_ptr == 0` + /// is an empty slot. Entries are never removed. + slots: Vec, + len: usize, +} + +impl AtomTable { + const fn new() -> Self { + AtomTable { + slots: Vec::new(), + len: 0, + } + } + + /// The atom for `bytes` (whose FNV-1a hash is `hash`), if any. + /// + /// # Safety + /// Every tabled pointer is a live string (the collector keeps them so). + unsafe fn lookup(&self, bytes: &[u8], hash: u64) -> Option<*const StringHeader> { + self.lookup_from(0, bytes, hash) + } + + /// [`Self::lookup`] for a string that may itself BE the atom (`from`, or + /// 0): a tabled pointer equal to it answers without comparing bytes, which + /// is the common case for a key a list already holds. + /// + /// # Safety + /// As [`Self::lookup`]. + unsafe fn lookup_from( + &self, + from: usize, + bytes: &[u8], + hash: u64, + ) -> Option<*const StringHeader> { + if self.slots.is_empty() { + return None; + } + let mask = self.slots.len() - 1; + let mut i = (hash as usize) & mask; + loop { + let entry = &self.slots[i]; + if entry.string_ptr == 0 { + return None; + } + if entry.hash == hash { + let existing = entry.string_ptr as *const StringHeader; + if entry.string_ptr == from { + return Some(existing); + } + if (*existing).byte_len as usize == bytes.len() + && std::slice::from_raw_parts(string_data(existing), bytes.len()) == bytes + { + return Some(existing); + } + } + i = (i + 1) & mask; + } + } + + /// Table `atom` under `hash`. The caller has proved no atom exists for + /// its text. Rust-heap only: never allocates on the GC heap. + fn insert(&mut self, hash: u64, atom: *const StringHeader) { + if (self.len + 1) * 4 > self.slots.len() * 3 { + let _ = ATOM_YOUNG.try_with(|log| log.borrow_mut().clear()); + let cap = (self.slots.len() * 2).max(256); + let old = std::mem::replace( + &mut self.slots, + vec![ + InternEntry { + hash: 0, + string_ptr: 0, + }; + cap + ], + ); + for entry in old.into_iter().filter(|e| e.string_ptr != 0) { + self.place(entry); + } + } + self.place(InternEntry { + hash, + string_ptr: atom as usize, + }); + self.len += 1; + } + + fn place(&mut self, entry: InternEntry) { + let mask = self.slots.len() - 1; + let mut i = (entry.hash as usize) & mask; + while self.slots[i].string_ptr != 0 { + i = (i + 1) & mask; + } + // Rule 1 of `gc/young_log.rs`: log the slot BEFORE it names the + // string. A rehash re-places every entry, so it re-logs from scratch. + arm_atom_young(i, entry.string_ptr); + self.slots[i] = entry; + } +} + +crate::perry_thread_local! { + /// Atom-table slots that may hold a string a minor can act on. The atoms + /// are strong roots, but they are minted at module init (young) and + /// promoted soon after; a minor-scoped pass visits only these slots + /// instead of the whole table (`gc/young_log.rs`). + static ATOM_YOUNG: std::cell::RefCell> = + const { std::cell::RefCell::new(crate::gc::young_log::YoungLog::new()) }; +} + +const ATOM_YOUNG_LOG_NAME: &str = "string.atom_table"; + +#[inline] +fn arm_atom_young(slot: usize, string_ptr: usize) { + if crate::gc::young_log::addr_is_minor_relevant(string_ptr) { + let _ = ATOM_YOUNG.try_with(|log| log.borrow_mut().note(slot as u32)); + } +} + +/// The atoms are strong roots, rewritten on move (a key's hash is its +/// content's, so a move never rehashes). Minor-scoped: only the logged slots; +/// full: every slot, rebuilding the log. +fn scan_atom_roots(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + let young = visitor.young_scope(); + let _ = ATOMS.try_with(|t| unsafe { + let table = &mut *t.get(); + let table_len = table.slots.len() as u64; + #[cfg(any(debug_assertions, test))] + if young { + let relevant: Vec = (0..table.slots.len()) + .filter(|&i| { + crate::gc::young_log::addr_is_minor_relevant(table.slots[i].string_ptr) + }) + .map(|i| i as u32) + .collect(); + ATOM_YOUNG.with(|log| { + log.borrow() + .debug_assert_logged(ATOM_YOUNG_LOG_NAME, &relevant) + }); + } + let mut kept = ATOM_YOUNG.with(|log| log.borrow_mut().take_spare()); + let batch: Vec = if young { + ATOM_YOUNG.with(|log| log.borrow_mut().take_sorted()) + } else { + let _ = ATOM_YOUNG.with(|log| log.borrow_mut().take_sorted()); + (0..table.slots.len() as u32).collect() + }; + let visited = batch.len() as u64; + for &slot in &batch { + let Some(entry) = table.slots.get_mut(slot as usize) else { + continue; + }; + if entry.string_ptr == 0 { + continue; + } + visitor.visit_tagged_usize_slot(&mut entry.string_ptr, crate::value::STRING_TAG); + if crate::gc::young_log::addr_is_minor_relevant(entry.string_ptr) { + kept.push(slot); + } + } + let kept_len = kept.len() as u64; + ATOM_YOUNG.with(|log| log.borrow_mut().extend(kept)); + crate::gc::young_log::note_walk( + ATOM_YOUNG_LOG_NAME, + crate::gc::young_log::YoungLogWalk { + partial: young, + logged: visited, + visited, + kept: kept_len, + table_len, + }, + ); + }); +} + +crate::perry_thread_local! { + /// Per agent, like the cache: an atom is a string in THIS agent's heap. + pub(crate) static ATOMS: std::cell::UnsafeCell = + std::cell::UnsafeCell::new(AtomTable::new()); +} + +/// The atom for `bytes`, without allocating. `None` when no atom exists — or +/// when the agent's table is already torn down. +#[inline] +pub(crate) fn atom_lookup(bytes: &[u8], hash: u64) -> Option<*const StringHeader> { + ATOMS + .try_with(|t| unsafe { (*t.get()).lookup(bytes, hash) }) + .ok() + .flatten() +} + +/// The atom with `key`'s text, if one exists. `hash` is the FNV-1a hash of +/// `key`'s bytes (`key_bytes_hash`). Allocation-free, GC-free. +/// +/// # Safety +/// `key` is a live heap `StringHeader`. +#[inline] +pub(crate) unsafe fn atom_for_key( + key: *const StringHeader, + hash: u64, +) -> Option<*const StringHeader> { + if key.is_null() || (*key).byte_len > INTERN_MAX_BYTE_LEN { + return None; + } + let bytes = std::slice::from_raw_parts(string_data(key), (*key).byte_len as usize); + ATOMS + .try_with(|t| (*t.get()).lookup_from(key as usize, bytes, hash)) + .ok() + .flatten() +} + +/// Mint (or find) the atom for a pooled literal: the one string object this +/// agent uses for that text from now on. Called from `__perry_init_strings_*` +/// in place of `js_string_from_bytes` for pool entries that can be property +/// keys (at most `INTERN_MAX_BYTE_LEN` bytes). `hash` is the pool's +/// precomputed FNV-1a hash of the bytes — the same function as every other +/// key hash here. +/// +/// A plain allocation, exactly what the pool minted before atoms existed: the +/// atom is not interned and does not adopt the intern cache's string (see +/// `AtomTable`: identity, never eligibility). Longer literals are not keys +/// worth an atom and take the plain allocation without a table entry. +#[no_mangle] +pub extern "C" fn js_string_pool_atom( + bytes: *const u8, + len: u32, + hash: u64, + is_wtf8: i32, +) -> *mut StringHeader { + if len == 0 || len > INTERN_MAX_BYTE_LEN || bytes.is_null() { + return if is_wtf8 != 0 { + js_string_from_wtf8_bytes(bytes, len) + } else { + js_string_from_bytes(bytes, len) + }; + } + let input = unsafe { std::slice::from_raw_parts(bytes, len as usize) }; + if let Some(atom) = atom_lookup(input, hash) { + return atom as *mut StringHeader; + } + // Nothing is held across the allocation: `bytes` is read-only data in the + // compiled image. Shared (`refcount = 0`) like every pool literal. + let atom: *const StringHeader = if is_wtf8 != 0 { + js_string_from_wtf8_bytes(bytes, len) + } else { + js_string_from_bytes(bytes, len) + }; + let _ = ATOMS.try_with(|t| unsafe { (*t.get()).insert(hash, atom) }); + atom as *mut StringHeader +} + +/// Test hook: evict `bytes` from the intern CACHE (a collision would). +#[cfg(test)] +pub(crate) fn test_evict_interned(bytes: &[u8]) { + let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); + with_intern_table(|table| unsafe { + (*table)[(hash as usize) & INTERN_TABLE_MASK] = InternEntry { + hash: 0, + string_ptr: 0, + }; + }); +} + +/// Test view: is `p` the atom of its text? +#[cfg(test)] +pub(crate) unsafe fn is_atom_for_test(p: *const StringHeader) -> bool { + let bytes = std::slice::from_raw_parts(string_data(p), (*p).byte_len as usize); + let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len()); + atom_lookup(bytes, hash) == Some(p) +} + crate::perry_thread_local! { /// Intern-table slots that may hold a string a minor can act on /// (`gc/young_log.rs`). A minor-scoped `scan_intern_table_roots_mut` @@ -315,6 +624,7 @@ pub fn scan_intern_table_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<' table_len: INTERN_TABLE_SIZE as u64, }, ); + scan_atom_roots(visitor); return; } let _ = INTERN_YOUNG.with(|log| log.borrow_mut().take_sorted()); @@ -328,6 +638,7 @@ pub fn scan_intern_table_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<' } } }); + scan_atom_roots(visitor); let kept_len = kept.len() as u64; INTERN_YOUNG.with(|log| log.borrow_mut().extend(kept)); crate::gc::young_log::note_walk( diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index 76ba087607..d614c8a536 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -235,6 +235,10 @@ pub use html::{ js_string_fontcolor, js_string_fontsize, js_string_italics, js_string_link, js_string_small, js_string_strike, js_string_sub, js_string_sup, }; +pub(crate) use intern::atom_for_key; +pub use intern::js_string_pool_atom; +#[cfg(test)] +pub(crate) use intern::{atom_lookup, is_atom_for_test, test_evict_interned}; pub use intern::{js_string_intern, scan_intern_table_roots, scan_intern_table_roots_mut}; #[cfg(test)] pub(crate) use intern::{ diff --git a/scripts/thread_exit_address_globals.json b/scripts/thread_exit_address_globals.json index 9170d11a7b..21541c063f 100644 --- a/scripts/thread_exit_address_globals.json +++ b/scripts/thread_exit_address_globals.json @@ -4091,6 +4091,14 @@ ], "verdict": "no_heap_address", "why": "Holds (Handle, Parked) rows: a Handle is a native-registry id in [1, 0x40000) minted by the process-global shared pool (perry_ffi::shared_handle_id_pool), never an arena address, and Parked is {kind, NativeRegistrationIdentity{domain, serial, numeric_id}, epoch}. Payloads live in the process-global common HANDLES map, not in any thread's arena. A row cannot name a different object after reuse: release() retires only via begin_retirement_of(identity), which rejects any registration whose serial differs, and ids are reissued only after a full trace proves them unreferenced (#11453)." + }, + { + "file": "crates/perry-runtime/src/object/shapes_store.rs", + "names": [ + "ORDINARY_DIR" + ], + "verdict": "per_thread", + "why": "`#[thread_local]` static (not `thread_local!`, so the megamorphic read reaches it with one thread-pointer-relative load): each thread mirrors ITS OWN ShapeSlab page directory (a Rust-heap Vec pointer and length, never an arena address), republished on every change to `pages` and cleared by ShapeSlab::drop before the Vec is freed; const-initialised to (null, 0) with no drop glue." } ] } diff --git a/test-files/test_gap_megamorphic_slot_guess.ts b/test-files/test_gap_megamorphic_slot_guess.ts new file mode 100644 index 0000000000..a8af38d056 --- /dev/null +++ b/test-files/test_gap_megamorphic_slot_guess.ts @@ -0,0 +1,101 @@ +// S3c: the inline shape-confirmed slot guess at a megamorphic read site. +// +// One read site (`readKind`) sees 60+ shapes, so it latches megamorphic and +// its reads take the inline guess: the site's slot guess is confirmed against +// the RECEIVER's own shape key list before any slot is loaded. Every receiver +// family below is built so that a guess which is NOT confirmed by the +// receiver's own shape would return a wrong value: +// * `kind` at slot 2 in most shapes and at other slots in some (the guess +// is right for some receivers, wrong for others); +// * a DIFFERENT key sitting at the guessed slot; +// * dictionary-mode receivers (many keys added one by one); +// * receivers that deleted a key (tombstones / compaction); +// * an accessor installed with defineProperty (descriptor shape); +// * `kind` inherited from a prototype, and `kind` absent; +// * class instances. +// The output is a per-family checksum and must equal node's. + +const EXTRA = []; +for (let i = 0; i < 48; i++) EXTRA.push("x" + i); + +function plain(i: number): any { + const o: any = { pos: i, end: i + 1, kind: i % 7 }; + o[EXTRA[i % 48]] = i; + return o; +} +function shifted(i: number): any { + // `kind` at slot 4: the site's guess (2) names `flags` here. + const o: any = { pos: i, end: i + 1, flags: 1000 + i, parent: null, kind: 50 + (i % 5) }; + o[EXTRA[(i + 7) % 48]] = i; + return o; +} +function dictionary(i: number): any { + const o: any = { pos: i, end: i + 1, kind: 200 + (i % 3) }; + for (let k = 0; k < 200; k++) o["d" + i + "_" + k] = k; + return o; +} +function deleted(i: number): any { + const o: any = { pos: i, end: i + 1, kind: 300 + (i % 4), gone: 1, also: 2 }; + o[EXTRA[(i + 3) % 48]] = i; + delete o.also; + delete o.end; + return o; +} +function accessor(i: number): any { + const o: any = { pos: i, end: i + 1, kind: -1 }; + Object.defineProperty(o, "kind", { get() { return 400 + (i % 6); }, enumerable: true }); + return o; +} +const proto = { kind: 500 }; +function inherited(i: number): any { + const o: any = Object.create(proto); + o.pos = i; + o.end = i + 1; + o.flags = i; + return o; +} +function absent(i: number): any { + const o: any = { pos: i, end: i + 1, flags: 600 + i }; + o[EXTRA[(i + 11) % 48]] = i; + return o; +} +class Node3 { + pos: number; end: number; kind: number; + constructor(i: number) { this.pos = i; this.end = i + 1; this.kind = 700 + (i % 9); } +} + +function readKind(node: any): any { + return node.kind; +} + +const families: [string, (i: number) => any][] = [ + ["plain", plain], ["shifted", shifted], ["dictionary", dictionary], ["deleted", deleted], + ["accessor", accessor], ["inherited", inherited], ["absent", absent], + ["class", (i: number) => new Node3(i)], +]; +const pool: [number, any][] = []; +for (let f = 0; f < families.length; f++) { + for (let i = 0; i < 64; i++) pool.push([f, families[f][1](i)]); +} +// Interleave so the site sees every family while latched. +const order: number[] = []; +for (let i = 0; i < pool.length; i++) order.push((i * 37) % pool.length); + +const sums: number[] = families.map(() => 0); +let undef = 0; +for (let round = 0; round < 50; round++) { + for (const j of order) { + const [f, o] = pool[j]; + const v = readKind(o); + if (v === undefined) undef++; + else sums[f] += v; + } + // Mutate some receivers between rounds: a moved key, a re-added key. + if (round === 20) { + for (const [f, o] of pool) { + if (f === 0 && o.pos % 5 === 0) { delete o.kind; o.kind = 900; } + } + } +} +for (let f = 0; f < families.length; f++) console.log(families[f][0] + " " + sums[f]); +console.log("undefined " + undef);