perf(runtime): megamorphic reads confirm a slot guess by key atom; 'answerable by position' is a shape fact

changelog.d/11633-megamorphic-read-key-atoms.md

@@ -0,0 +1,5 @@
+Megamorphic property reads confirm the site's last slot against the receiver's
+shape with one pointer compare: property-key literals are now one string per
+key text (key atoms), and shape key lists hold that string. A 40-shape
+`o.kind` read drops from ~300 to ~140 instructions. Atoms do not change which
+keys count as interned.

crates/perry-codegen/src/codegen/string_pool.rs

@@ -96,6 +96,13 @@ impl<'a> InitChunker<'a> {
     }
 }
 
+/// Pool literals at most this long are minted as ATOMS. **Must equal
+/// `INTERN_MAX_BYTE_LEN` in `perry-runtime/src/string/intern.rs`**, the
+/// longest key the runtime interns; `js_string_pool_atom` falls back to a
+/// plain allocation past it, so a mismatch costs only the atom, never
+/// correctness.
+pub(crate) const POOL_ATOM_MAX_BYTE_LEN: usize = 64;
+
 /// Emit the string pool into the module: byte-array constants, handle
 /// globals, and the `__perry_init_strings_<prefix>` function that
 /// allocates + NaN-boxes + GC-roots each handle exactly once at startup.
@@ -446,12 +453,30 @@ pub(super) fn emit_string_pool(
         let bytes_ref = format!("@{}", entry.bytes_global);
         let handle_ref = format!("@{}", entry.handle_global);
         let len_str = entry.byte_len.to_string();
-        let from_bytes_fn = if entry.is_wtf8 {
-            "js_string_from_wtf8_bytes"
+        // A literal short enough to be a property key becomes its text's ATOM
+        // (`js_string_pool_atom`): one string object per key text for the
+        // whole agent, shared by every module's pool, every canonical shape
+        // key list and every intern hit — so a read site's key and the
+        // receiver's shape key compare by pointer (S3b). Longer literals, and
+        // WTF-8 ones (lone surrogates: never an identifier key), keep the
+        // plain allocation.
+        let atomize =
+            !entry.is_wtf8 && entry.byte_len > 0 && entry.byte_len <= POOL_ATOM_MAX_BYTE_LEN;
+        let handle = if atomize {
+            let hash = crate::nanbox::i64_literal(entry.dispatch_hash);
+            blk.call(
+                I64,
+                "js_string_pool_atom",
+                &[(PTR, &bytes_ref), (I32, &len_str), (I64, &hash), (I32, "0")],
+            )
         } else {
-            "js_string_from_bytes"
+            let from_bytes_fn = if entry.is_wtf8 {
+                "js_string_from_wtf8_bytes"
+            } else {
+                "js_string_from_bytes"
+            };
+            blk.call(I64, from_bytes_fn, &[(PTR, &bytes_ref), (I32, &len_str)])
         };
-        let handle = blk.call(I64, from_bytes_fn, &[(PTR, &bytes_ref), (I32, &len_str)]);
         let nanboxed = blk.call(DOUBLE, "js_nanbox_string", &[(I64, &handle)]);
         // Plain store, no remembered-set write barrier: the handle slot is
         // registered as a permanent global root on the very next line (always

crates/perry-codegen/src/gc_effects/linux-x86_64.tsv

@@ -3029,6 +3029,7 @@ js_sqlite_stmt_get	Reenters
 js_sqlite_stmt_raw	Reenters
 js_sqlite_stmt_run	Reenters
 js_sqlite_transaction	Reenters
+js_stack_overflow	Reenters
 js_state_get	Reenters
 js_state_init	Reenters
 js_state_set	Reenters
@@ -3128,6 +3129,7 @@ js_string_normalize	Reenters
 js_string_pad_end	Reenters
 js_string_pad_fill	Reenters
 js_string_pad_start	Reenters
+js_string_pool_atom	Reenters
 js_string_position_to_index	Leaf
 js_string_print	Leaf
 js_string_raw	Reenters

crates/perry-codegen/src/runtime_decls/mod.rs

@@ -159,6 +159,10 @@ pub fn declare_phase1(module: &mut LlModule) {
     // Strings (enough to produce string literals for later phases).
     module.declare_function("js_string_from_bytes", I64, &[PTR, I32]);
     module.declare_function("js_string_from_wtf8_bytes", I64, &[PTR, I32]);
+    // S3b: a pooled literal short enough to be a key is minted as the atom of
+    // its text (`string/intern.rs::js_string_pool_atom`): bytes, len, FNV-1a
+    // hash, is_wtf8.
+    module.declare_function("js_string_pool_atom", I64, &[PTR, I32, I64, I32]);
 
     // Type checks.
     module.declare_function("js_is_truthy", I32, &[DOUBLE]);

crates/perry-codegen/src/wasm32/runtime_abi.tsv

@@ -3617,6 +3617,7 @@ js_string_normalize	ptr	ptr,f64
 js_string_pad_end	ptr	ptr,f64,ptr
 js_string_pad_fill	ptr	f64
 js_string_pad_start	ptr	ptr,f64,ptr
+js_string_pool_atom	ptr	ptr,i32u,i64,i32s
 js_string_position_to_index	i32s	f64
 js_string_print	void	ptr
 js_string_raw	ptr	f64,f64

crates/perry-runtime/src/gc/tests/canonical_keys_holders.rs

@@ -657,3 +657,140 @@ fn a_dead_weak_keys_entry_is_dropped_before_its_storage_is_reused() {
         .remove(&DEAD_MEMO_CLASS_ID);
     canonical_keys::reset_for_test();
 }
+
+// ------------------------------------------------------- S3b: key atoms
+
+fn atom_hash(text: &[u8]) -> u64 {
+    crate::object::key_bytes_hash(text.as_ptr(), text.len())
+}
+
+fn atom_bits(atom: usize) -> u64 {
+    crate::value::js_nanbox_string(atom as i64).to_bits()
+}
+
+/// S3b: a key text has ONE string object in an agent — its atom — and every
+/// canonical list written after the atom exists holds it. The atom table holds
+/// its strings strongly and REWRITES them on a move (the intern-table root
+/// scanner), so after a moving minor the table, the list and a fresh pool mint
+/// all name the atom at its NEW address, and none names the address it moved
+/// away from. Interning is separate: the intern cache never hands out an atom.
+#[test]
+fn an_atom_and_the_lists_holding_it_follow_a_moving_minor() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    register_object_model_scanners();
+    gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut);
+    canonical_keys::reset_for_test();
+    let scope = RuntimeHandleScope::new();
+    let text = b"atom_mv_kind";
+    let hash = atom_hash(text);
+    unsafe {
+        let atom =
+            crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize;
+        assert!(
+            crate::arena::pointer_in_nursery(atom),
+            "premise: the atom is young, so a minor can move it"
+        );
+        // A receiver grows the key through a DIFFERENT string with the text.
+        let copy = nursery_key("atom_mv_kind");
+        assert_ne!(copy as usize, atom, "premise: two string objects, one text");
+        let o = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0));
+        set(o, nursery_key("atom_mv_a"), 1.0);
+        set(o, copy, 2.0);
+        let list = keys_of(o);
+        assert_eq!(
+            crate::array::js_array_get(list, 1).bits(),
+            atom_bits(atom),
+            "INVARIANT: the written list holds the atom, not the grower's copy"
+        );
+
+        let trace = collect_minor_trace(GcTriggerKind::Direct);
+        assert!(
+            trace.copying_nursery.copied_objects > 0,
+            "premise: the minor copied"
+        );
+        let moved = crate::string::atom_lookup(text, hash).expect("the atom survives") as usize;
+        assert_ne!(moved, atom, "premise: the minor moved the atom");
+        let header = crate::value::addr_class::try_read_tracked_gc_header(moved)
+            .expect("the moved atom is a tracked cell");
+        assert_eq!(
+            (*header.as_ptr()).gc_flags & GC_FLAG_FORWARDED,
+            0,
+            "INVARIANT: the table names the live copy, not a forwarding header"
+        );
+        assert_eq!(
+            crate::array::js_array_get(keys_of(o), 1).bits(),
+            atom_bits(moved),
+            "INVARIANT: the list follows its atom through the move"
+        );
+        // After the move: minting again returns the moved atom — never a third
+        // string. Interning a copy does NOT: an atom is identity, not
+        // eligibility (`string::intern::AtomTable`).
+        assert_ne!(
+            crate::string::js_string_intern(nursery_key("atom_mv_kind"), hash) as usize,
+            moved,
+            "INVARIANT: the intern cache never hands out an atom"
+        );
+        assert_eq!(
+            crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize,
+            moved,
+            "a second mint finds the moved atom"
+        );
+        // A list written after the move holds the moved atom.
+        let o2 = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0));
+        set(o2, nursery_key("atom_mv_b"), 1.0);
+        set(o2, nursery_key("atom_mv_kind"), 2.0);
+        assert_eq!(
+            crate::array::js_array_get(keys_of(o2), 1).bits(),
+            atom_bits(moved),
+            "INVARIANT: a list written after the move holds the moved atom"
+        );
+    }
+}
+
+/// S3b, a collection DURING the write: the canonical backing allocation that
+/// publishes a list holding an atom is itself the collection point, and moves
+/// the atom. The published list must hold the atom's LIVE address.
+#[test]
+fn a_list_written_while_its_atom_moves_holds_the_live_atom() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _pacing = crate::gc::policy::force_alloc_point_minor_pacing();
+    let _scan = NoConservativeScan::new();
+    let trigger = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    register_object_model_scanners();
+    gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut);
+    canonical_keys::reset_for_test();
+    let text = b"atom_during_kind";
+    let hash = atom_hash(text);
+    unsafe {
+        let atom =
+            crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) as usize;
+        assert!(
+            crate::arena::pointer_in_nursery(atom),
+            "premise: the atom is young"
+        );
+        let scope = RuntimeHandleScope::new();
+        let copy = scope.root_string_ptr(nursery_key("atom_during_kind"));
+        // The empty list plus this key is a fresh backing: its allocation is
+        // the collection point.
+        arm_collection_on_next_block(&trigger);
+        let list = copy.with_const_ptr(|k: *const crate::StringHeader| {
+            canonical_keys::extend_key(
+                &SharedLayout::shape_cache_entry(),
+                canonical_keys::CanonicalKeys::EMPTY,
+                k,
+            )
+        });
+        let live = crate::string::atom_lookup(text, hash).expect("the atom survives") as usize;
+        assert_ne!(
+            live, atom,
+            "premise: the backing allocation did not move the atom, so this run proved \
+             nothing. Check the trigger arming."
+        );
+        assert_eq!(
+            crate::array::js_array_get(list.as_ptr(), 0).bits(),
+            atom_bits(live),
+            "INVARIANT: the list holds the atom's live address"
+        );
+    }
+}

crates/perry-runtime/src/gc/tests/minor_fixed_cost.rs

@@ -178,3 +178,44 @@ fn small_int_cache_writer_publishing_a_young_string_is_caught() {
     crate::string::test_write_small_int_cache_slot(255, young);
     crate::string::debug_assert_small_string_caches_not_minor_relevant();
 }
+
+/// An atom minted from a young string and reachable ONLY through the atom
+/// table survives a moving minor: the table names the forwarded, live copy
+/// (found again by text and by `atom_for_key`), not from-space. The atom
+/// young log is what makes the minor visit that slot.
+#[test]
+fn young_atom_is_rewritten_through_the_atom_young_log() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _clear = ClearTablesOnDrop;
+    gc_register_mutable_root_scanner(crate::string::scan_intern_table_roots_mut);
+    crate::string::test_clear_intern_table();
+
+    let bytes = b"minor-fixed-cost-young-atom";
+    let hash = crate::object::key_bytes_hash(bytes.as_ptr(), bytes.len());
+    let young = crate::string::js_string_pool_atom(bytes.as_ptr(), bytes.len() as u32, hash, 0);
+    assert!(crate::arena::pointer_in_nursery(young as usize));
+    assert_eq!(
+        crate::string::atom_lookup(bytes, hash),
+        Some(young as *const _)
+    );
+
+    let _ = gc_collect_minor();
+
+    let tabled = crate::string::atom_lookup(bytes, hash).expect("the atom must stay tabled");
+    assert_ne!(
+        tabled as usize, young as usize,
+        "the table must name the evacuated copy, not from-space"
+    );
+    unsafe {
+        assert_string_bytes(tabled, bytes);
+        let fresh = crate::string::js_string_pool_atom(bytes.as_ptr(), bytes.len() as u32, hash, 0);
+        assert_eq!(
+            fresh as usize, tabled as usize,
+            "the pool must reuse the live atom"
+        );
+        assert!(crate::string::is_atom_for_test(tabled));
+        // A second string with the same text resolves to the same atom.
+        let other = crate::string::js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32);
+        assert_eq!(crate::string::atom_for_key(other, hash), Some(tabled));
+    }
+}

crates/perry-runtime/src/object/canonical_keys.rs

@@ -676,6 +676,41 @@ impl Appended {
         }
     }
 
+    /// The same slot, spelled with its text's ATOM when one exists — the one
+    /// string a read site's pooled key also is (`string::intern::AtomTable`).
+    ///
+    /// Applied to every key this module WRITES into a list, and nowhere else:
+    /// the trie validates edges by bytes, so which string object a list holds
+    /// never changes which node a probe reaches, only whether a later key
+    /// compare against the list can stop at pointer equality. Heap strings
+    /// only — an SSO slot stays an SSO slot, so `is_pointer` (and with it the
+    /// backing's all-pointer layout) is unchanged by the substitution.
+    ///
+    /// `h` is this slot's `edge_hash`, which for a string IS the FNV-1a hash of
+    /// its bytes — the atom table's hash.
+    ///
+    /// # Safety
+    /// The operand is live.
+    unsafe fn atomized(self, h: u64) -> Self {
+        match self {
+            Appended::Key(key) if !key.is_null() => match crate::string::atom_for_key(key, h) {
+                Some(atom) => Appended::Key(atom),
+                None => self,
+            },
+            // An SSO short string carries its bytes in the value itself, so
+            // its bits ARE its identity: equal texts are already equal words
+            // and there is no heap string to replace with an atom.
+            Appended::Slot(v) if v.is_short_string() => self,
+            Appended::Slot(v) if v.is_string() => {
+                match crate::string::atom_for_key(v.as_string_ptr(), h) {
+                    Some(atom) => Appended::Slot(JSValue::string_ptr(atom as *mut StringHeader)),
+                    None => self,
+                }
+            }
+            _ => self,
+        }
+    }
+
     /// Is the appended slot a heap string POINTER? An SSO short string and a
     /// tombstone are not, and either one costs the child its all-pointer
     /// layout — see `Node::all_ptr`.
@@ -789,6 +824,8 @@ pub(crate) unsafe fn extend_slot(
     if let Some(hit) = probe(parent, parent_len, appended, entry, h) {
         return hit;
     }
+    // A new list is about to be WRITTEN: it holds the atom of this key's text.
+    let appended = appended.atomized(appended.slot_hash());
 
     // Whether the child's slots are all heap string pointers is the parent's
     // answer AND this slot's, so it is read before the allocation and never
@@ -1180,6 +1217,20 @@ pub(crate) unsafe fn canonicalize(
     // GC_STORE_AUDIT(INIT): fresh is unpublished; publish length only after
     // all elements are initialized, with no intervening GC allocation.
     std::ptr::copy_nonoverlapping(slots, dst, len as usize);
+    // Every heap key of the new list is its text's atom where one exists (see
+    // `Appended::atomized`). Heap string to heap string, so `all_ptr` holds.
+    for i in 0..len as usize {
+        let slot = Appended::Slot(JSValue::from_bits((*dst.add(i)).to_bits()));
+        if let Appended::Slot(v) = slot {
+            if v.is_string() {
+                if let Appended::Slot(atom) = slot.atomized(slot.slot_hash()) {
+                    // GC_STORE_AUDIT(INIT): `fresh` is unpublished; its length
+                    // is set on the next line, after the last slot is written.
+                    *dst.add(i) = f64::from_bits(atom.bits());
+                }
+            }
+        }
+    }
     if with_attrs {
         let attrs = crate::object::key_attrs::keys_attrs(fresh);
         crate::object::key_attrs::copy_entries(keys, 0, attrs, len);