From 3d6aeea0ad5393c60e15838c0f4c11c26cd50d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 26 Sep 2026 07:14:10 +0000 Subject: [PATCH 1/6] perf(runtime): a representation change takes no typed-feedback lock with feedback off Every key-add on a typed-layout receiver (an object literal) retires its layout record through invalidate_representation_change, which took the typed-feedback registry lock, bumped the GC-root lock depth and flushed deferred collection requests on release, only to bump two counters that nothing but the typed-feedback trace reads. With feedback off it now returns first: about 180 instructions per such add (literal key-add fixture 693.9 -> 513.6 instr/op). --- crates/perry-runtime/src/typed_feedback.rs | 11 +++++++++- .../perry-runtime/src/typed_feedback/tests.rs | 20 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/perry-runtime/src/typed_feedback.rs b/crates/perry-runtime/src/typed_feedback.rs index 5f72e10861..06c9455036 100644 --- a/crates/perry-runtime/src/typed_feedback.rs +++ b/crates/perry-runtime/src/typed_feedback.rs @@ -2949,7 +2949,16 @@ pub(crate) fn invalidate_method_change(class_id: u32) { const REPRESENTATION_INVALIDATION_SCAN_BUDGET: u64 = 50_000_000; pub(crate) fn invalidate_representation_change(obj_addr: usize) { - if obj_addr == 0 { + invalidate_representation_change_when(obj_addr, typed_feedback_enabled()); +} + +fn invalidate_representation_change_when(obj_addr: usize, feedback_on: bool) { + // Both counters this bumps are read only by the typed-feedback trace, and + // every site this could credit is recorded only while feedback is on. Off + // (every production run), taking the registry lock to learn that cost a + // key-add on a typed-layout receiver ~250 instructions: the lock, the + // GC-root lock depth, and the deferred-collection flush on its release. + if obj_addr == 0 || !feedback_on { return; } let mut reg = registry(); diff --git a/crates/perry-runtime/src/typed_feedback/tests.rs b/crates/perry-runtime/src/typed_feedback/tests.rs index 248f6867a4..109f148e70 100644 --- a/crates/perry-runtime/src/typed_feedback/tests.rs +++ b/crates/perry-runtime/src/typed_feedback/tests.rs @@ -3183,3 +3183,23 @@ fn class_field_get_ic_throws_a_type_error_on_a_nullish_receiver() { ); } } + +/// A key-add on a typed-layout receiver retires its layout record through +/// `invalidate_representation_change`. With feedback off (every production +/// run) nothing can read what it counts, so it must return before the +/// registry lock; the control arm proves the counter this test reads moves. +#[test] +fn representation_change_takes_no_registry_lock_with_feedback_off() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + let addr = 0x7000_0000usize; + invalidate_representation_change_when(addr, false); + assert_eq!( + typed_feedback_snapshot().representation_invalidations, + 0, + "feedback off: the registry must not be touched" + ); + invalidate_representation_change_when(addr, true); + assert_eq!(typed_feedback_snapshot().representation_invalidations, 1); + reset_typed_feedback_for_tests(); +} From d44ba716e0ab5776099e2abc263179ce250dc308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 26 Sep 2026 07:14:10 +0000 Subject: [PATCH 2/6] perf(codegen,runtime): a polymorphic key-add site serves its memos inline at their home way A displaced key-add memo is placed at its pre-shape home way in the site block (top 6 bits of sid * 0x9E3779B1; the next free way when an earlier memo holds it), and the emitted hit compares the home and the next way after the primary memo, whatever the number of shapes. On tsc the hot memo of a polymorphic site sits behind transient first-instance shapes (3rd or 15th in arrival order), so no fixed prefix of an in-order list would hold it. The primary add memo is compared before the existing-key ways, and the hot path tests the header word once (refused bits and layout record together), sorting out a layout record in a cold block. --- .../src/expr/put_value_store_ic.rs | 266 +++++++++++++----- crates/perry-codegen/src/expr/store_census.rs | 3 + .../tests/native_proof_regressions.rs | 32 +++ .../src/proxy/put_value/packed_add.rs | 74 ++++- .../src/proxy/put_value/packed_add_tests.rs | 76 +++++ crates/perry/tests/keyadd_store_ic.rs | 79 +++++- 6 files changed, 433 insertions(+), 97 deletions(-) diff --git a/crates/perry-codegen/src/expr/put_value_store_ic.rs b/crates/perry-codegen/src/expr/put_value_store_ic.rs index 17de8138f5..07bb5294f2 100644 --- a/crates/perry-codegen/src/expr/put_value_store_ic.rs +++ b/crates/perry-codegen/src/expr/put_value_store_ic.rs @@ -138,6 +138,22 @@ pub(crate) const PACKED_SET_SITE_WORDS: usize = 4; pub(crate) const ADD_SHAPES_WORD: usize = 1; pub(crate) const ADD_GUARD_WORD: usize = 2; pub(crate) const ADD_SLOT_BITS: u32 = 16; +/// The site word holding the runtime's `*AddWay` block (0 = none), and how +/// the emitted code finds the ways of it that it compares after the primary +/// memo: from the receiver ShapeId's HOME, the top `ADD_WAYS_LOG2` bits of +/// `sid * ADD_WAY_HASH` (mod 2^32). A way is two words in the primary pair's +/// format, `{shapes, guard}`, so the site's words +/// `ADD_SHAPES_WORD..=ADD_GUARD_WORD` are a way too. **Must equal +/// `perry_runtime::proxy::put_value::packed_add::{ADD_WAYS_WORD, +/// ADD_WAY_WORDS, ADD_WAYS_LOG2, ADD_WAY_HASH}`** (and `add_way_home`); +/// pinned by the runtime's `packed_set_site_layout_matches_codegen`. +pub(crate) const ADD_WAYS_WORD: usize = 3; +pub(crate) const ADD_WAY_WORDS: usize = 2; +pub(crate) const ADD_WAYS_LOG2: u32 = 6; +pub(crate) const ADD_WAY_HASH: u32 = 0x9E37_79B1; +/// Ways compared from the home on: the home, then the next (mod the block), +/// where the runtime places a memo whose home an earlier one holds. +pub(crate) const ADD_WAY_PROBES: usize = 2; const ADD_SLOT_MASK: u64 = (1 << ADD_SLOT_BITS) - 1; /// Block-name stem of the key-add hit. const ADD_STEM: &str = "put.add"; @@ -160,6 +176,14 @@ fn add_header_refuse_mask() -> i32 { ((ADD_REFUSE_RESERVED << 16) | (ADD_REFUSE_GC_FLAGS << 8)) as i32 } +/// The hot key-add test over the same word: nothing refused AND no layout +/// record to retire (`ADD_LAYOUT_RESERVED`). Its zero is the common case; a +/// non-zero result is sorted out by [`add_header_refuse_mask`] off the hot +/// path. +fn add_header_hot_mask() -> i32 { + add_header_refuse_mask() | (ADD_LAYOUT_RESERVED << 16) as i32 +} + /// The barrier-census stem. Shared with the census registry /// (`barrier_stem_census_tests::VERIFIED_BARRIER_STEMS`). pub(crate) const STORE_IC_STEM: &str = "put.pic"; @@ -283,22 +307,45 @@ pub(crate) fn emit_static_store_ic( let sid = ctx.block().load(I32, &sid_ptr); let stamp = ctx.block().trunc(I64, &word, I32); let shape_eq = ctx.block().icmp_eq(I32, &sid, &stamp); + ctx.block().cond_br(&shape_eq, &kind_label, &add_label); + let mut word_incoming: Vec<(String, String)> = vec![(word, tok_label.clone())]; + + // A word miss: the key-add memo's primary pre-shape next, BEFORE the + // existing-key ways. A site that has only ever added keys then pays one + // compare of the adjacent word instead of the ways-cache load; a site + // with existing-key ways pays that one compare more. The two can never + // both match one ShapeId: an existing-key memo names a shape that HAS + // the key, an add memo one that lacks it. + ctx.current_block = add_idx; let ways_entry_idx = ctx.new_block(&format!("{STORE_IC_STEM}.ways")); let ways_entry_label = ctx.block_label(ways_entry_idx); + let add_ways_idx = ctx.new_block(&format!("{ADD_STEM}.ways")); + let add_ways_label = ctx.block_label(add_ways_idx); + let add_hit_idx = ctx.new_block(&format!("{ADD_STEM}.chain")); + let add_hit_label = ctx.block_label(add_hit_idx); + let primary_ptr = ctx + .block() + .gep(I64, &packed_ref, &[(I64, &ADD_SHAPES_WORD.to_string())]); + let primary = ctx.block().load_atomic_monotonic(I64, &primary_ptr, 8); + let primary_pre = ctx.block().trunc(I64, &primary, I32); + let primary_eq = ctx.block().icmp_eq(I32, &sid, &primary_pre); ctx.block() - .cond_br(&shape_eq, &kind_label, &ways_entry_label); - let mut word_incoming: Vec<(String, String)> = vec![(word, tok_label.clone())]; - - // A word miss: compare the first ways of the site's cache (the read path's - // #7753 structure), each in the word's own format, so a hit on any of them - // is the same ONE ShapeId compare and flows into the same store. A spill + .cond_br(&primary_eq, &add_hit_label, &ways_entry_label); + // Each entry: (the memo's shapes word, the address of its pair, block). + let mut memo_incoming: Vec<(String, String, String)> = + vec![(primary, primary_ptr, add_label.clone())]; + + // Compare the first ways of the existing-key cache (the read path's #7753 + // structure), each in the word's own format, so a hit on any of them is + // the same ONE ShapeId compare and flows into the same store. A spill // entry is flipped out of the ShapeId range and never matches here. The // cache is lazily allocated: a site that has never primed has none. ctx.current_block = ways_entry_idx; let ways = super::emit_inline_cache_slot(ctx, &cache_name); let first_way_idx = ctx.new_block(&format!("{STORE_IC_STEM}.way")); let mut way_label = ctx.block_label(first_way_idx); - ctx.block().cond_br(&ways.present, &way_label, &add_label); + ctx.block() + .cond_br(&ways.present, &way_label, &add_ways_label); let mut way_idx = first_way_idx; for w in 0..PACKED_SET_INLINE_WAYS { ctx.current_block = way_idx; @@ -310,13 +357,74 @@ pub(crate) fn emit_static_store_ic( way_idx = ctx.new_block(&format!("{STORE_IC_STEM}.way")); ctx.block_label(way_idx) } else { - add_label.clone() + add_ways_label.clone() }; ctx.block().cond_br(&way_eq, &kind_label, &next_label); word_incoming.push((entry, way_label.clone())); way_label = next_label; } + // The key-add ways at the receiver ShapeId's home in the runtime's block + // (`packed_add::add_way_home`) and the one after it: a displaced memo is + // placed at its home, or when an earlier memo holds that, at the next + // free way from it. Whichever pre-shapes a polymorphic site keeps hot, + // each is one or two compares away, the same compare as the primary's. + // A hit reads its guard from the same pair. + ctx.current_block = add_ways_idx; + let block_ptr = ctx + .block() + .gep(I64, &packed_ref, &[(I64, &ADD_WAYS_WORD.to_string())]); + let block_word = ctx.block().load_atomic_monotonic(I64, &block_ptr, 8); + let has_block = ctx.block().icmp_ne(I64, &block_word, "0"); + let add_block = ctx.block().inttoptr(I64, &block_word); + let mut add_way_idx = ctx.new_block(&format!("{ADD_STEM}.way")); + let mut add_way_label = ctx.block_label(add_way_idx); + ctx.block().cond_br(&has_block, &add_way_label, &miss_label); + ctx.current_block = add_way_idx; + let hashed = ctx + .block() + .mul(I32, &sid, &(ADD_WAY_HASH as i32).to_string()); + let home = ctx + .block() + .lshr(I32, &hashed, &(32 - ADD_WAYS_LOG2).to_string()); + for probe in 0..ADD_WAY_PROBES { + ctx.current_block = add_way_idx; + let way = if probe == 0 { + home.clone() + } else { + let next = ctx.block().add(I32, &home, &probe.to_string()); + ctx.block() + .and(I32, &next, &((1u32 << ADD_WAYS_LOG2) - 1).to_string()) + }; + let way_wide = ctx.block().zext(I32, &way, I64); + let word_index = ctx.block().mul(I64, &way_wide, &ADD_WAY_WORDS.to_string()); + let pair_ptr = ctx.block().gep(I64, &add_block, &[(I64, &word_index)]); + let shapes = ctx.block().load_atomic_monotonic(I64, &pair_ptr, 8); + let pre = ctx.block().trunc(I64, &shapes, I32); + let way_eq = ctx.block().icmp_eq(I32, &sid, &pre); + let next_label = if probe + 1 < ADD_WAY_PROBES { + add_way_idx = ctx.new_block(&format!("{ADD_STEM}.way")); + ctx.block_label(add_way_idx) + } else { + miss_label.clone() + }; + let hit_label = if super::store_census::enabled() { + // A census build counts a way hit on its own edge. + let count_idx = ctx.new_block(&format!("{ADD_STEM}.way.census")); + let count_label = ctx.block_label(count_idx); + ctx.block().cond_br(&way_eq, &count_label, &next_label); + ctx.current_block = count_idx; + super::store_census::bump(ctx, super::store_census::ADD_WAY_HIT); + ctx.block().br(&add_hit_label); + count_label + } else { + ctx.block().cond_br(&way_eq, &add_hit_label, &next_label); + add_way_label.clone() + }; + memo_incoming.push((shapes, pair_ptr, hit_label)); + add_way_label = next_label; + } + // The per-object facts the shape does not carry (see the module doc), as // a branch chain rather than one flat predicate (#7883). ctx.current_block = kind_idx; @@ -376,12 +484,25 @@ pub(crate) fn emit_static_store_ic( let hit_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); - ctx.current_block = add_idx; + ctx.current_block = add_hit_idx; + let (shapes, pair_ptr) = { + let shapes_in: Vec<(&str, &str)> = memo_incoming + .iter() + .map(|(s, _, l)| (s.as_str(), l.as_str())) + .collect(); + let pairs_in: Vec<(&str, &str)> = memo_incoming + .iter() + .map(|(_, p, l)| (p.as_str(), l.as_str())) + .collect(); + let shapes = ctx.block().phi(I64, &shapes_in); + let pair_ptr = ctx.block().phi(PTR, &pairs_in); + (shapes, pair_ptr) + }; let add_end_label = emit_key_add_hit( ctx, - &packed_ref, + &shapes, + &pair_ptr, &handle, - &sid, value_double, value_bits, &miss_label, @@ -417,67 +538,61 @@ pub(crate) fn emit_static_store_ic( ) } -/// The key-add hit: `k` is not own on the receiver, and the site's add words -/// (`perry_runtime::proxy::put_value::packed_add`) memo the transition from -/// the receiver's PRE-shape. Entered after the existing-key word and ways -/// missed, with the receiver's handle and ShapeId already loaded. Returns the -/// label of the block that branches to `merge_label` on a hit; every refusal -/// branches to `miss_label` with nothing written. +/// The key-add hit: `k` is not own on the receiver, and one of the site's +/// add memos (`perry_runtime::proxy::put_value::packed_add`) names the +/// receiver's PRE-shape. Entered from the pre-shape compare that matched, +/// with that memo's `shapes` word and the address of its `{shapes, guard}` +/// pair. Returns the label of the block that branches to `merge_label` on a +/// hit; every refusal branches to `miss_label` with nothing written. /// /// ```text -/// ONE pre-shape compare sid == low half of word 1 -/// the chain verdict PROTO_VALIDITY + VTABLE_GEN == word 2 >> 16 -/// per-object facts GcHeader word: not TENURED, layout state -/// UNKNOWN / POINTER_FREE, no numeric proof, -/// tombstones or descriptor flag; meta == null; -/// the receiver-kind admission -/// the successor ShapeId high half of word 1 -> handle + 4 -/// the store and barrier slot = word 2 & 0xFFFF +/// ONE pre-shape compare sid == low half of a memo's shapes (caller) +/// the chain verdict PROTO_VALIDITY + VTABLE_GEN == guard >> 16 +/// the receiver-kind admission class id / _reserved, as the existing-key hit +/// per-object facts ONE test of the GcHeader word: not TENURED, +/// no numeric proof, tombstones or descriptor +/// flag, and no layout record to retire +/// the successor ShapeId high half of shapes -> handle + 4 +/// the store and barrier slot = guard & 0xFFFF /// ``` /// +/// A receiver with a layout record (side mask or typed layout) leaves the +/// one test for a cold block that refuses exactly what the hot test refuses +/// and retires the record before the same store. +/// /// Nothing between the caller's re-read of the receiver and the stores can /// collect: plain loads, compares, and two stores. #[allow(clippy::too_many_arguments)] fn emit_key_add_hit( ctx: &mut FnCtx<'_>, - packed_ref: &str, + shapes: &str, + pair_ptr: &str, handle: &str, - sid: &str, value_double: &str, value_bits: &str, miss_label: &str, merge_label: &str, ) -> String { - let gen_idx = ctx.new_block(&format!("{ADD_STEM}.chain")); - let layout_idx = ctx.new_block(&format!("{ADD_STEM}.layout")); - let forget_idx = ctx.new_block(&format!("{ADD_STEM}.layout.forget")); let obj_idx = ctx.new_block(&format!("{ADD_STEM}.object")); - let class_idx = ctx.new_block(&format!("{ADD_STEM}.class")); let classless_idx = ctx.new_block(&format!("{ADD_STEM}.classless")); + let layout_idx = ctx.new_block(&format!("{ADD_STEM}.layout")); + let slow_idx = ctx.new_block(&format!("{ADD_STEM}.layout.slow")); + let forget_idx = ctx.new_block(&format!("{ADD_STEM}.layout.forget")); let store_idx = ctx.new_block(&format!("{ADD_STEM}.hit.store")); - let gen_label = ctx.block_label(gen_idx); let obj_label = ctx.block_label(obj_idx); - let class_label = ctx.block_label(class_idx); let classless_label = ctx.block_label(classless_idx); - let store_label = ctx.block_label(store_idx); let layout_label = ctx.block_label(layout_idx); + let slow_label = ctx.block_label(slow_idx); let forget_label = ctx.block_label(forget_idx); + let store_label = ctx.block_label(store_idx); - // The pre-shape compare. A spill-slot memo is published flipped out of - // the ShapeId range, so it can never match here. - let shapes_ptr = ctx - .block() - .gep(I64, packed_ref, &[(I64, &ADD_SHAPES_WORD.to_string())]); - let shapes = ctx.block().load_atomic_monotonic(I64, &shapes_ptr, 8); - let pre = ctx.block().trunc(I64, &shapes, I32); - let pre_eq = ctx.block().icmp_eq(I32, sid, &pre); - ctx.block().cond_br(&pre_eq, &gen_label, miss_label); - - // The chain verdict's generation: the one global prototype-validity word. - ctx.current_block = gen_idx; - let guard_ptr = ctx - .block() - .gep(I64, packed_ref, &[(I64, &ADD_GUARD_WORD.to_string())]); + // The chain verdict's generation: the one global prototype-validity word, + // against the guard of the memo that matched. + let guard_ptr = ctx.block().gep( + I64, + pair_ptr, + &[(I64, &(ADD_GUARD_WORD - ADD_SHAPES_WORD).to_string())], + ); let guard = ctx.block().load_atomic_monotonic(I64, &guard_ptr, 8); let now = ctx .block() @@ -486,19 +601,14 @@ fn emit_key_add_hit( let gen_eq = ctx.block().icmp_eq(I64, &now, &recorded); ctx.block().cond_br(&gen_eq, &obj_label, miss_label); - // The GcHeader's first word: obj_type | gc_flags << 8 | _reserved << 16. + // The GcHeader's first word (obj_type | gc_flags << 8 | _reserved << 16) + // and the receiver-kind admission, as the existing-key hit. ctx.current_block = obj_idx; let hdr_addr = ctx.block().sub(I64, handle, "8"); let hdr_ptr = ctx.block().inttoptr(I64, &hdr_addr); let hdr = ctx.block().load(I32, &hdr_ptr); - let refused = ctx - .block() - .and(I32, &hdr, &add_header_refuse_mask().to_string()); - let hdr_ok = ctx.block().icmp_eq(I32, &refused, "0"); - ctx.block().cond_br(&hdr_ok, &class_label, miss_label); - - // The receiver-kind admission, as the existing-key hit. - ctx.current_block = class_idx; + let reserved_i32 = ctx.block().lshr(I32, &hdr, "16"); + let reserved = ctx.block().trunc(I32, &reserved_i32, I16); let class_ptr = ctx.block().inttoptr(I64, handle); let class_id = ctx.block().load(I32, &class_ptr); let class_biased = ctx.block().add(I32, &class_id, "2"); @@ -507,36 +617,50 @@ fn emit_key_add_hit( .cond_br(&has_class, &layout_label, &classless_label); ctx.current_block = classless_idx; - let reserved_i32 = ctx.block().lshr(I32, &hdr, "16"); - let reserved = ctx.block().trunc(I32, &reserved_i32, I16); let admit_bits = ctx.block().and(I16, &reserved, CLASSLESS_ADMIT_MASK_I16); let admitted = ctx.block().icmp_eq(I16, &admit_bits, CLASSLESS_ADMIT_I16); let classless = ctx.block().icmp_eq(I32, &class_id, "0"); let plain_ok = ctx.block().and(I1, &admitted, &classless); ctx.block().cond_br(&plain_ok, &layout_label, miss_label); - // A side-mask or typed-layout receiver: its layout record describes the - // PRE-shape, so retire it first, exactly as the transition lane does. The - // callee edits side tables only and cannot collect. + // ONE test: nothing refused and no layout record. ctx.current_block = layout_idx; - let layout_bits = ctx + let hot_bits = ctx .block() - .and(I32, &hdr, &((ADD_LAYOUT_RESERVED << 16) as i32).to_string()); - let layout_plain = ctx.block().icmp_eq(I32, &layout_bits, "0"); - ctx.block() - .cond_br(&layout_plain, &store_label, &forget_label); + .and(I32, &hdr, &add_header_hot_mask().to_string()); + let hot_ok = ctx.block().icmp_eq(I32, &hot_bits, "0"); + ctx.block().cond_br(&hot_ok, &store_label, &slow_label); + + // Cold: a refused receiver misses; a side-mask or typed-layout receiver's + // layout record describes the PRE-shape, so it is retired first, exactly + // as the transition lane does. The callee edits side tables only and + // cannot collect. + ctx.current_block = slow_idx; + let refused = ctx + .block() + .and(I32, &hdr, &add_header_refuse_mask().to_string()); + let hdr_ok = ctx.block().icmp_eq(I32, &refused, "0"); + ctx.block().cond_br(&hdr_ok, &forget_label, miss_label); ctx.current_block = forget_idx; super::store_census::bump(ctx, super::store_census::ADD_LAYOUT_FORGET); ctx.block() .call_void("js_gc_key_add_layout_unknown", &[(I64, handle)]); + // Re-read: the call changed the layout bits the bookkeeping tests. + let reserved_addr = ctx.block().sub(I64, handle, "6"); + let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); + let reserved_after = ctx.block().load(I16, &reserved_ptr); ctx.block().br(&store_label); // The transition: stamp the successor, then store the value, then the // GC's obligations for the bits stored. ctx.current_block = store_idx; + let reserved = ctx.block().phi( + I16, + &[(&reserved, &layout_label), (&reserved_after, &forget_label)], + ); super::store_census::bump(ctx, super::store_census::ADD_HIT); - let post_wide = ctx.block().lshr(I64, &shapes, "32"); + let post_wide = ctx.block().lshr(I64, shapes, "32"); let post = ctx.block().trunc(I64, &post_wide, I32); let sid_addr = ctx.block().add(I64, handle, "4"); let sid_ptr = ctx.block().inttoptr(I64, &sid_addr); @@ -578,10 +702,6 @@ fn emit_key_add_hit( // bookkeeping below is guarded only by live tests of the stored bits and // of the receiver's header. ctx.block().store(DOUBLE, &fixed, &slot_ptr); - // Re-read: the layout call above may have changed the layout bits. - let reserved_addr = ctx.block().sub(I64, handle, "6"); - let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); - let reserved = ctx.block().load(I16, &reserved_ptr); emit_static_store_ic_bookkeeping( ctx, handle, &slot, &slot_ptr, &reserved, &fixed, value_bits, "put.pic", ); diff --git a/crates/perry-codegen/src/expr/store_census.rs b/crates/perry-codegen/src/expr/store_census.rs index 3d80c07920..2f1bb94e49 100644 --- a/crates/perry-codegen/src/expr/store_census.rs +++ b/crates/perry-codegen/src/expr/store_census.rs @@ -40,6 +40,9 @@ pub(crate) const BY_NAME_RUNTIME: usize = 11; pub(crate) const BY_NAME_PUT_VALUE: usize = 12; /// Key-add inline hit that first retired the receiver's layout record. pub(crate) const ADD_LAYOUT_FORGET: usize = 13; +/// Key-add inline hit on one of the runtime block's first ways (counted on +/// its own edge, then also as [`ADD_HIT`]). +pub(crate) const ADD_WAY_HIT: usize = 14; pub(crate) fn enabled() -> bool { static ON: std::sync::OnceLock = std::sync::OnceLock::new(); diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index 87e100d6c6..ed626c030f 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -15433,6 +15433,38 @@ fn static_put_value_uses_write_pic_for_call_free_rhs() { ir.contains("put.add.check") && ir.contains("put.add.hit.store"), "a word and way miss must compare the key-add memo before the call:\n{ir}" ); + // The add memo's primary pre-shape is compared right after the word, and + // only its miss reaches the existing-key ways. + let check_block: Vec<&str> = ir + .lines() + .skip_while(|l| !(l.starts_with("put.add.check") && l.trim_end().ends_with(':'))) + .skip(1) + .take_while(|l| !l.trim_end().ends_with(':')) + .collect(); + let check_br = check_block + .iter() + .find(|l| l.contains(" br ")) + .copied() + .unwrap_or(""); + assert!( + check_br.contains("%put.add.chain") && check_br.contains("%put.pic.ways"), + "the key-add primary compare must branch to the add hit or the existing-key ways:\n{ir}" + ); + // After the primary memo, TWO key-add ways: the receiver ShapeId's home, + // `(sid * 0x9E3779B1) >> 26` (packed_add::add_way_home), and the next. + let way_blocks: Vec<&str> = ir + .lines() + .filter(|l| l.starts_with("put.add.way.") && l.trim_end().ends_with(':')) + .collect(); + assert_eq!( + way_blocks.len(), + 2, + "after the primary memo the home way and the next are compared inline:\n{ir}" + ); + assert!( + ir.contains("mul i32") && ir.contains("-1640531535") && ir.contains("lshr i32"), + "the inline way is the ShapeId's home, sid * ADD_WAY_HASH >> 26:\n{ir}" + ); assert_eq!( ir.lines() .filter(|l| l.starts_with("put.pic.way.") && l.trim_end().ends_with(':')) diff --git a/crates/perry-runtime/src/proxy/put_value/packed_add.rs b/crates/perry-runtime/src/proxy/put_value/packed_add.rs index 88f50d4ddc..f98c109e08 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_add.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_add.rs @@ -95,27 +95,58 @@ pub struct PackedSetSite { /// A `*mut AddWays` (0 = none): the memos of further pre-shapes, served /// by [`packed_add_try`]. A base-class constructor's key-add sees one /// pre-shape per subclass (the prototype is part of the shape), so such - /// a site is polymorphic by construction. Emitted code never reads it. + /// a site is polymorphic by construction. The emitted hit compares the + /// ways at the pre-shape's home ([`add_way_home`]) and the one after it, + /// after the primary words. pub add_ways: AtomicU64, } -/// One further memo, in the primary words' format. +/// One further memo, in the primary words' format (`add_shapes`, +/// `add_guard` are a way too: the emitted hit reads either through one +/// pointer). #[repr(C)] pub struct AddWay { shapes: AtomicU64, guard: AtomicU64, } -/// Further memos per site. Filled in order, never evicted (so a site with -/// more stable pre-shapes than ways settles instead of cycling); a site that -/// overflows them re-primes its primary words. 48, not 8: Zod 3's `ZodType` +/// Further memos per site, never evicted (so a site with more stable +/// pre-shapes than ways settles instead of cycling); a site that overflows +/// them re-primes its primary words. A memo is placed at its pre-shape's HOME +/// way ([`add_way_home`]) when that way is free, and otherwise at the next +/// free way from it; the emitted hit compares the home and the way after it +/// ([`ADD_WAY_PROBES`]), the runtime every way. Placement by pre-shape rather +/// than by arrival matters: on +/// tsc the hot memo of a polymorphic site is typically NOT among its first +/// (8 sites whose hits all land on their 3rd way, 10 on their 15th, behind +/// transient first-instance shapes), so no fixed prefix of an in-order list +/// is where the hits are. +/// +/// 64 (a power of two for the home hash), not 8: Zod 3's `ZodType` /// constructor adds its keys to one pre-shape per subclass (36 of them), and /// with 8 ways 15,069 of its 78,250 executed key-adds per 200 parses re-ran -/// the full `[[Set]]` and re-primed. A linear scan of 48 words is a small -/// fraction of that walk, and only a polymorphic site allocates them. -pub const ADD_WAYS: usize = 48; +/// the full `[[Set]]` and re-primed. Only a polymorphic site allocates them. +pub const ADD_WAYS: usize = 1 << ADD_WAYS_LOG2; +/// `log2(ADD_WAYS)`: the home is the top bits of a 32-bit product. +pub const ADD_WAYS_LOG2: u32 = 6; +/// The multiplier of [`add_way_home`] (2^32 / golden ratio): consecutive +/// ShapeIds, which subclass shapes minted in sequence are, land far apart. +pub const ADD_WAY_HASH: u32 = 0x9E37_79B1; +/// Ways the emitted hit compares from the home on (the home, then the next +/// mod [`ADD_WAYS`]): a memo whose home an earlier memo holds lands on the +/// next free way, which is most often the very next. +#[cfg_attr(not(test), allow(dead_code))] +pub const ADD_WAY_PROBES: usize = 2; type AddWays = [AddWay; ADD_WAYS]; +/// The way the emitted hit compares for a receiver of ShapeId `pre`: +/// the top [`ADD_WAYS_LOG2`] bits of `pre * ADD_WAY_HASH` (mod 2^32). +/// **perry-codegen computes the same (`emit_static_store_ic`).** +#[inline] +pub fn add_way_home(pre: u32) -> usize { + (pre.wrapping_mul(ADD_WAY_HASH) >> (32 - ADD_WAYS_LOG2)) as usize +} + impl PackedSetSite { pub const fn empty() -> Self { Self { @@ -134,6 +165,11 @@ pub const ADD_SHAPES_WORD: usize = 1; pub const ADD_GUARD_WORD: usize = 2; #[cfg_attr(not(test), allow(dead_code))] pub const PACKED_SET_SITE_WORDS: usize = 4; +#[cfg_attr(not(test), allow(dead_code))] +pub const ADD_WAYS_WORD: usize = 3; +/// Words of one [`AddWay`]. +#[cfg_attr(not(test), allow(dead_code))] +pub const ADD_WAY_WORDS: usize = 2; /// Low bits of the guard word that hold the slot. pub const ADD_SLOT_BITS: u32 = 16; const ADD_SLOT_MASK: u64 = (1 << ADD_SLOT_BITS) - 1; @@ -263,7 +299,7 @@ const CENSUS_NAMES: [&str; 32] = [ "emit.by_name.runtime", "emit.by_name.put_value", "emit.add.layout_forget", - "emit.14", + "emit.add.way_hit", "emit.15", "rt.add.memo_inline", "rt.add.memo_spill", @@ -364,8 +400,12 @@ pub(crate) unsafe fn packed_add_try( let (shapes, guard) = if matches(primary) { (primary, (*site).add_guard.load(Ordering::Relaxed)) } else { - let way = site_ways(site)? - .iter() + let ways = site_ways(site)?; + // A memo sits at its home way unless that was taken when it was + // placed; the emitted hit has already compared the home. + let home = add_way_home(sid); + let way = (0..ADD_WAYS) + .map(|i| &ways[(home + i) % ADD_WAYS]) .find(|way| matches(way.shapes.load(Ordering::Relaxed)))?; ( way.shapes.load(Ordering::Relaxed), @@ -629,10 +669,14 @@ pub(crate) unsafe fn packed_add_prime( } let displaced_pre = unflip(primary as u32); if let Some(way) = site_ways(site_ptr).and_then(|ways| { - ways.iter().find(|way| { - let word = way.shapes.load(Ordering::Relaxed); - word == PACKED_SET_EMPTY || unflip(word as u32) == displaced_pre - }) + // The home way first, then the rest in order from it. + let home = add_way_home(displaced_pre); + (0..ADD_WAYS) + .map(|i| &ways[(home + i) % ADD_WAYS]) + .find(|way| { + let word = way.shapes.load(Ordering::Relaxed); + word == PACKED_SET_EMPTY || unflip(word as u32) == displaced_pre + }) }) { way.shapes.store(PACKED_SET_EMPTY, Ordering::Relaxed); way.guard.store(displaced_guard, Ordering::Relaxed); diff --git a/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs b/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs index 1fd7539792..a0449d9fea 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs @@ -22,6 +22,36 @@ fn packed_set_site_layout_matches_codegen() { 8 * ADD_GUARD_WORD ); assert_eq!((ADD_SHAPES_WORD, ADD_GUARD_WORD, ADD_SLOT_BITS), (1, 2, 16)); + // ADD_WAYS_WORD, ADD_WAY_WORDS, ADD_WAYS_LOG2, ADD_WAY_HASH: the emitted + // hit reads way `add_way_home(sid)` at `block + 8 * ADD_WAY_WORDS * i`, + // the primary pair through the same pointer arithmetic from + // `ADD_SHAPES_WORD`. + assert_eq!( + std::mem::offset_of!(PackedSetSite, add_ways), + 8 * ADD_WAYS_WORD + ); + assert_eq!(std::mem::size_of::(), 8 * ADD_WAY_WORDS); + assert_eq!(std::mem::offset_of!(AddWay, shapes), 0); + assert_eq!( + std::mem::offset_of!(AddWay, guard), + std::mem::offset_of!(PackedSetSite, add_guard) + - std::mem::offset_of!(PackedSetSite, add_shapes) + ); + assert_eq!( + (ADD_WAYS_WORD, ADD_WAY_WORDS, ADD_WAYS_LOG2, ADD_WAY_HASH), + (3, 2, 6, 0x9E37_79B1) + ); + // ADD_WAY_PROBES: the home and the next way. + assert_eq!(ADD_WAY_PROBES, 2); + assert_eq!(ADD_WAYS, 1 << ADD_WAYS_LOG2); + // The home is a way of the block for every ShapeId. + for sid in [0u32, 1, 0x8000_0000, 0x8000_0001, u32::MAX] { + assert!(add_way_home(sid) < ADD_WAYS); + } + assert_eq!( + add_way_home(0x8000_0001), + (0x8000_0001u32.wrapping_mul(0x9E37_79B1) >> 26) as usize + ); // An empty site's pre half is unmatchable. let empty = PackedSetSite::empty(); assert!(empty.add_shapes.load(Ordering::Relaxed) as u32 >= crate::object::shapes::SHAPE_ID_END); @@ -164,3 +194,49 @@ fn a_published_memo_owns_both_shapes_across_a_full_trace() { "a published memo must own its pre- and post-shape" ); } + +/// A polymorphic site's displaced memos sit at their pre-shape's HOME way — +/// the one way the emitted hit compares — unless that way was already +/// taken, and the runtime serves every memo wherever it sits. Sabotage: +/// placement in arrival order (way 0, 1, ...) -> a home way stays empty +/// while its memo sits elsewhere. +#[test] +fn a_displaced_memo_sits_at_its_home_way() { + let key = interned(b"added_home"); + let srcs: [&[u8]; 6] = [ + b"{\"h0\":1}", + b"{\"h1\":1}", + b"{\"h2\":1}", + b"{\"h3\":1}", + b"{\"h4\":1}", + b"{\"h5\":1}", + ]; + let site = leaked_site(); + let mut pres = Vec::new(); + for src in srcs { + let first = parsed(src); + pres.push(stamp(first)); + miss(site, first, key, 1.0); + } + let ways = unsafe { site_ways(site) }.expect("a polymorphic site has ways"); + let primary = site.add_shapes.load(Ordering::Relaxed) as u32; + assert_eq!(primary, *pres.last().unwrap(), "the newest memo is primary"); + for &pre in &pres[..pres.len() - 1] { + let at_home = ways[add_way_home(pre)].shapes.load(Ordering::Relaxed); + assert!( + at_home as u32 == pre || at_home != PACKED_SET_EMPTY, + "memo {pre:#x}: its home way {} is empty, so it must sit there", + add_way_home(pre) + ); + } + for (i, src) in srcs.iter().enumerate() { + let next = parsed(src); + assert_eq!(stamp(next), pres[i]); + let served = if pres[i] == primary { + Some(2.0) + } else { + unsafe { packed_add_try(site, next, 2.0) } + }; + assert_eq!(served, Some(2.0), "memo {i} must be served"); + } +} diff --git a/crates/perry/tests/keyadd_store_ic.rs b/crates/perry/tests/keyadd_store_ic.rs index abeea691e6..616175ad1d 100644 --- a/crates/perry/tests/keyadd_store_ic.rs +++ b/crates/perry/tests/keyadd_store_ic.rs @@ -19,6 +19,21 @@ fn perry_bin() -> PathBuf { /// Compile `source` with the store census, run it, and return (stdout, the /// inline add hits, the runtime memo serves). fn run(source: &str) -> (String, u64, u64) { + let (stdout, stderr) = run_census(source); + let memo = census(&stderr, "rt.add.memo_inline") + census(&stderr, "rt.add.memo_spill"); + (stdout, census(&stderr, "emit.add.inline_hit"), memo) +} + +/// One counter of the census line the binary printed to stderr. +fn census(stderr: &str, name: &str) -> u64 { + stderr + .split_whitespace() + .find_map(|w| w.strip_prefix(name)?.strip_prefix('=')?.parse().ok()) + .unwrap_or(0) +} + +/// Compile `source` with the store census, run it, and return (stdout, stderr). +fn run_census(source: &str) -> (String, String) { let dir = tempfile::tempdir().expect("tempdir"); let entry = dir.path().join("main.ts"); let output = dir.path().join("main_bin"); @@ -50,17 +65,9 @@ fn run(source: &str) -> (String, u64, u64) { "binary failed ({:?})\nstderr:\n{stderr}", run.status ); - let count = |name: &str| -> u64 { - stderr - .split_whitespace() - .find_map(|w| w.strip_prefix(name)?.strip_prefix('=')?.parse().ok()) - .unwrap_or(0) - }; - let memo = count("rt.add.memo_inline") + count("rt.add.memo_spill"); ( String::from_utf8_lossy(&run.stdout).trim().to_owned(), - count("emit.add.inline_hit"), - memo, + stderr, ) } @@ -314,3 +321,57 @@ console.log(s, before, after, Object.keys(P).join(","), Q.k, Object.keys(Q).leng "the prototype must miss the inline add (first receiver primes)" ); } + +/// One site fed receivers of four key lists holds four memos: the primary +/// and three ways of the runtime's block, each at its pre-shape's home way +/// unless an earlier memo took it, then at the next free way (the emitted +/// hit compares the home and the next). Each +/// has its own successor shape and slot (`{}` adds at slot 0, the others at +/// slot 1), and an inherited setter appearing later must stop every one of +/// them. Sabotage: a way hit reads the PRIMARY pair's guard -> `z` of a +/// one-key receiver lands in slot 0 over its first key. +#[test] +fn a_polymorphic_site_serves_its_first_ways_inline() { + let (stdout, stderr) = run_census( + r#"// One key-add site fed receivers of four key lists, so it holds four memos: +// the primary and three ways, with different successor shapes and slots. +function addZ(o: any, v: number) { o.z = v; } +function mk(i: number): any { + const o: any = {}; + const k = i % 4; + if (k === 1) o.a = i; + if (k === 2) o.b = i; + if (k === 3) o.c = i; + return o; +} +const objs: any[] = []; +for (let i = 0; i < 4000; i++) { const o = mk(i); addZ(o, i * 2); objs.push(o); } +let s = 0; +for (let i = 0; i < 4000; i++) s += objs[i].z * (i % 4 + 1); +const shapes = [objs[3996], objs[3997], objs[3998], objs[3999]].map((o) => Object.keys(o).join("") + "=" + Object.values(o).join(",")).join(" "); +// An inherited setter for the key appears: every memo, way or primary, must refuse. +const log: number[] = []; +Object.defineProperty(Object.prototype, "z", { set(v: number) { log.push(v); }, configurable: true }); +for (let i = 0; i < 4; i++) addZ(mk(i), 100 + i); +delete (Object.prototype as any).z; +console.log(s, shapes, log.join(",")); +"#, + ); + assert_eq!( + stdout, + r#"40000000 z=7992 az=3997,7994 bz=3998,7996 cz=3999,7998 100,101,102,103"# + ); + // Each shape's first receiver primes: 999 more of each hit a memo, the + // three displaced ones inline at their home way or the next. + let way_hits = census(&stderr, "emit.add.way_hit"); + let memo = census(&stderr, "rt.add.memo_inline"); + assert_eq!( + way_hits + memo, + 2997, + "the three displaced memos serve 999 adds each (census: {stderr})" + ); + assert!( + way_hits >= 1998, + "at most one memo can sit beyond its home's next way (census: {stderr})" + ); +} From 0a3228b046de2ec0f90a66787cbe19f6b8e1f7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 26 Sep 2026 07:14:10 +0000 Subject: [PATCH 3/6] changelog: keyadd-poly --- changelog.d/keyadd-poly.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog.d/keyadd-poly.md diff --git a/changelog.d/keyadd-poly.md b/changelog.d/keyadd-poly.md new file mode 100644 index 0000000000..af82a5c309 --- /dev/null +++ b/changelog.d/keyadd-poly.md @@ -0,0 +1,10 @@ +Polymorphic key-adding stores are now served inline. A store site that holds +several key-add memos (one per receiver pre-shape, e.g. a base-class +constructor seeing each subclass) places each further memo at its pre-shape's +home way, a hash of the ShapeId, and the emitted hit compares that one way +after the primary memo: one extra compare whatever the number of shapes. The +key-add memo is also compared before the existing-key ways, the per-object +header checks are one test on the hot path, and a key-add on a typed-layout +receiver (an object literal) no longer takes the typed-feedback registry lock +when typed feedback is off, which cut about 180 instructions from each such +add. From e3c6339ed79b3d8480ae90bc81904bf432c571e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 26 Sep 2026 10:31:25 +0000 Subject: [PATCH 4/6] perf(runtime): a key-add memo served from beyond the inline ways moves into them On tsc eight polymorphic sites keep 21 memos each and serve every hit from a way 2-3 past its home: the home and the next were taken by transient first-instance shapes placed earlier. When the runtime serves a memo from beyond the two ways the emitted hit compares, it now moves the memo into the second of them (or the home), trading places with a memo that is not at its own home. tsc runtime-served key-adds per transpile: 406,464 -> 152,424. --- .../src/proxy/put_value/packed_add.rs | 62 +++++++++++++++++-- .../src/proxy/put_value/packed_add_tests.rs | 55 ++++++++++++++++ 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/crates/perry-runtime/src/proxy/put_value/packed_add.rs b/crates/perry-runtime/src/proxy/put_value/packed_add.rs index f98c109e08..a3aa5e202c 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_add.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_add.rs @@ -115,7 +115,8 @@ pub struct AddWay { /// them re-primes its primary words. A memo is placed at its pre-shape's HOME /// way ([`add_way_home`]) when that way is free, and otherwise at the next /// free way from it; the emitted hit compares the home and the way after it -/// ([`ADD_WAY_PROBES`]), the runtime every way. Placement by pre-shape rather +/// ([`ADD_WAY_PROBES`]), the runtime every way, and a memo the runtime serves +/// from further away is moved into one of the two ([`promote_way`]). Placement by pre-shape rather /// than by arrival matters: on /// tsc the hot memo of a polymorphic site is typically NOT among its first /// (8 sites whose hits all land on their 3rd way, 10 on their 15th, behind @@ -404,13 +405,17 @@ pub(crate) unsafe fn packed_add_try( // A memo sits at its home way unless that was taken when it was // placed; the emitted hit has already compared the home. let home = add_way_home(sid); - let way = (0..ADD_WAYS) - .map(|i| &ways[(home + i) % ADD_WAYS]) - .find(|way| matches(way.shapes.load(Ordering::Relaxed)))?; - ( + let distance = (0..ADD_WAYS) + .find(|&i| matches(ways[(home + i) % ADD_WAYS].shapes.load(Ordering::Relaxed)))?; + let way = &ways[(home + distance) % ADD_WAYS]; + let found = ( way.shapes.load(Ordering::Relaxed), way.guard.load(Ordering::Relaxed), - ) + ); + if distance >= ADD_WAY_PROBES { + promote_way(ways, home, distance); + } + found }; let spill = shapes as u32 != sid; if guard >> ADD_SLOT_BITS != add_generation() { @@ -461,6 +466,51 @@ pub(crate) unsafe fn packed_add_try( Some(value) } +/// A memo the runtime just served lies beyond the ways the emitted hit +/// compares (its home and the next were taken when it was placed, typically +/// by a polymorphic site's transient first-instance shapes). Move it into +/// one of those two ways, so its next receiver is served inline, and move +/// that way's memo to where it was. A way whose memo sits at its OWN home is +/// kept (its receivers are served inline already); with both kept nothing +/// moves. Every memo stays in the block, so the runtime still serves each. +/// +/// Only the primary agent publishes a site's memos (see `# Agents`), and only +/// it ever matches them, so the moves are ordered with its own reads. Each +/// way is retired (`shapes` EMPTY) before its guard changes and republished +/// last, as [`packed_add_prime`] does. +fn promote_way(ways: &AddWays, home: usize, distance: usize) { + if crate::agent::current_agent() != crate::agent::PRIMARY_AGENT { + return; + } + let from = (home + distance) % ADD_WAYS; + let shapes = ways[from].shapes.load(Ordering::Relaxed); + if shapes as u32 != unflip(shapes as u32) { + // A spill memo: the emitted hit never takes it, wherever it sits. + return; + } + let at_own_home = |idx: usize| { + let word = ways[idx].shapes.load(Ordering::Relaxed); + word != PACKED_SET_EMPTY && add_way_home(unflip(word as u32)) == idx + }; + let second = (home + 1) % ADD_WAYS; + let Some(to) = [second, home].into_iter().find(|&idx| !at_own_home(idx)) else { + return; + }; + let (to_shapes, to_guard) = ( + ways[to].shapes.load(Ordering::Relaxed), + ways[to].guard.load(Ordering::Relaxed), + ); + let guard = ways[from].guard.load(Ordering::Relaxed); + ways[from].shapes.store(PACKED_SET_EMPTY, Ordering::Relaxed); + ways[to].shapes.store(PACKED_SET_EMPTY, Ordering::Relaxed); + ways[to].guard.store(guard, Ordering::Relaxed); + ways[to].shapes.store(shapes, Ordering::Relaxed); + if to_shapes != PACKED_SET_EMPTY { + ways[from].guard.store(to_guard, Ordering::Relaxed); + ways[from].shapes.store(to_shapes, Ordering::Relaxed); + } +} + /// The key-add hit's layout retirement: a receiver whose layout record /// (side mask or typed descriptor) described its PRE-shape. Exactly what the /// transition lane runs before its stamp. Edits header bits and layout / diff --git a/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs b/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs index a0449d9fea..50ca188bf4 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_add_tests.rs @@ -240,3 +240,58 @@ fn a_displaced_memo_sits_at_its_home_way() { assert_eq!(served, Some(2.0), "memo {i} must be served"); } } + +fn fresh_ways() -> Box { + Box::new(std::array::from_fn(|_| AddWay { + shapes: AtomicU64::new(PACKED_SET_EMPTY), + guard: AtomicU64::new(0), + })) +} + +/// The first ShapeId at or after `from` whose home is `home`. +fn sid_with_home(home: usize, from: u32) -> u32 { + (from..).find(|&sid| add_way_home(sid) == home).unwrap() +} + +/// A memo the runtime serves from beyond the two ways the emitted hit +/// compares moves into the second of them, trading places with a memo that +/// was not at its own home; a way whose memo IS at its own home is kept. +/// Sabotage: `promote_way` moves nothing -> the hot memo stays out of reach +/// of the emitted hit (tsc: 8 sites, every hit 2-3 ways from home). +#[test] +fn a_far_memo_moves_into_the_inline_ways() { + let base = crate::object::shapes::SHAPE_ID_BASE; + let h = 10usize; + let hot = sid_with_home(h, base); + let at_home = sid_with_home(h, hot + 1); + let stray = sid_with_home(40, base); + let ways = fresh_ways(); + let word = |sid: u32| u64::from(sid) | (u64::from(sid + 1) << 32); + ways[h].shapes.store(word(at_home), Ordering::Relaxed); + ways[h].guard.store(1, Ordering::Relaxed); + ways[h + 1].shapes.store(word(stray), Ordering::Relaxed); + ways[h + 1].guard.store(2, Ordering::Relaxed); + ways[h + 3].shapes.store(word(hot), Ordering::Relaxed); + ways[h + 3].guard.store(3, Ordering::Relaxed); + promote_way(&ways, h, 3); + let at = |i: usize| { + ( + ways[i].shapes.load(Ordering::Relaxed) as u32, + ways[i].guard.load(Ordering::Relaxed), + ) + }; + assert_eq!(at(h), (at_home, 1), "a memo at its own home is kept"); + assert_eq!( + at(h + 1), + (hot, 3), + "the served memo moves in with its guard" + ); + assert_eq!(at(h + 3), (stray, 2), "the displaced memo takes its place"); + // Both inline ways hold memos at their own homes: nothing moves. + let next_home = sid_with_home(h + 1, base); + ways[h + 1].shapes.store(word(next_home), Ordering::Relaxed); + ways[h + 3].shapes.store(word(hot), Ordering::Relaxed); + promote_way(&ways, h, 3); + assert_eq!(at(h + 1).0, next_home); + assert_eq!(at(h + 3).0, hot); +} From 451cdadf5c9cc3139c477a97c5a43ed262b0a31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 26 Sep 2026 14:03:42 +0000 Subject: [PATCH 5/6] changelog: keyadd-poly names the far-memo move --- changelog.d/keyadd-poly.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changelog.d/keyadd-poly.md b/changelog.d/keyadd-poly.md index af82a5c309..ad40a1c8b3 100644 --- a/changelog.d/keyadd-poly.md +++ b/changelog.d/keyadd-poly.md @@ -8,3 +8,7 @@ header checks are one test on the hot path, and a key-add on a typed-layout receiver (an object literal) no longer takes the typed-feedback registry lock when typed feedback is off, which cut about 180 instructions from each such add. +A memo the runtime serves from beyond the two ways the emitted hit compares +(its home was taken by an earlier, often transient, shape) now moves into one +of them, so a site's hot shapes end up served inline; on tsc this cut +runtime-served key-adds per transpile from 406,464 to 152,424. From c96b58470a15279158b50de4029f4d451c557240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 26 Sep 2026 21:55:08 +0000 Subject: [PATCH 6/6] changelog: name the fragment after PR #11436 --- changelog.d/{keyadd-poly.md => 11436-keyadd-poly.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{keyadd-poly.md => 11436-keyadd-poly.md} (100%) diff --git a/changelog.d/keyadd-poly.md b/changelog.d/11436-keyadd-poly.md similarity index 100% rename from changelog.d/keyadd-poly.md rename to changelog.d/11436-keyadd-poly.md