diff --git a/changelog.d/11657-megamorphic-read-miss-front.md b/changelog.d/11657-megamorphic-read-miss-front.md new file mode 100644 index 0000000000..13d0cb22ad --- /dev/null +++ b/changelog.d/11657-megamorphic-read-miss-front.md @@ -0,0 +1,11 @@ +A generic property read keeps only the ShapeId compare and the slot load +inline. Its miss makes one GC-leaf call, `js_object_get_field_ic_front`, which +answers a polymorphic way, a spill entry, or a latched megamorphic site whose +slot guess the receiver's own shape record confirms (one POSBOUND bound +compare and one key-atom word compare, with a bounded second chance over the +first 32 positional keys that re-aims the guess). Only what the front declines +reaches the collecting slow entry, which also asks the inherited-read cache +for a never-primed site. Nothing is spilled or relocated across the front +call, and the site reads its agent's shape directory without a call on ELF +executables, Windows x86-64 and Apple aarch64. The shape record grows from 40 +to 48 bytes; tsc's `.text` shrinks by 9%. diff --git a/crates/perry-abi/src/lib.rs b/crates/perry-abi/src/lib.rs index 5c358a16c8..3ede028951 100644 --- a/crates/perry-abi/src/lib.rs +++ b/crates/perry-abi/src/lib.rs @@ -18,10 +18,13 @@ pub const STATIC_SHAPE_ID_COUNT: u32 = 1 << 20; pub const ARRAY_HEADER_SIZE: usize = 8; /// `agent_ptrs::PERRY_AGENT_PTRS`: the number of per-agent pointer slots. -/// Slot 0 is reserved (the megamorphic follow-up's shape-record directory); -/// slot 1 held the implicit-`this` cell's address until this-as-a-parameter -/// deleted the cell, and is free; slot 2 is the stack limit. pub const AGENT_PTR_SLOTS: usize = 4; +/// Slot 0: the address of this agent's ordinary shape-directory mirror +/// (`shapes_store::ORDINARY_DIR`), which a generic read site passes to its +/// GC-leaf miss front (`js_object_get_field_ic_front`) so the front reads no +/// thread-local. Slot 1 held the implicit-`this` cell's address until +/// this-as-a-parameter deleted the cell, and is free; slot 2 is the stack limit. +pub const AGENT_PTR_SHAPE_DIR: usize = 0; /// Slot 2: this agent's stack limit (#10812) — not a pointer to anything, the /// lowest frame address a compiled prologue accepts before it throws /// `RangeError: Maximum call stack size exceeded`. Null means unchecked. diff --git a/crates/perry-codegen/src/expr/agent_ptr.rs b/crates/perry-codegen/src/expr/agent_ptr.rs index 7ec3f6bdb0..5425c9d7d7 100644 --- a/crates/perry-codegen/src/expr/agent_ptr.rs +++ b/crates/perry-codegen/src/expr/agent_ptr.rs @@ -15,10 +15,22 @@ //! model (every thread-local access is a TLV thunk call), so the block's //! address is read from the runtime's `HotTls` cache through the pthread //! TSD fast path (`hot_tls.rs`), at `HOT_TLS_AGENT_PTRS_OFFSET`. -//! * [`AgentPtrAccess::Call`] — everything else (Windows, wasm, arm64_32, -//! x86-64 Darwin, dylib/staticlib outputs): the runtime accessor. +//! * [`AgentPtrAccess::WindowsTeb`] — Windows x86-64, any output kind: the +//! same sequence the compiler emits for a native thread-local, spelled out +//! because the runtime cannot export the block under a stable name there +//! (`agent_ptrs.rs`): `gs:[0x58]` (the TEB's `ThreadLocalStoragePointer`) +//! indexed by the image's `_tls_index` gives this thread's TLS block for +//! the image, and the block sits at `PERRY_AGENT_PTRS_SECREL` (a `.secrel32` +//! the runtime emits for its own static) inside it. Emitted code and the +//! runtime are linked into ONE image, so `_tls_index` is theirs. +//! * [`AgentPtrAccess::Call`] — everything else (wasm, arm64_32, Windows +//! aarch64, x86-64 Darwin, ELF dylib/staticlib outputs): the runtime +//! accessor. x86-64 Darwin has no call-free thread-local model (every +//! Mach-O thread-local access is a TLV thunk call) and the runtime's +//! pthread-TSD fast path (`HotTls`, the Apple aarch64 route) is built for +//! aarch64 only. //! -//! In both inline forms a null slot means "not published yet" and takes the +//! In every inline form a null slot means "not published yet" and takes the //! accessor call, which publishes it; so the inline forms and the call are //! equivalent by construction. @@ -35,10 +47,18 @@ pub(crate) const AGENT_PTRS_SYMBOL: &str = "PERRY_AGENT_PTRS"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum AgentPtrAccess { InitialExec, + WindowsTeb, AppleTsd, Call, } +/// `PERRY_AGENT_PTRS`'s offset in the image's TLS block on Windows x86-64 +/// (`agent_ptrs.rs`), and the image's TLS index. +pub(crate) const AGENT_PTRS_SECREL_SYMBOL: &str = "PERRY_AGENT_PTRS_SECREL"; +pub(crate) const TLS_INDEX_SYMBOL: &str = "_tls_index"; +/// `NT_TIB64`/`TEB64.ThreadLocalStoragePointer`, off `gs`. +const TEB_TLS_POINTER_OFFSET: &str = "88"; + thread_local! { /// Whether the module being compiled is linked into an EXECUTABLE (set per /// module by `codegen::compile_module`); anything else must not assume @@ -62,12 +82,107 @@ pub(crate) fn agent_ptr_access(ctx: &FnCtx<'_>) -> AgentPtrAccess { if elf && OUTPUT_IS_EXECUTABLE.with(|c| c.get()) { return AgentPtrAccess::InitialExec; } + if triple.starts_with("x86_64") && triple.contains("windows") { + return AgentPtrAccess::WindowsTeb; + } if super::hot_tls::inline_hot_tls_enabled(ctx) { return AgentPtrAccess::AppleTsd; } AgentPtrAccess::Call } +/// The current value of per-agent pointer `slot`, for a GC-leaf callee that +/// accepts `absent` (a constant operand meaning "not available here") in its +/// place: one initial-exec load in an ELF executable (the slot must never be +/// null there); the `HotTls` read on Apple aarch64, `absent` when the direct +/// TSD path is unavailable or the block is not published; the slot's `gc-leaf` +/// runtime accessor everywhere else. No null test and no fallback call on the +/// inline forms, so a site pays only the read. Ends in the block where the +/// returned register holds the value. +pub(crate) fn emit_agent_ptr_or(ctx: &mut FnCtx<'_>, slot: usize, absent: &str) -> String { + debug_assert!(slot < AGENT_PTR_SLOTS); + let slot_off = (slot * 8).to_string(); + let access = agent_ptr_access(ctx); + match access { + AgentPtrAccess::InitialExec | AgentPtrAccess::WindowsTeb => { + let at = emit_slot_addr(ctx, access, &slot_off); + ctx.block().load(PTR, &at) + } + AgentPtrAccess::AppleTsd => { + let lookup = super::hot_tls::emit_hot_tls_lookup(ctx, "agent_ptr"); + let field = super::hot_tls::hot_tls_field( + ctx, + &lookup.hot, + &HOT_TLS_AGENT_PTRS_OFFSET.to_string(), + ); + let blk = ctx.block(); + let block_ptr = blk.load(PTR, &field); + let at = blk.gep( + crate::types::I8, + &block_ptr, + &[(crate::types::I64, &slot_off)], + ); + let val = blk.load(PTR, &at); + let fast_pred = blk.label.clone(); + let join_idx = ctx.new_block("agent_ptr.join"); + let join_label = ctx.block_label(join_idx); + ctx.block().br(&join_label); + ctx.current_block = lookup.slow_idx; + let slow_pred = ctx.block().label.clone(); + ctx.block().br(&join_label); + ctx.current_block = join_idx; + ctx.block() + .phi(PTR, &[(&val, &fast_pred), (absent, &slow_pred)]) + } + AgentPtrAccess::Call => ctx.block().call(PTR, agent_ptr_accessor(slot), &[]), + } +} + +/// The address of the slot `slot_off` bytes into this thread's block, for the +/// two forms that name the block through the thread pointer (module docs). +pub(crate) fn emit_slot_addr( + ctx: &mut FnCtx<'_>, + access: AgentPtrAccess, + slot_off: &str, +) -> String { + let blk = ctx.block(); + let block = match access { + AgentPtrAccess::InitialExec => format!("@{AGENT_PTRS_SYMBOL}"), + AgentPtrAccess::WindowsTeb => { + // `mov gs:[0x58]` — a plain load in the x86 `gs` address space + // (256). Plain, not volatile: it is re-read after every call, and + // a thread switch happens only inside a call. + let tls_array = blk.next_reg(); + blk.emit_raw(format!( + " {tls_array} = load ptr, ptr addrspace(256) inttoptr (i64 {TEB_TLS_POINTER_OFFSET} to ptr addrspace(256)), align 8" + )); + let index = blk.load(crate::types::I32, &format!("@{TLS_INDEX_SYMBOL}")); + let index = blk.zext(crate::types::I32, &index, crate::types::I64); + let entry = blk.gep(PTR, &tls_array, &[(crate::types::I64, &index)]); + let image_block = blk.load(PTR, &entry); + let secrel = blk.load(crate::types::I32, &format!("@{AGENT_PTRS_SECREL_SYMBOL}")); + let secrel = blk.zext(crate::types::I32, &secrel, crate::types::I64); + blk.gep( + crate::types::I8, + &image_block, + &[(crate::types::I64, &secrel)], + ) + } + AgentPtrAccess::AppleTsd | AgentPtrAccess::Call => { + unreachable!("{access:?} does not name the block through the thread pointer") + } + }; + blk.gep(crate::types::I8, &block, &[(crate::types::I64, slot_off)]) +} + +/// The `gc-leaf` runtime accessor of per-agent pointer `slot`. +fn agent_ptr_accessor(slot: usize) -> &'static str { + match slot { + crate::runtime_abi::AGENT_PTR_SHAPE_DIR => "perry_shape_dir_cell", + _ => unreachable!("agent pointer slot {slot} has no accessor"), + } +} + /// Emit a load of per-agent pointer `slot`, falling back to `fallback_fn` /// (a `gc-leaf` runtime accessor `() -> ptr` that also publishes it). Ends /// in a fresh block where the returned register holds the pointer. @@ -79,15 +194,11 @@ pub(crate) fn emit_agent_ptr(ctx: &mut FnCtx<'_>, slot: usize, fallback_fn: &str } let slot_off = (slot * 8).to_string(); let (fast_pred, fast_val, slow_idx) = match access { - AgentPtrAccess::InitialExec => { + AgentPtrAccess::InitialExec | AgentPtrAccess::WindowsTeb => { let slow_idx = ctx.new_block("agent_ptr.slow"); let fast_idx = ctx.new_block("agent_ptr.fast"); + let at = emit_slot_addr(ctx, access, &slot_off); let blk = ctx.block(); - let at = blk.gep( - crate::types::I8, - &format!("@{AGENT_PTRS_SYMBOL}"), - &[(crate::types::I64, &slot_off)], - ); let val = blk.load(PTR, &at); let ok = blk.icmp_ne(PTR, &val, "null"); let fast_label = ctx.block_label(fast_idx); diff --git a/crates/perry-codegen/src/expr/property_get/array_length_tests.rs b/crates/perry-codegen/src/expr/property_get/array_length_tests.rs index d664efe8d6..51819db528 100644 --- a/crates/perry-codegen/src/expr/property_get/array_length_tests.rs +++ b/crates/perry-codegen/src/expr/property_get/array_length_tests.rs @@ -329,7 +329,26 @@ fn a_length_read_serves_a_live_plain_array_off_the_shape_compare() { // cannot heal in one edge — continues exactly where the compare's false // edge used to go. let refused = assert_plain_array_arm(&blocks, "pget.array_kind", true); - assert_eq!(strip_suffix(&refused), "pic.token.miss"); + assert_eq!(strip_suffix(&refused), "pic.miss.front"); + + // S6: a `length` site's miss front is handed the runtime's EMPTY + // directory, so its latched edge is never confirmed from the receiver's + // shape. An Array-subclass receiver serves `length` from its elements + // store; the `length` its shape may name is not the answer + // (`read_confirm::tests::a_length_site_is_never_confirmed_from_the_shape` + // is the runtime half). + let front = ir + .lines() + .find(|l| l.contains(" = call double @js_object_get_field_ic_front(")) + .unwrap_or_else(|| panic!("expected the miss front call:\n{ir}")); + assert!( + front.contains("@js_object_get_field_ic_front(ptr @PERRY_EMPTY_SHAPE_DIR, "), + "a `length` site must pass the empty directory:\n{front}" + ); + assert!( + !ir.contains("ptr @PERRY_AGENT_PTRS, i64 0"), + "a `length` site reads no directory at all:\n{ir}" + ); // The merge takes the arm's value. let (load_label, load_body) = block(&blocks, "pget.array_length"); diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 8c4f3e0074..3f8ec19c87 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -19,8 +19,8 @@ use crate::types::{DOUBLE, I1, I32, I64, I8, PTR}; /// Since #9708 codegen emits only the 8-byte slot (`@perry_ic_N = private /// global ptr null`) and the runtime allocates the words itself, sized from /// its own `PicCache` — so the constant is no longer an emission width, but -/// the emitted way GEPs (`PIC_WAY_BASE + PIC_WAYS * 2` words) must still -/// land inside that allocation. perry-codegen does not depend on +/// the runtime's miss entry reads its ways at `PIC_WAY_BASE + PIC_WAYS * 2` +/// words, which the pairing tests keep inside that allocation. perry-codegen does not depend on /// perry-runtime (the same reason `INLINE_SLOT_FLOOR` is duplicated in /// `target_layout`), so the pairing is held by `pic_cache_layout_matches_runtime` /// here and `pic_cache_words_match_codegen` in the runtime: change one and both @@ -29,9 +29,11 @@ use crate::types::{DOUBLE, I1, I32, I64, I8, PTR}; pub(crate) const PIC_CACHE_WORDS: usize = 12; /// First word of the polymorphic way array (words 0..2 are the MRU entry and /// word 3 is the gate). Mirrors the runtime's `PIC_WAY_BASE`. +#[cfg(test)] pub(crate) const PIC_WAY_BASE: usize = 4; /// `(token, slot)` ways beyond the MRU entry; a site resolves `PIC_WAYS + 1` /// shapes inline. Mirrors the runtime's `PIC_WAYS`. +#[cfg(test)] pub(crate) const PIC_WAYS: usize = 4; /// The value a per-site compact MRU word (`@perry_ic_N_packed_get`) holds /// before anything has primed it. @@ -63,14 +65,17 @@ pub(crate) const PACKED_GET_EMPTY: i64 = 0xFFFF_FFFF; /// The hit path's compare therefore REFUSES a spill entry without asking a /// question of its own, which is what lets the overflow-bit test (a 10-byte /// `movabs`, a `test` and a branch, on every read of every site) leave the hit -/// path entirely. The spill entry is still served: `pic.token.miss` un-flips -/// the bit, and a match branches straight to the slow entry, which decodes the -/// same word. See the design note at the head of this function. +/// path entirely. The spill entry is still served: the site's one miss call +/// (`js_object_get_field_ic_slow`) un-flips the bit first thing. Codegen no +/// longer reads the word's encoding; the mirror is kept for the pairing tests. +#[cfg(test)] pub(crate) const PACKED_SPILL_FLIP: i64 = 0xC000_0000; /// Way-state word: `> 0` means at least one way is populated and the compares /// are worth running; `0` (fresh) and a negative megamorphic countdown -/// both skip them. Mirrors the runtime's `PIC_WAY_STATE`. +/// both skip them. Mirrors the runtime's `PIC_WAY_STATE` (read there only, +/// by the miss entry; kept here for the pairing tests). +#[cfg(test)] pub(crate) const PIC_WAY_STATE: usize = 3; // Word 2 is unused: it held the Array-subclass named-prefix token, site state // not derived from one shape, retired by S6. A site holds `(ShapeId, slot)` @@ -90,6 +95,11 @@ pub(crate) const PIC_WAY_STATE: usize = 3; /// re-reading it at each consumer is not merely cheap, it is the correct /// reading: every cold block sees the pool's current address rather than one /// captured before whatever collected. +/// The runtime's never-written empty shape directory (`shapes_store.rs`), +/// which confirms nothing: a `length` site's front operand, and the value +/// where the agent's own directory is not readable inline. +const EMPTY_SHAPE_DIR: &str = "@PERRY_EMPTY_SHAPE_DIR"; + fn emit_key_handle(ctx: &mut FnCtx<'_>, key_handle_global: &str) -> String { let blk = ctx.block(); let key_box = blk.load(DOUBLE, key_handle_global); @@ -778,9 +788,7 @@ pub(crate) fn lower_generic_property_get( // the hit path pays a `jmp` to the survivor instead of falling through. let hit_live_idx = crate::expr::typed_feedback_emission_enabled().then(|| ctx.new_block("pic.hit.live")); - let miss_idx = ctx.new_block("pic.miss"); let hit_label = ctx.block_label(hit_idx); - let miss_label = ctx.block_label(miss_idx); // Small-handle receivers (native-module registry ids) must never be // dereferenced. Pre-#7883 they were kept out of the loads by selecting a // sentinel address and AND-ing `is_real_ptr` into `hit`; the branch does @@ -879,12 +887,11 @@ pub(crate) fn lower_generic_property_get( ctx.block().inttoptr(I64, &pcid_addr) } }; - // The hot ShapeId load has exactly ONE use: the compare. The two cold - // consumers of the same word — the spill compare in `pic.token.miss` and - // the way token in `pic.ways` — read it AGAIN there, through an atomic - // load that GVN will not merge with this one. That is a deliberate - // re-derivation on the miss path (one load, on a path that is about to - // spend hundreds), and it is what lets isel fold this load into the + // The hot ShapeId load has exactly ONE use: the compare. The cold + // consumers of the same word — the spill compare and the way tokens — + // live in the miss front, which reads it AGAIN from the receiver. That is + // a deliberate re-derivation on the miss path (one load, on a path that + // is about to make a call), and it is what lets isel fold this load into the // compare itself: `cmp %ecx, 4(%rdi)` instead of a `mov` and a `cmp`, // one instruction fewer on every hit. With the word live into the cold // blocks it had to sit in a register. @@ -895,11 +902,27 @@ pub(crate) fn lower_generic_property_get( // proves the shape without a discriminator OR or a wide token mask. let packed_stamp = ctx.block().trunc(I64, &packed_word, I32); let token_eq = ctx.block().icmp_eq(I32, &pcid, &packed_stamp); - let token_miss_idx = ctx.new_block("pic.token.miss"); - let token_miss_label = ctx.block_label(token_miss_idx); + // What stays inline is the hit: the ShapeId compare and the load + // (first-read D3). Everything else a miss can be — a spill entry, a + // polymorphic way, a latched site's slot guess, an inherited read, the + // collecting miss — is answered by a runtime call. + // + // Outside profiling builds, the compare's false edge first makes ONE + // GC-leaf call, `js_object_get_field_ic_front` (`pic.miss.front`): it + // answers a spill entry, a polymorphic way and a latched site's + // shape-confirmed slot guess, and returns `TAG_HOLE` for anything else. + // Only then does the site branch to the collecting slow call, so the + // statepoint spills and reloads that call needs sit on that cold edge + // alone. Receiver-validation failures skip the front: it answers + // nothing for a receiver that is not a real object. + let front_idx = + (!crate::expr::typed_feedback_emission_enabled()).then(|| ctx.new_block("pic.miss.front")); + let token_miss_label = front_idx + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| cold_label.clone()); // `.length` on a plain Array (#10714), tested on the compare's FALSE edge // and nowhere earlier. See `emit_plain_array_length_arm` for what it - // answers and what it leaves to `pic.token.miss`. + // answers and what it leaves to the miss front (`pic.miss.front`). // // An Array can never take the hit: its `+4` word is `capacity`, a count // rule 3 (#10828) bounds below the ShapeId floor, so the compare above @@ -933,102 +956,6 @@ pub(crate) fn lower_generic_property_get( .cond_br(&token_eq, &hit_label, &token_miss_label); } - ctx.current_block = token_miss_idx; - // The cold re-read of the ShapeId word — see the hot load above. - let pcid = ctx.block().load_atomic_monotonic(I32, &pcid_ptr, 4); - let pcid64 = ctx.block().zext(I32, &pcid, I64); - // pic_prime_get is the only production writer of get-cache tokens and - // refuses the zero-ShapeId token. All remaining tokens carry a valid, - // never-reused ShapeId; vacant entries are zero. Equality therefore - // proves a nonzero stamp without another check on every property read. - // Keyless Object.create(proto) receivers still miss and walk prototypes. - let token = ctx.block().or(I64, &pcid64, "4611686018427387904"); - // The SPILL entry — tested HERE, and nowhere on the hit path. - // - // A key past the object's inline region used to publish its slot into the - // compact word with `IC_SLOT_OVERFLOW_BIT` set, and every read of every - // site paid to ask whether the bit was there: LLVM folds - // `((packed >> 32) & (1 << 30)) == 0` into `packed & (1 << 62)`, which is - // a 10-byte `movabs`, a `test` and a branch on the hit path of sites whose - // field is inline and can never see the bit. - // - // Now a spill entry publishes the SAME ShapeId with `PACKED_SPILL_FLIP` - // flipped into it, which lands it outside the ShapeId range, so the hit - // path's compare refuses it for free. Un-flipping the bit here recognises - // it in three instructions ON THE MISS PATH ONLY, and a match is served - // by `pic.spill.hit` below — skipping the full cache's resolution and the - // polymorphic ways, neither of which can serve a spill key anyway - // (`pic_prime_get` refuses to cascade an encoded slot into a way). - let spill_stamp = ctx - .block() - .xor(I32, &packed_stamp, &PACKED_SPILL_FLIP.to_string()); - let is_spill = ctx.block().icmp_eq(I32, &pcid, &spill_stamp); - let ways_entry_idx = ctx.new_block("pic.token.ways"); - let ways_entry_label = ctx.block_label(ways_entry_idx); - let spill_hit_idx = ctx.new_block("pic.spill.hit"); - let spill_hit_label = ctx.block_label(spill_hit_idx); - ctx.block() - .cond_br(&is_spill, &spill_hit_label, &ways_entry_label); - - // S5: the SPILL hit. The flipped entry is a `(ShapeId, index)` fact like - // the inline one, and the ShapeId alone proves where the value is: the - // key list and the live inline-slot bound it names fix the key's - // position, a position at or past the bound IS its index in the spill - // buffer, and every carrier of the shape has that storage (the runtime - // reserves it for a key claimed without a value, keeps a stored - // `undefined` across buffer growth, and publishes no spill entry while - // spill storage is disabled — `spill_reserve_claimed`, - // `spill_get_present`, `packed_get::prime_get`). So the hit is two - // dependent loads to reach the buffer and one at the fixed index, with no - // null, bound or hole test: - // - // meta = [handle + META] ObjectHeader.meta - // spill = [meta + 32] ObjectMeta.spill - // value = [spill + 8 + index * 8] past the u32 length/capacity words - // - // Nothing here allocates or can collect, so the receiver needs no root. - ctx.current_block = spill_hit_idx; - crate::expr::receiver_range::emit_route_note( - ctx.block(), - crate::expr::receiver_range::Route::GenericSpillHit, - ); - let (val_spill, spill_end_label) = emit_spill_hit( - ctx, - fused_recv.as_ref(), - &entry_handle, - &packed_word, - &merge_label, - ); - - // Every way load still requires a resolved full cache. A site that has - // never primed has no cache, so there is nothing to compare against. - // - // That "never primed" edge is also exactly where an INHERITED read lives: - // a key on the prototype chain is never an own slot on the receiver's - // shape, so a site that only ever reads it never resolves its cache, and - // every read of it reaches this branch with `present` false. So that - // edge, and no other, asks the inherited-read cache (#10834/#10842) - // before calling out — see `pic.miss.inherited` below. Every other path - // to the exit (a small handle, a spill entry, an MRU or way miss at a site - // that HAS primed) is unchanged to the instruction; the first placement - // asked on all of them and cost every own-key miss the price of a - // declining probe (+88 on a megamorphic site, +89 on a spill read). - // - // Under `--typed-feedback` the edge keeps its old target: the recording - // blocks put a guard-fail and a fallback-call record on precisely this - // edge, and a read served without a call would have to change one of - // those records. Feedback builds are profiling builds; they keep their - // signal byte-identical and go without the hook. - ctx.current_block = ways_entry_idx; - let token_cache = crate::expr::emit_inline_cache_slot(ctx, &cache_name); - let inherited_idx = (!crate::expr::typed_feedback_emission_enabled()) - .then(|| ctx.new_block("pic.miss.inherited")); - let never_primed_label = inherited_idx - .map(|idx| ctx.block_label(idx)) - .unwrap_or_else(|| cold_label.clone()); - ctx.block() - .cond_br(&token_cache.present, &miss_label, &never_primed_label); - // `js_object_get_field_ic_miss` primes only slots below the descriptor's // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact // token hit permanently proves that the cached slot remains live and @@ -1042,7 +969,7 @@ pub(crate) fn lower_generic_property_get( } // A matched compact word is now, by construction, an INLINE slot: a // spill-located key publishes its ShapeId flipped by `PACKED_SPILL_FLIP` - // and is recognised in `pic.token.miss` instead. The overflow-bit test + // and is recognised by the miss front instead. The overflow-bit test // that used to stand between this shift and the load is gone from the hit // path — see the note there for what it cost and where it went. let slot = ctx.block().lshr(I64, &packed_word, "32"); @@ -1109,169 +1036,6 @@ pub(crate) fn lower_generic_property_get( } }; - // PIC miss on the MRU entry — before paying for the call, try the - // polymorphic ways (#7753). - // - // The slow entry is not a cheap fallback: it re-derives the receiver kind - // from scratch (proxy band, closure magic, registered-buffer and - // typed-array registries, small-handle dispatch), reads the - // accessors-in-use thread-local, then linear-scans the keys array with a - // `js_string_equals` per key. On a site whose receiver alternates between a - // handful of shapes — the shape of every discriminated-union dispatch — - // a single-entry cache misses on essentially every read and that whole - // ladder runs per field access. Measured on a tree-walking interpreter it - // was ~34% of run time. - // - // The ways are consulted only here, so a genuinely monomorphic site keeps - // the exact instruction sequence it had before this block existed. The - // typed-feedback counters are also recorded before the way compares, so a - // way hit still reports guard-fail + fallback-call exactly as it did when - // it was a real miss — the feedback heuristics see an unchanged signal - // (the site IS polymorphic; only the cost of that changed). - // - // # Why this block is DOMINATED by `pic.token` (#7907) - // - // Its only predecessor is `pic.token.miss`, which is `pic.token`'s. The - // exact descriptor identity proves cached-slot bounds, so `token` is - // everything the way compares need, and the cache pointer arrives on one - // edge rather than through a phi. - // - // #7883 could not rely on that: it routed the two receiver-validation - // failures here as well, which left the values live on only some edges, so - // the block **re-derived them** — header and identity loads, the token - // select, and a safe-address select for small-handle receivers. - // That was correct, and it was justified as cold. It is not cold: on a site - // whose receiver rotates over more shapes than the MRU entry holds — the - // shape #7753's ways exist for — this block runs on nearly every read, so - // the duplicate ladder sat on the hot path. Measured on `interp.ts`'s - // `evalNode`, the single hottest instruction in the whole program was the - // redundant receiver reconstruction inside this block. - ctx.current_block = miss_idx; - let cache_ref = token_cache.cache.clone(); - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_guard_fail", - &[(I64, &feedback_site_id)], - ); - crate::expr::emit_typed_feedback_record_call( - ctx.block(), - "js_typed_feedback_record_fallback_call", - &[(I64, &feedback_site_id)], - ); - - // Every way contains a ShapeId token. A non-zero receiver token keeps an - // empty way from matching; no GC-epoch guard is necessary because ids are - // never reused and descriptor identity survives key relocation. - // - // The compares sit behind their own branch on `cache[PIC_WAY_STATE] > 0` - // rather than being folded into one flat predicate, because a site whose - // receiver rotation is WIDER than the ways hold never hits one and would - // otherwise pay four dependent loads on every read: measured at **+37%** on - // a 7-shape site, against a 2.5x speedup on a 5-shape one. `pic_prime_get` - // latches that state to `-1` once a site proves itself megamorphic, and a - // fresh site reads `0`, so for both the branch is one load, - // one compare, and a perfectly predicted fall-through to the call — which - // is exactly the pre-#7753 code path. - let state_ptr = ctx - .block() - .gep(I64, &cache_ref, &[(I64, &PIC_WAY_STATE.to_string())]); - let way_state = ctx.block().load(I64, &state_ptr); - let ways_live = ctx.block().icmp_sgt(I64, &way_state, "0"); - let ways_idx = ctx.new_block("pic.ways"); - let ways_label = ctx.block_label(ways_idx); - ctx.block().cond_br(&ways_live, &ways_label, &call_label); - - ctx.current_block = ways_idx; - // `is_object` is not ANDed in any more: it is statically true on every edge - // that reaches here (#7907 — see the dominance note above). - // Reduced as a BALANCED TREE, not as a left fold. At most one way can hold - // a given token (`pic_prime_get` evicts a duplicate before it writes one, - // and pic_prime_get excludes zero-ShapeId tokens), so the association is - // free to change — but the fold made `way_slot` a chain of `PIC_WAYS` - // dependent `csel`s whose last node is the operand of the bounds compare - // that gates the branch out of this block. On `interp.ts` that node was the - // hottest instruction in `evalNode` (#7907). The tree halves the chain. - let mut lanes: Vec<(String, String)> = Vec::with_capacity(PIC_WAYS); - for w in 0..PIC_WAYS { - let tok_ptr = ctx.block().gep( - I64, - &cache_ref, - &[(I64, &(PIC_WAY_BASE + w * 2).to_string())], - ); - let way_tok = ctx.block().load(I64, &tok_ptr); - let eq = ctx.block().icmp_eq(I64, &way_tok, &token); - let slot_ptr = ctx.block().gep( - I64, - &cache_ref, - &[(I64, &(PIC_WAY_BASE + w * 2 + 1).to_string())], - ); - let way_slot_val = ctx.block().load(I64, &slot_ptr); - let lane_slot = ctx.block().select(I1, &eq, I64, &way_slot_val, "0"); - lanes.push((eq, lane_slot)); - } - while lanes.len() > 1 { - let mut merged: Vec<(String, String)> = Vec::with_capacity(lanes.len().div_ceil(2)); - for pair in lanes.chunks(2) { - match pair { - [(a_any, a_slot), (b_any, b_slot)] => { - let any = ctx.block().or(I1, a_any, b_any); - let slot = ctx.block().select(I1, a_any, I64, a_slot, b_slot); - merged.push((any, slot)); - } - [single] => merged.push(single.clone()), - _ => unreachable!("chunks(2) yields one or two elements"), - } - } - lanes = merged; - } - let (way_any, way_slot) = lanes - .pop() - .expect("PIC_WAYS is non-zero, so the reduction leaves exactly one lane"); - let way_load_idx = ctx.new_block("pic.way.load"); - let way_load_label = ctx.block_label(way_load_idx); - ctx.block().cond_br(&way_any, &way_load_label, &call_label); - - ctx.current_block = way_load_idx; - if fused_recv.is_some() { - crate::expr::receiver_range::emit_route_note( - ctx.block(), - crate::expr::receiver_range::Route::GenericWayHit, - ); - } - let way_offset = ctx.block().shl(I64, &way_slot, "3"); - let way_handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); - let way_base = ctx.block().add(I64, &way_handle, &obj_header_size); - let way_field_addr = ctx.block().add(I64, &way_base, &way_offset); - let way_field_ptr = ctx.block().inttoptr(I64, &way_field_addr); - let val_way = ctx.block().load(DOUBLE, &way_field_ptr); - // The loaded value is the answer here too, for the reason the shape-gated - // hit above needs no `TAG_HOLE` compare (#10826: a successful delete - // ALWAYS moves the receiver's ShapeId, so an exact-id match proves the - // slot it names is live). - // - // A way pair is not a second kind of cache entry needing its own - // argument. `pic_prime_get` is the ONLY writer of a way, and the only - // values it ever writes into one are `prev_tok`/`prev_slot` — the pair - // that was sitting in the MRU entry. Every `(token, slot)` a way holds is - // therefore an MRU pair that aged out; the token it is compared against is - // the same receiver ShapeId word the MRU compare reads; and ShapeIds are - // never reused. Whatever makes the MRU pair safe to load without a hole - // check makes the way pair safe — the entry did not become weaker by - // moving one word over. - // - // The two ways in which a way pair differs from an MRU pair both narrow - // it: an overflow-encoded slot is refused entry to a way at all, and a way - // is consulted only after the MRU entry has already missed. - let way_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); - - // #7907: receiver-validation failure. A receiver that gets here can never - // match a way — the compares require a real pointer to a plain - // descriptor-free `ObjectHeader` — so it goes straight to the handler, - // which reproduces the whole ladder anyway (proxy band, closure magic, - // buffer/typed-array registries, small-handle dispatch). The typed-feedback - // counters are the same two records on the same edges, so the feedback - // signal is byte-identical to what the pre-T1 blocks reported. if let Some(cold_idx) = cold_idx { ctx.current_block = cold_idx; crate::expr::emit_typed_feedback_record_call( @@ -1287,48 +1051,14 @@ pub(crate) fn lower_generic_property_get( ctx.block().br(&call_label); } - // The inherited-read hook, on the never-primed edge only (see the branch - // that reaches it, in `pic.token.ways`). A read whose key lives on the - // prototype chain can never take the own-slot hit — the receiver's shape - // says the key is not own — so before this block it paid the slow entry's - // prologue and dispatch (79 of an inherited read's 204 instructions, - // measured by the inherited-reads lane) just to reach the same lookup - // inside `get_field_ic_miss_impl`. `js_inherited_read_cache_hit_f64` is - // a pure state read — it allocates nothing, triggers no GC and runs no - // user code — so it is a leaf in `gc_call_effects.rs` and - // `root_reload.rs`: no spill, no reload around it. `TAG_HOLE` is its - // decline sentinel, which no ordinary value can be, so the answer is one - // compare, with the SERVED edge as the true edge like every guard-passing - // edge in this tower (#7883); a decline continues to the one exit exactly - // as the never-primed edge did before. Nothing is primed from here: - // priming stays in the miss handler, the one place that already knows - // the key is not own without a second search. The versioned-loop deopt - // note is emitted here as it is on the exit, so entering either cold arm - // still records the bailout. - let inherited_arm = inherited_idx.map(|idx| { - ctx.current_block = idx; - crate::expr::emit_versioned_loop_callback_deopt(ctx); - let inh_key_handle = emit_key_handle(ctx, &key_handle_global); - let handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); - let recv_ptr = ctx.block().inttoptr(I64, &handle); - let key_ptr = ctx.block().inttoptr(I64, &inh_key_handle); - let val_inherited = ctx.block().call( - DOUBLE, - "js_inherited_read_cache_hit_f64", - &[(PTR, &recv_ptr), (PTR, &key_ptr)], - ); - let inherited_bits = ctx.block().bitcast_double_to_i64(&val_inherited); - let inherited_served = - ctx.block() - .icmp_ne(I64, &inherited_bits, crate::nanbox::TAG_HOLE_I64); - let inherited_end_label = ctx.block().label.clone(); - ctx.block() - .cond_br(&inherited_served, &merge_label, &cold_label); - (val_inherited, inherited_end_label) - }); - - // The object exit: one call reproducing every pointer-path arm this tower - // used to expand. + // The collecting exit. It receives what the miss front declined (and, + // without a front, every miss) with the same four operands as before + // first-read D3: a never-primed site's inherited-read cache + // (#10834/#10842) is asked inside it, then the full miss body runs. + // Under `--typed-feedback` every miss records guard-fail + fallback-call + // on the way in (`cold` above), the signal those builds always saw for a + // non-hit. The versioned-loop deopt note is emitted here as well, so + // entering this cold arm still records the bailout. ctx.current_block = call_idx; crate::expr::emit_versioned_loop_callback_deopt(ctx); let miss_key_handle = emit_key_handle(ctx, &key_handle_global); @@ -1346,6 +1076,53 @@ pub(crate) fn lower_generic_property_get( let miss_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); + // The front (see `token_miss_label`). Its operands: the agent's + // shape-directory mirror (`PERRY_AGENT_PTRS` slot 0), so the front reads + // no thread-local; the receiver; the key exactly as the pool global holds + // it — STRING-tagged, the form a canonical key list stores, so the + // latched confirm compares one word; and the site's two cache words. A + // `length` site passes the runtime's empty directory + // (`PERRY_EMPTY_SHAPE_DIR`): an Array-subclass receiver serves `length` + // from its elements store, which no key list names, so its latched edge + // must not be confirmed from the shape. The call is a + // `"gc-leaf-function"` (the front is `Leaf` in the generated call-effects + // table): nothing live across it is spilled or relocated. + let front_arm = front_idx.map(|front_idx| { + ctx.current_block = front_idx; + let dir = if property == "length" { + EMPTY_SHAPE_DIR.to_string() + } else { + crate::expr::agent_ptr::emit_agent_ptr_or( + ctx, + crate::runtime_abi::AGENT_PTR_SHAPE_DIR, + EMPTY_SHAPE_DIR, + ) + }; + let front_handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); + let key_box = ctx.block().load(DOUBLE, &key_handle_global); + let key_bits = ctx.block().bitcast_double_to_i64(&key_box); + let answered = ctx.block().call( + DOUBLE, + "js_object_get_field_ic_front", + &[ + (PTR, &dir), + (I64, &front_handle), + (I64, &key_bits), + (PTR, &cache_slot_ref), + (PTR, &packed_ref), + ], + ); + let answered_bits = ctx.block().bitcast_double_to_i64(&answered); + let served = ctx + .block() + .icmp_ne(I64, &answered_bits, crate::nanbox::TAG_HOLE_I64); + let front_end_label = ctx.block().label.clone(); + // The SERVED edge is the true edge, like every guard-passing edge in + // the tower (#7883). + ctx.block().cond_br(&served, &merge_label, &call_label); + (answered, front_end_label) + }); + // Native Map/Set `.size`: their common leading field was admitted only by // the exact live GC-kind checks above. Keep the read inline; calling // `js_map_size` / `js_set_size` would reclassify the same receiver again. @@ -1377,12 +1154,11 @@ pub(crate) fn lower_generic_property_get( ctx.current_block = merge_idx; let mut incoming: Vec<(&str, &str)> = vec![ (&val_hit, &hit_end_label), - (&val_way, &way_end_label), (&val_miss, &miss_end_label), (&val_nonptr, &nonptr_end_label), ]; - if let Some((val_inherited, inherited_end_label)) = inherited_arm.as_ref() { - incoming.push((val_inherited, inherited_end_label)); + if let Some((answered, front_end_label)) = front_arm.as_ref() { + incoming.push((answered, front_end_label)); } if let Some((sso_val, sso_end_label)) = sso_arm.as_ref() { incoming.push((sso_val, sso_end_label)); @@ -1396,63 +1172,5 @@ pub(crate) fn lower_generic_property_get( if let Some((len, array_end_label)) = array_length_arm.as_ref() { incoming.push((len, array_end_label)); } - incoming.push((&val_spill, &spill_end_label)); Ok(ctx.block().phi(DOUBLE, &incoming)) } - -/// `pic.spill.hit`'s loads (see the note at its branch): the value at spill -/// index `packed_word >> 32` of the receiver's spill buffer. Returns the value -/// and the label of the block that branches to `merge_label`. -fn emit_spill_hit( - ctx: &mut FnCtx<'_>, - fused_recv: Option<&crate::expr::receiver_range::FusedReceiver>, - entry_handle: &str, - packed_word: &str, - merge_label: &str, -) -> (String, String) { - let ilp32 = crate::target_layout::target_is_ilp32(ctx.target_triple); - let meta_offset = crate::target_layout::object_meta_slot_offset_bytes(ctx.target_triple); - let meta_slot = match fused_recv { - // `handle + META`, addressed from the biased value (`receiver_range`). - Some(f) => { - crate::expr::receiver_range::emit_field_ptr(ctx.block(), &f.biased, meta_offset as i64) - } - None => { - let addr = ctx.block().add(I64, entry_handle, &meta_offset.to_string()); - ctx.block().inttoptr(I64, &addr) - } - }; - let meta = if ilp32 { - let narrow = ctx.block().load(I32, &meta_slot); - ctx.block().zext(I32, &narrow, I64) - } else { - ctx.block().load(I64, &meta_slot) - }; - let meta_ptr = ctx.block().inttoptr(I64, &meta); - let spill_slot = ctx.block().gep( - I8, - &meta_ptr, - &[( - I64, - &crate::target_layout::OBJECT_META_SPILL_OFFSET_BYTES.to_string(), - )], - ); - // `ObjectMeta.spill` is a `u64` on every target (the buffer address, - // zero-extended on ILP32). - let spill = ctx.block().load(I64, &spill_slot); - let spill_ptr = ctx.block().inttoptr(I64, &spill); - let index = ctx.block().lshr(I64, packed_word, "32"); - let elements = ctx.block().gep( - I8, - &spill_ptr, - &[( - I64, - &crate::target_layout::ARRAY_HEADER_SIZE_BYTES.to_string(), - )], - ); - let value_ptr = ctx.block().gep(DOUBLE, &elements, &[(I64, &index)]); - let value = ctx.block().load(DOUBLE, &value_ptr); - let end_label = ctx.block().label.clone(); - ctx.block().br(merge_label); - (value, end_label) -} diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index bf11d52055..40d5c061ee 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -202,6 +202,13 @@ fn no_call_location_without_debug_symbols() { /// #8067: the primary property-read PIC identity is the authoritative ShapeId /// only. Word 2 may carry the independent Array-subclass named-prefix proof, /// but it is consulted only after this exact ShapeId predicate fails. +/// +/// First-read D3: the hit is the receiver's `+4` word compared, as an `i32`, +/// with the compact word's low half — no discriminated token is formed at the +/// site at all. The polymorphic ways' `PIC_ID_TOKEN_BIT | ShapeId` tokens are +/// compared inside the miss front (`js_object_get_field_ic_front`), so the +/// token bit appearing in emitted IR again would mean a way compare crept back +/// inline. #[test] fn generic_property_get_hit_path_is_shape_id_only() { let ir = emit(false, None); @@ -209,9 +216,25 @@ fn generic_property_get_hit_path_is_shape_id_only() { ir.contains("@perry_ic_"), "test premise: the generic read reaches the inline monomorphic PIC:\n{ir}" ); + let token = ir + .find("\npic.token") + .unwrap_or_else(|| panic!("expected a pic.token block:\n{ir}")); + let token_body = &ir[token + ..ir[token + 1..] + .find("\n\n") + .map(|o| o + token + 1) + .unwrap_or(ir.len())]; + assert!( + token_body.contains("load i32") + && token_body.contains("trunc i64") + && token_body.contains("icmp eq i32"), + "the hit is the ShapeId word compared with the compact word's low \ + half:\n{token_body}" + ); assert!( - ir.contains("4611686018427387904"), - "hit path must form a discriminated ShapeId token:\n{ir}" + !ir.contains("4611686018427387904"), + "no discriminated way token may be formed at the site — the ways are \ + the miss front's:\n{ir}" ); assert!( !ir.contains("@PERRY_IC_EPOCH"), @@ -357,7 +380,13 @@ fn fs_promises_native_module_value_uses_submodule_singleton() { /// be the pre-#9708 shape coming back, with its 96 B of zero-fill per site. #[test] fn pic_cache_layout_matches_runtime() { - use crate::expr::property_get::generic_dispatch::{PIC_CACHE_WORDS, PIC_WAYS, PIC_WAY_BASE}; + use crate::expr::property_get::generic_dispatch::{ + PIC_CACHE_WORDS, PIC_WAYS, PIC_WAY_BASE, PIC_WAY_STATE, + }; + assert!( + PIC_WAY_STATE < PIC_WAY_BASE, + "the way-state word sits below the ways, as in perry-runtime" + ); assert_eq!( PIC_CACHE_WORDS, 12, "perry-runtime's PIC_CACHE_WORDS is 12; update both sides together" @@ -396,10 +425,22 @@ fn pic_cache_layout_matches_runtime() { fills on the first prime (#9708), got:\n{def}\n\nIR:\n{ir}" ); } + // The full cache is the runtime's to dereference: the site hands the slot's + // ADDRESS to the miss front and the slow entry, which test it for null + // (`read_confirm::tests::the_front_answers_a_way_and_declines_a_null_cache`, + // `ic_slow::tests::an_unresolved_cache_slot_is_never_dereferenced`). A + // site that loaded the slot itself would have to prove it non-null first. assert!( - ir.contains("load ptr, ptr @perry_ic_") && ir.contains("icmp ne ptr "), - "the full-cache fallback must prove the slot non-null before reading \ - a cache word:\n{ir}" + !ir.contains("load ptr, ptr @perry_ic_"), + "the site must not dereference the full-cache slot:\n{ir}" + ); + let front = ir + .lines() + .find(|l| l.contains(" = call double @js_object_get_field_ic_front(")) + .unwrap_or_else(|| panic!("expected the miss front call:\n{ir}")); + assert!( + front.contains("ptr @perry_ic_") && front.contains("_packed_get)"), + "the front must receive the cache slot and the compact word:\n{front}" ); } @@ -456,253 +497,244 @@ fn array_subclass_named_prefix_proof_is_reached_through_the_one_exit() { ); } -/// #7753: the polymorphic ways must be consulted BEFORE the miss call, and the -/// monomorphic path must not have grown any work. +/// The emitted function holding the generic tower, split into +/// `(label, trimmed body lines)` blocks. Register names restart in every +/// function, so every def/use question must be asked inside this one. +fn tower_blocks(ir: &str) -> Vec<(String, Vec)> { + let func = ir + .split("\ndefine ") + .find(|f| f.contains("\npic.miss.call")) + .unwrap_or_else(|| panic!("no function contains the generic tower:\n{ir}")); + let mut blocks: Vec<(String, Vec)> = Vec::new(); + for line in func.lines() { + if !line.starts_with(' ') && line.ends_with(':') { + blocks.push((line.trim_end_matches(':').to_string(), Vec::new())); + } else if let Some((_, body)) = blocks.last_mut() { + if !line.trim().is_empty() { + body.push(line.trim().to_string()); + } + } + } + blocks +} + +/// The one block whose label starts with `prefix`. +fn tower_block<'b>(blocks: &'b [(String, Vec)], prefix: &str) -> (&'b str, &'b [String]) { + let found: Vec<_> = blocks + .iter() + .filter(|(l, _)| l.starts_with(prefix)) + .collect(); + assert_eq!(found.len(), 1, "expected one `{prefix}` block: {blocks:?}"); + (found[0].0.as_str(), &found[0].1) +} + +/// `(cond, true target, false target)` of a block's `br i1` terminator. +fn tower_cond_br(body: &[String]) -> (String, String, String) { + let term = body.last().expect("a terminated block"); + let parts: Vec<&str> = term + .strip_prefix("br i1 ") + .unwrap_or_else(|| panic!("expected a conditional branch: {term}")) + .split(", ") + .collect(); + let label = |s: &str| s.trim_start_matches("label %").to_string(); + (parts[0].to_string(), label(parts[1]), label(parts[2])) +} + +/// #7753: the polymorphic ways must be consulted BEFORE the collecting miss +/// call, and the monomorphic path must not have grown any work. /// /// A one-entry cache misses on essentially every read at a site whose receiver /// alternates between shapes — the shape of every discriminated-union dispatch -/// — and each miss runs the full `js_object_get_field_ic_miss` ladder -/// (proxy/closure/buffer/typed-array probes, an accessors thread-local, then a -/// linear keys scan with a `js_string_equals` per key). If the way block is -/// ever deleted or floated below the call it stops paying for itself entirely, -/// and nothing else in the suite would show it — the program still computes the -/// right answer, just slowly. So assert the ORDER, not merely the presence. +/// — and each miss that reaches the collecting slow entry pays its statepoint +/// and the full miss ladder. If the ways are ever moved behind that call they +/// stop paying for themselves, and nothing else in the suite would show it — +/// the program still computes the right answer, just slowly. So assert the +/// ORDER, not merely the presence. +/// +/// First-read D3: the ways are asked by the GC-leaf miss front +/// (`js_object_get_field_ic_front`, ways first — pinned by +/// `read_confirm::tests::the_front_answers_a_way_and_declines_a_null_cache`), +/// so the order is a CFG fact here: the ShapeId compare's false edge is the +/// front, the front's SERVED edge is the merge, and the slow call is reached +/// from the front only on its decline edge. #[test] fn generic_property_get_tries_ways_before_calling_the_miss_handler() { let ir = emit(false, None); + let blocks = tower_blocks(&ir); + let (_, token) = tower_block(&blocks, "pic.token"); + let (_, on_hit, on_miss) = tower_cond_br(token); + assert!(on_hit.starts_with("pic.hit"), "{token:?}"); assert!( - ir.contains("@perry_ic_"), - "test premise: the generic read reaches the inline PIC:\n{ir}" + on_miss.starts_with("pic.miss.front"), + "the compare's miss edge must reach the front (the ways) first: {token:?}" ); - use crate::expr::property_get::generic_dispatch::{PIC_WAYS, PIC_WAY_BASE, PIC_WAY_STATE}; - - // Block *text* order is an artifact of emission order, so assert the CFG - // instead: the block that calls the miss handler must be reachable only as - // a branch target of the way block, never straight-line after it. - let ways = ir - .find("\npic.ways") - .unwrap_or_else(|| panic!("expected a pic.ways block:\n{ir}")); - let way_load = ir - .find("\npic.way.load") - .unwrap_or_else(|| panic!("expected a pic.way.load block:\n{ir}")); - let call_block = ir - .find("\npic.miss.call") - .unwrap_or_else(|| panic!("expected a pic.miss.call block:\n{ir}")); - let ways_body = &ir[ways..[way_load, call_block, ir.len()] - .into_iter() - .filter(|&x| x > ways) - .min() - .unwrap()]; + let (front_label, front) = tower_block(&blocks, "pic.miss.front"); assert!( - ways_body.contains("pic.way.load") && ways_body.contains("pic.miss.call"), - "pic.ways must end in a branch choosing between the way load and the \ - miss call — otherwise the compares are not gating anything:\n{ways_body}" + front + .iter() + .any(|l| l.contains("call double @js_object_get_field_ic_front(")), + "{front:?}" ); assert!( - !ways_body.contains("call double @js_object_get_field_ic"), - "the slow call must not sit inside the way block:\n{ways_body}" - ); - // The way compares read (token, slot) pairs at words PIC_WAY_BASE.. and the - // gate reads the state word — all inside pic.ways, none anywhere else. - for w in 0..PIC_WAYS { - for word in [PIC_WAY_BASE + w * 2, PIC_WAY_BASE + w * 2 + 1] { - assert!( - ways_body.contains(&format!("i64 {word}\n")), - "way word {word} is never read in the way block:\n{ways_body}" - ); - } - } + !front + .iter() + .any(|l| l.contains("@js_object_get_field_ic_slow(")), + "the slow call must not sit inside the front block: {front:?}" + ); + let (_, served, declined) = tower_cond_br(front); + assert!(served.starts_with("pget.recv_merge"), "{front:?}"); + assert!(declined.starts_with("pic.miss.call"), "{front:?}"); + // Every way of reaching the slow call from the object path goes through + // the front: its only other predecessor is the receiver-validation + // failure, which the front could not answer (no real object). + let (call_label, _) = tower_block(&blocks, "pic.miss.call"); + let preds: Vec<&str> = blocks + .iter() + .filter(|(_, body)| { + body.iter() + .any(|l| l.starts_with("br ") && l.contains(&format!("label %{call_label}"))) + }) + .map(|(l, _)| l.as_str()) + .collect(); assert!( - ir.contains(&format!("i64 {PIC_WAY_STATE}\n")), - "the megamorphic gate must read the way-state word:\n{ir}" + preds.contains(&front_label) + && preds + .iter() + .all(|p| *p == front_label || p.starts_with("pget.recv_")), + "the slow call is reached from the front's decline or a receiver \ + failure only: {preds:?}" ); } -/// #7907: `pic.miss` must be DOMINATED by `pic.token`, so the way compares can -/// use the values that block already computed instead of re-deriving them. +/// #7907: the miss path must be DOMINATED by `pic.token`, so it can use the +/// values that block already computed instead of re-deriving them. /// /// #7883 routed all four failure edges — small-handle receiver, non-object /// receiver, MRU token mismatch, cached slot out of bounds — into one block, -/// which left `token` / `token_nonnull` / `shape_id_eq` live on only some of them -/// and forced the block to reload the whole header ladder. That block is not -/// cold: on a receiver rotation wider than the MRU entry it runs on nearly -/// every read, so the duplicate ladder was hot code. The fix is purely -/// structural — send the two receiver-validation failures to `pic.miss.cold` -/// (they can never resolve a way, since `way_hit` requires a real object) and -/// the dominance follows. +/// which left the token values live on only some of them and forced the block +/// to reload the whole header ladder. That block is not cold: on a receiver +/// rotation wider than the MRU entry it runs on nearly every read, so the +/// duplicate ladder was hot code. The receiver-validation failures go to the +/// slow exit directly (they can never resolve a way: a way hit requires a real +/// object), and the dominance follows. /// -/// Assert the *consequences*, not the block names alone: a re-derivation would -/// show up as duplicate header loads or the small-handle sentinel `select`. +/// First-read D3: that block is `pic.miss.front`, the GC-leaf front call. It +/// must have exactly ONE predecessor, `pic.token`'s false edge, and the site +/// must not re-derive a receiver predicate for it: the front re-reads the +/// ShapeId word itself, so the hot load keeps a single use (the compare) and +/// isel folds it into `cmp %ecx, 4(%rdi)`. #[test] fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { let ir = emit(false, None); - let main_start = ir - .find("define i32 @main()") - .expect("entry module should define main"); - let main_rest = &ir[main_start..]; - let main_end = main_rest - .find("\n}\n") - .expect("main should have a closing brace"); - let main = &main_rest[..main_end]; - assert!( - main.contains("@perry_ic_"), - "test premise: the generic read reaches the inline PIC:\n{ir}" - ); - // T1: the landing block is now the single slow exit itself, and the - // dominance is structural — `pic.miss` has exactly ONE predecessor, - // `pic.token.miss`, which `pic.token` dominates. Assert that directly: - // routing any receiver-validation failure back into `pic.miss` would add a - // predecessor and immediately re-introduce the phis #7907 removed. - // `pic.miss` carries a numeric suffix and `pic.miss.call` starts with the - // same text, so match the block's own label exactly and then count the - // branches whose TARGET is that label (a `br i1` naming both blocks counts - // once, for the right one). - let miss_label = main - .lines() - .filter(|l| !l.starts_with(' ') && l.ends_with(':')) - .map(|l| l.trim_end_matches(':')) - .find(|l| { - l.strip_prefix("pic.miss.") - .is_some_and(|tail| tail.chars().all(|c| c.is_ascii_digit())) - }) - .unwrap_or_else(|| panic!("expected a pic.miss block:\n{ir}")) - .to_string(); - let preds = main - .lines() - .filter(|l| l.trim_start().starts_with("br ")) - .filter(|l| { - l.split("label %") - .skip(1) - .any(|t| t.trim_end_matches(&[',', ' '][..]) == miss_label) + let blocks = tower_blocks(&ir); + let (front_label, _) = tower_block(&blocks, "pic.miss.front"); + let (token_label, token) = tower_block(&blocks, "pic.token"); + let preds: Vec<&str> = blocks + .iter() + .filter(|(_, body)| { + body.iter().any(|l| { + l.starts_with("br ") + && l.split("label %") + .skip(1) + .any(|t| t.trim_end_matches(&[',', ' '][..]) == front_label) + }) }) - .count(); + .map(|(l, _)| l.as_str()) + .collect(); assert_eq!( - preds, 1, - "pic.miss must have exactly one predecessor (pic.token.miss), or it is \ - no longer dominated by pic.token:\n{ir}" - ); - assert!( - main.contains("label %pic.miss.call"), - "every receiver-validation failure must land on the single slow \ - exit:\n{ir}" + preds, + vec![token_label], + "the front must have exactly one predecessor (pic.token), or it is no \ + longer dominated by it: {blocks:?}" ); + let all: Vec<&String> = blocks.iter().flat_map(|(_, b)| b.iter()).collect(); assert!( - !main.contains("@PERRY_IC_EPOCH"), - "the removed keys-pointer epoch global must not appear:\n{ir}" + !all.iter().any(|l| l.contains("@PERRY_IC_EPOCH")), + "the removed keys-pointer epoch global must not appear" ); assert!( - !main.contains("ptrtoint ptr @perry_ic_"), - "the small-handle sentinel select only existed because an invalid \ - receiver could reach the way compares; it must be gone:\n{ir}" + !all.iter().any(|l| l.contains("ptrtoint ptr @perry_ic_")), + "the small-handle sentinel select must be gone" ); - // The receiver predicates, exactly once each. `icmp eq i32 %` is two: the - // ShapeId identity compare on the hit path and the spill compare in - // `pic.token.miss` that replaced the hit path's overflow-bit test. There - // is no GC-kind compare at all any more (#10828), so a single `icmp eq - // i8` would mean the header load has crept back somewhere. + // The receiver predicates, exactly once each: the ShapeId identity + // compare is the only `icmp eq i32`, and there is no GC-kind compare at + // all (#10828). for (needle, what, expect) in [ ("icmp eq i8 ", "the GC_TYPE_OBJECT compare", 0), - ("icmp eq i32 %", "the ShapeId identity compare", 2), + ("icmp eq i32 %", "the ShapeId identity compare", 1), ] { - let n = main.matches(needle).count(); + let n = all.iter().filter(|l| l.contains(needle)).count(); assert_eq!( n, expect, "{what} appears {n} times, expected {expect} — a receiver \ - predicate is being re-derived or has crept back:\n{ir}" + predicate is being re-derived or has crept back: {blocks:?}" ); } - // The ONE re-derivation that is deliberate: `pic.token.miss` re-reads the - // ShapeId word through an atomic load rather than reusing the hot load's - // value, so that the hot load has a single use and isel folds it into - // the compare (`cmp %ecx, 4(%rdi)`). A plain second load would be merged - // back into the first by GVN and the hot word would be live into the - // cold blocks again. - let token_miss = main - .find("\npic.token.miss") - .unwrap_or_else(|| panic!("expected a pic.token.miss block:\n{ir}")); - let token_miss_body = &main[token_miss - ..main[token_miss + 1..] - .find("\npic.") - .map(|o| o + token_miss + 1) - .unwrap_or(main.len())]; - assert!( - token_miss_body.contains("load atomic i32"), - "pic.token.miss must re-read the ShapeId word atomically so the hot \ - load stays single-use:\n{token_miss_body}" + let shape_word = token + .iter() + .find(|l| l.contains(" = load i32, ")) + .and_then(|l| l.split_once(" = ")) + .map(|(r, _)| r.to_string()) + .unwrap_or_else(|| panic!("the ShapeId word load: {token:?}")); + let uses = all + .iter() + .filter(|l| { + l.split(|c: char| c == ',' || c == ' ' || c == '(' || c == ')') + .any(|t| t == shape_word) + }) + .count(); + assert_eq!( + uses, 2, + "the hot ShapeId load must have exactly one use (the compare), so it \ + folds into it and stays dead on the miss edge: {blocks:?}" ); } -/// S5: a matched SPILL entry is served inline. `pic.token.miss` branches to -/// `pic.spill.hit` (not to the slow exit) on the un-flipped compare, and that -/// block is exactly the three dependent loads the ShapeId licenses — -/// `ObjectHeader.meta`, `ObjectMeta.spill`, the element at the word's index — -/// with no call, no compare and no hole test, straight to the merge. +/// S5: a matched SPILL entry is served without reaching the collecting call. +/// +/// First-read D3: the miss front recognises it — the compact word holding the +/// receiver's ShapeId flipped by `PACKED_SPILL_FLIP` — and answers with the +/// three dependent loads the ShapeId licenses (`ObjectHeader.meta`, +/// `ObjectMeta.spill`, the element at the word's index); its behaviour is +/// `read_confirm::tests::the_front_serves_a_spill_entry_only_for_a_real_shape_id`. +/// The CODEGEN half: the site hands the front the compact word (the spill +/// index and flipped id live there), the front's SERVED edge lands on the +/// merge without a second call, and no spill arithmetic is expanded inline. #[test] -fn a_spill_entry_is_served_inline_by_three_loads() { +fn a_spill_entry_is_served_by_the_leaf_front_before_the_slow_call() { + use crate::expr::property_get::generic_dispatch::PACKED_SPILL_FLIP; let ir = emit(false, None); - let main_start = ir - .find("define i32 @main()") - .expect("entry module should define main"); - let main_rest = &ir[main_start..]; - let main = &main_rest[..main_rest.find("\n}\n").expect("main closes")]; - // A block: its label line (by prefix, labels carry a numeric suffix) and - // every indented line after it. - let block = |prefix: &str| -> String { - let mut lines = main - .lines() - .skip_while(|l| !(l.starts_with(prefix) && l.ends_with(':'))); - let label = lines - .next() - .unwrap_or_else(|| panic!("expected a {prefix} block:\n{main}")); - let body: Vec<&str> = lines.take_while(|l| l.starts_with(' ')).collect(); - format!("{label}\n{}", body.join("\n")) - }; - let token_miss = block("pic.token.miss"); - let spill_label = main - .lines() - .filter(|l| !l.starts_with(' ') && l.ends_with(':')) - .map(|l| l.trim_end_matches(':')) - .find(|l| l.starts_with("pic.spill.hit")) - .unwrap_or_else(|| panic!("expected a pic.spill.hit block:\n{main}")) - .to_string(); + let blocks = tower_blocks(&ir); + let (front_label, front) = tower_block(&blocks, "pic.miss.front"); + let call = front + .iter() + .find(|l| l.contains("@js_object_get_field_ic_front(")) + .unwrap_or_else(|| panic!("the front call: {front:?}")); assert!( - token_miss.contains(&format!("label %{spill_label}")), - "the spill compare in pic.token.miss must branch to {spill_label}, not \ - to the slow exit:\n{token_miss}" + call.ends_with("_packed_get)"), + "the front must receive the compact word it decodes a spill entry \ + from: {call}" ); + let answer = call.split_once(" = ").map(|(r, _)| r).unwrap(); + let (_, merge) = tower_block(&blocks, "pget.recv_merge"); + let phi = merge.iter().find(|l| l.contains(" = phi double ")).unwrap(); assert!( - !token_miss.contains("label %pic.miss.call"), - "a matched spill entry must no longer call out:\n{token_miss}" + phi.contains(&format!("[ {answer}, %{front_label} ]")), + "the merge must take the front's answer straight from its block: {phi}" ); - let hit = block(&spill_label); - let loads: Vec<&str> = hit.lines().filter(|l| l.contains(" = load ")).collect(); - assert_eq!( - loads.len(), - 3, - "the spill hit is exactly meta, spill and the value:\n{hit}" - ); - assert!(loads[0].contains("load i64") && loads[1].contains("load i64")); - assert!(loads[2].contains("load double"), "{hit}"); - for forbidden in ["call ", "icmp", "select", "atomic"] { - assert!( - !hit.contains(forbidden), - "the ShapeId match is the whole proof: no `{forbidden}` on the spill \ - hit:\n{hit}" - ); + for (label, body) in &blocks { + for gone in [ + PACKED_SPILL_FLIP.to_string(), + "pic.spill".to_string(), + "xor i32 ".to_string(), + ] { + assert!( + !label.starts_with(gone.as_str()) && !body.iter().any(|l| l.contains(&gone)), + "no spill recognition may be expanded at the site, found \ + `{gone}` in {label}: {body:?}" + ); + } } - let meta = crate::target_layout::object_meta_slot_offset_bytes("x86_64-unknown-linux-gnu"); - let spill = crate::target_layout::OBJECT_META_SPILL_OFFSET_BYTES; - let elems = crate::target_layout::ARRAY_HEADER_SIZE_BYTES; - assert_eq!((meta, spill, elems), (8, 32, 8)); - assert!( - hit.contains(&format!("i64 {spill}")) && hit.contains(&format!("i64 {elems}")), - "the loads use the paired layout constants:\n{hit}" - ); - assert!( - hit.contains("lshr i64") && hit.contains(", 32"), - "the spill index is the compact word's high half:\n{hit}" - ); - assert!(hit.contains("br label %pget.recv_merge"), "{hit}"); } /// #8067: an exact ShapeId match proves the cached slot's descriptor facts, so @@ -713,7 +745,7 @@ fn cached_slot_bound_comes_from_the_shape_descriptor_match() { let floor = crate::target_layout::INLINE_SLOT_FLOOR_LIT; let ir = emit(false, None); assert!( - ir.contains("4611686018427387904") && ir.contains("@perry_ic_"), + ir.contains("_packed_get") && ir.contains("@perry_ic_"), "test premise: the emitted read uses a ShapeId PIC:\n{ir}" ); assert!( @@ -724,38 +756,32 @@ fn cached_slot_bound_comes_from_the_shape_descriptor_match() { ); } -/// #7907: the way `(token, slot)` reduction is a balanced tree, so the slot -/// select chain is `log2(PIC_WAYS)` deep instead of `PIC_WAYS` deep. Its last -/// node feeds the bounds compare that gates the branch out of `pic.ways`, so -/// the chain depth is directly on the critical path. +/// #7907: the way `(token, slot)` reduction must not sit on the critical path +/// of a way hit. It used to be a balanced select tree expanded per site, whose +/// last node fed the bounds compare gating the branch out of `pic.ways`. /// -/// At most one way can hold a given token — `pic_prime_get` evicts a duplicate -/// before writing one, and a zero token is excluded by `token_nonnull` — so -/// reassociating is value-preserving. +/// First-read D3: the ways are compared in the miss front, which returns on +/// the first matching way (at most one way holds a given token — +/// `pic_prime_get` evicts a duplicate before writing one, and an empty way's 0 +/// cannot match), so there is no reduction left at all. Pin that the site +/// expands none: no `pic.ways` block and no `select` anywhere in the tower. #[test] -fn way_slot_reduction_is_a_balanced_tree() { - use crate::expr::property_get::generic_dispatch::PIC_WAYS; +fn way_slot_reduction_is_not_expanded_per_site() { let ir = emit(false, None); - let ways = ir - .find("\npic.ways") - .unwrap_or_else(|| panic!("expected a pic.ways block:\n{ir}")); - // Block labels carry a numeric suffix (`pic.ways.16:`), so the search for - // the NEXT block has to start past this one's own label or it matches - // itself and slices an empty body — which reads as "the tree is missing". - let end = ir[ways + 1..] - .find("\npic.") - .map(|o| o + ways + 1) - .unwrap_or(ir.len()); - let body = &ir[ways..end]; - // A left fold emits PIC_WAYS selects whose 3rd operand is the previous - // select; the tree emits PIC_WAYS lane selects against the literal 0 plus - // PIC_WAYS-1 merges. Count the "select against 0" lanes: a fold has one. - let lanes = body.matches(", i64 0\n").count(); - assert_eq!( - lanes, PIC_WAYS, - "expected one `select … , i64 , i64 0` per way (a balanced tree); \ - a left fold produces exactly one:\n{body}" - ); + let blocks = tower_blocks(&ir); + for (label, body) in &blocks { + assert!( + !label.starts_with("pic.way"), + "no way block may be expanded per site: {label}" + ); + if label.starts_with("pic.") || label.starts_with("pget.") { + assert!( + !body.iter().any(|l| l.contains(" = select ")), + "no way reduction may be expanded per site, found a select in \ + {label}: {body:?}" + ); + } + } } /// #7189 — `B.ns` where the imported module says `export * as ns from "./m.ts"`. @@ -1148,22 +1174,40 @@ fn generic_property_get_slot_load_is_reached_only_through_every_guard() { "the hit block must end in the slot load and an unconditional branch \ to the merge:\n{hit_body}" ); - let way_body = blocks + // A way hit is answered in the miss front now (first-read D3), from the + // same proof: a way holds an aged MRU pair compared against the same + // ShapeId word. The front's answer is branched on only to tell a served + // value from its `TAG_HOLE` decline — the served edge is the TRUE edge + // and lands on the merge — never to re-test a served slot. + assert!( + !blocks.iter().any(|(l, _)| l.starts_with("pic.way")), + "no way block may be expanded per site:\n{func}" + ); + let front_body = blocks .iter() - .find(|(l, _)| l.starts_with("pic.way.load")) + .find(|(l, _)| l.starts_with("pic.miss.front")) .map(|(_, body)| body.join("\n")) - .expect("the way load block"); + .expect("the miss front block"); + let term = front_body + .lines() + .rev() + .find(|l| l.trim_start().starts_with("br ")) + .unwrap(); assert!( - !way_body.contains(crate::nanbox::TAG_HOLE_I64), - "the way path must not compare the loaded slot against TAG_HOLE — a \ - way holds an aged MRU pair and its token is the same ShapeId word, \ - so a way hit carries the same liveness proof as an MRU hit:\n\ - {way_body}" + front_body.contains(&format!(", {}", crate::nanbox::TAG_HOLE_I64)) + && term.contains("br i1 ") + && term.contains(", label %pget.recv_merge") + && term.contains("label %pic.miss.call"), + "the front's decline test must send the served value to the merge on \ + the TRUE edge and the decline to the slow call:\n{front_body}" + ); + let (served_at, declined_at) = ( + term.find("label %pget.recv_merge").unwrap(), + term.find("label %pic.miss.call").unwrap(), ); assert!( - way_body.contains("load double") && way_body.contains("br label %"), - "the way load block must end in the slot load and an unconditional \ - branch to the merge:\n{way_body}" + served_at < declined_at, + "served must be the TRUE edge: {term}" ); } @@ -1440,6 +1484,56 @@ fn no_gc_header_load_on_any_target() { } } +/// First-read D3: the miss front's directory operand (`PERRY_AGENT_PTRS` +/// slot 0) is read WITHOUT a call wherever the target has a call-free +/// thread-pointer path: an initial-exec load on an ELF executable, the TEB's +/// TLS array on Windows x86-64 (`agent_ptr::AgentPtrAccess::WindowsTeb`), the +/// pthread TSD on Apple aarch64. x86-64 Darwin keeps the `gc-leaf` accessor: +/// Mach-O has no call-free thread-local model there (`agent_ptr.rs`). +#[test] +fn the_front_reads_its_directory_without_a_call_where_the_target_allows() { + for (target, inline_form) in [ + ( + "x86_64-unknown-linux-gnu", + Some("getelementptr i8, ptr @PERRY_AGENT_PTRS, i64 0"), + ), + ( + "x86_64-pc-windows-msvc", + Some("load ptr, ptr addrspace(256) inttoptr (i64 88 to ptr addrspace(256))"), + ), + ("aarch64-apple-darwin", Some("mrs $0, tpidrro_el0")), + ("x86_64-apple-darwin", None), + ] { + let mut opts = ir_opts(false, None); + opts.target = Some(target.to_string()); + let ir = + String::from_utf8(compile_module(&module_with_nullish_read(), opts).unwrap()).unwrap(); + let func = ir + .split("\ndefine ") + .find(|f| f.contains("\npic.miss.front")) + .unwrap_or_else(|| panic!("{target}: no function contains the front:\n{ir}")); + let dir_call = func.contains("call ptr @perry_shape_dir_cell("); + match inline_form { + Some(form) => { + assert!( + func.contains(form) && !dir_call, + "{target}: the directory must be read inline (`{form}`), \ + with no accessor call:\n{func}" + ); + } + None => assert!(dir_call, "{target}: the accessor call:\n{func}"), + } + if target.contains("windows") { + for global in ["@_tls_index", "@PERRY_AGENT_PTRS_SECREL"] { + assert!( + func.contains(&format!("load i32, ptr {global}")), + "{target}: the TEB form reads {global}:\n{func}" + ); + } + } + } +} + #[test] fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { use crate::expr::property_get::generic_dispatch::PACKED_GET_EMPTY; @@ -1459,7 +1553,7 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { // `icmp ne i64 %packed, 0` beside it any more: the sentinel above makes // the ShapeId compare prove the site is primed as well. Named by the // packed word's register: a blanket "no `icmp ne i64`" would now also - // forbid the inherited-read hook's decline compare on the exit edge, + // forbid the miss front's `TAG_HOLE` decline compare, // which is a different question about a different value. let packed = ir .lines() @@ -1474,15 +1568,16 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { "the compact word must not be tested against zero:\n{ir}" ); assert!( - ir.contains("pic.token.miss"), - "a full-cache dereference must still guard a null site: {ir}" + !ir.contains("load ptr, ptr @perry_ic_"), + "the full cache stays lazy: the site never dereferences its slot (the \ + front and the slow entry null-test it): {ir}" ); } -/// T1: the whole point — per untyped `obj.prop` the emitted tower is TWO calls -/// and a handful of blocks, with the inline hit and the polymorphic ways kept. +/// T1: per untyped `obj.prop` the emitted tower is a bounded handful of blocks +/// and calls, with only the inline hit kept inline. /// -/// This is a ratchet, so it is an EXACT count in both dimensions. The tower it +/// This is a ratchet, so it is an EXACT count in both dimensions. The tower T1 /// replaced expanded 33 tower blocks and SIX runtime call sites per site /// (`js_object_get_field_by_name_f64` twice, the feedback-wrapped class-ref /// helper, `js_throw_type_error_property_access`, @@ -1492,58 +1587,20 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { /// sites; a single arm creeping back inline is a regression measured in /// megabytes of `.text`, and nothing else in the suite would report it. /// -/// Two and not one: a single shared exit let SimplifyCFG fold the receiver-tag -/// test and the small-handle test into one flat predicate, costing +4.00 -/// instructions on every HIT (measured, before those two tests became the one -/// fused compare). The separate non-pointer callee still keeps the `.length` -/// tower's chain branchy, so the count below is 2 — and a change that makes it -/// 1 is a hit-path regression, not a size win. -/// A SPILL-located key must still be RECOGNISED — just not on the hit path. +/// Two collecting exits and not one: a single shared exit let SimplifyCFG fold +/// the receiver-tag test and the small-handle test into one flat predicate, +/// costing +4.00 instructions on every HIT (measured, before those two tests +/// became the one fused compare). The separate non-pointer callee still keeps +/// the `.length` tower's chain branchy — a change that makes it one is a +/// hit-path regression, not a size win. /// -/// Taking the overflow-bit test off the hit path is only sound if the entry it -/// used to catch is caught somewhere else. `pic.token.miss` un-flips -/// `PACKED_SPILL_FLIP` and branches straight to `pic.spill.hit` (S5), skipping -/// the full cache's resolution and the ways (neither can hold an encoded -/// slot). Without this test, deleting the spill compare would leave every -/// spill read correct-but-slow — it would walk the ways, miss, call out, and -/// re-scan the keys array on every read, which is invisible in program -/// output. -#[test] -fn a_spill_entry_is_recognised_in_the_token_miss_block_and_nowhere_else() { - use crate::expr::property_get::generic_dispatch::PACKED_SPILL_FLIP; - let ir = emit(false, None); - let func = ir - .split("\ndefine ") - .find(|f| f.contains("\npic.token.miss")) - .unwrap_or_else(|| panic!("no function contains the generic tower:\n{ir}")); - - // Split the function into blocks and find `pic.token.miss`'s body. - let mut body: Vec<&str> = Vec::new(); - let mut inside = false; - for line in func.lines() { - if !line.starts_with(' ') && line.ends_with(':') { - inside = line.trim_end_matches(':').starts_with("pic.token.miss"); - continue; - } - if inside { - body.push(line); - } - } - let body = body.join("\n"); - assert!( - body.contains("xor i32 ") && body.contains(&PACKED_SPILL_FLIP.to_string()), - "`pic.token.miss` must un-flip PACKED_SPILL_FLIP to recognise a spill \ - entry:\n{body}" - ); - assert!( - body.contains("label %pic.spill.hit"), - "a recognised spill entry must branch straight to the inline spill \ - hit, not walk the ways or call out:\n{body}" - ); -} - +/// First-read D3: the spill entry, the ways, the inherited-read cache and the +/// latched confirm are no longer expanded per site. The ShapeId compare's +/// false edge makes ONE plain call to the GC-leaf front +/// (`js_object_get_field_ic_front`), whose `TAG_HOLE` decline continues to the +/// collecting slow call. #[test] -fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { +fn the_generic_tower_is_one_leaf_call_two_exits_and_a_bounded_number_of_blocks() { let ir = emit(false, None); let func = ir .split("\ndefine ") @@ -1552,8 +1609,8 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { // Every call/invoke in the whole function, by callee. Feedback records are // compile-time gated and absent from this build; anything else must be the - // one exit (the fixture's module init contributes its own calls, so match - // on the property-GET family rather than on a total). + // front or one of the two exits (the fixture's module init contributes its + // own calls, so match on the property-GET family rather than on a total). let pget_calls: Vec<&str> = func .lines() .filter(|l| l.contains(" call ") || l.contains(" invoke ")) @@ -1562,6 +1619,7 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { .filter(|c| { c.starts_with("js_object_get_field") || c.starts_with("js_typed_feedback_object_get_field") + || c.starts_with("js_inherited_read_cache") || *c == "js_throw_type_error_property_access" }) .collect(); @@ -1570,10 +1628,34 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { assert_eq!( sorted, vec![ + "js_object_get_field_ic_front", "js_object_get_field_ic_nonptr", "js_object_get_field_ic_slow" ], - "the tower must expand exactly two property-GET call sites:\n{func}" + "the tower must expand the front and exactly two collecting exits:\n{func}" + ); + // The front is nounwind: a plain call, never an invoke (its GC-leaf + // classification is `gc_call_effects`' test). + let fronts: Vec<&str> = func + .lines() + .filter(|l| l.contains("@js_object_get_field_ic_front(")) + .collect(); + assert_eq!(fronts.len(), 1, "one front call per tower:\n{func}"); + assert!( + fronts[0].contains(" = call double "), + "the front is nounwind, a plain call:\n{func}" + ); + // A non-`length` site confirms from this agent's own directory: the dir + // operand is slot 0 of `PERRY_AGENT_PTRS` (one initial-exec load in this + // ELF executable), never the empty directory a `length` site passes. + assert!( + !fronts[0].contains("@PERRY_EMPTY_SHAPE_DIR"), + "only a `length` site passes the empty directory:\n{}", + fronts[0] + ); + assert!( + func.contains("getelementptr i8, ptr @PERRY_AGENT_PTRS, i64 0"), + "the dir operand is PERRY_AGENT_PTRS slot 0:\n{func}" ); let blocks: Vec<&str> = func @@ -1590,41 +1672,25 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { // its split (cold): a POINTER-tagged small handle fails the fused // receiver test too and goes on to the object exit from here "pget.recv_nonptr", - // `pic.recv_hdr` is GONE: it existed to load the GC header word, and - // the ShapeId compare in `pic.token` now proves the kind (#10828) and - // the descriptor state (#10824) that word was loaded for. + // `pic.recv_hdr` is GONE: the ShapeId compare in `pic.token` proves + // the kind (#10828) and the descriptor state (#10824). "pic.token", - "pic.token.miss", - // `pic.token.miss` recognises a SPILL-located key by un-flipping - // PACKED_SPILL_FLIP and branches to `pic.spill.hit` (S5: three - // dependent loads, no call); everything else continues here to the - // full cache and the ways. - "pic.spill.hit", - // `pic.hit.inline` is GONE: with spill entries - // refused by the ShapeId compare itself, the hit block has nothing to - // decide between and the load sits directly in `pic.hit`. - "pic.token.ways", - // The hit block ends in the slot load and a branch to the merge: - // `pic.hit.deleted` is GONE with the `TAG_HOLE` compare (#10826 made - // delete a shape transition, so a ShapeId hit proves the slot live); - // `pic.hit.live` exists only when typed feedback has something to - // record on the live edge. + // The hit block is the slot load and a branch to the merge: no + // overflow-bit test (a spill entry is refused by the compare itself) + // and no `TAG_HOLE` compare (#10826 made delete a shape transition). "pic.hit", - // the polymorphic ways, deliberately still inline (#7753) - "pic.miss", - "pic.ways", - // `pic.way.live` is GONE with the way path's `TAG_HOLE` compare: the - // load block has nothing left to decide and branches to the merge. - "pic.way.load", - // the inherited-read hook, on the never-primed edge out of - // `pic.token.ways` and nowhere else (`js_inherited_read_cache_hit_f64`, - // a leaf); a decline continues to the one exit - "pic.miss.inherited", + // First-read D3: the compare's false edge. One GC-leaf call answers + // a way, a spill entry or a latched site's confirmed guess; its + // decline continues to the one exit. `pic.token.miss`, + // `pic.spill.hit`, `pic.token.ways`, `pic.miss`, `pic.ways`, + // `pic.way.load`, `pic.not_ways`, `pic.mega` and `pic.miss.inherited` + // are GONE into it and into the slow entry. + "pic.miss.front", // the one exit, and the join "pic.miss.call", "pget.recv_merge", ]; - // Labels carry a numeric suffix (`pic.ways.16`); strip it for comparison. + // Labels carry a numeric suffix (`pic.token.6`); strip it for comparison. let mut normalized: Vec = blocks .iter() .map(|b| { @@ -1644,198 +1710,123 @@ fn the_generic_tower_is_two_calls_and_a_bounded_number_of_blocks() { ); } +/// A SPILL-located key must still be RECOGNISED — just not on the hit path. +/// +/// Taking the overflow-bit test off the hit path is only sound if the entry it +/// used to catch is caught somewhere else. Without that, every spill read +/// would be correct-but-slow — it would miss, call out, and re-scan the keys +/// array on every read, which is invisible in program output. +/// +/// First-read D3: the miss front recognises it (un-flips `PACKED_SPILL_FLIP`, +/// checks the result is a real ShapeId, and loads the value — +/// `read_confirm::tests::the_front_serves_a_spill_entry_only_for_a_real_shape_id`), +/// so the site recognises it nowhere: the flip constant is not emitted, and the +/// only block the compare's false edge reaches is the front. +#[test] +fn a_spill_entry_is_recognised_by_the_front_and_nowhere_at_the_site() { + use crate::expr::property_get::generic_dispatch::PACKED_SPILL_FLIP; + let ir = emit(false, None); + let blocks = tower_blocks(&ir); + let flip = PACKED_SPILL_FLIP.to_string(); + let flip_i32 = (PACKED_SPILL_FLIP as i32).to_string(); + for (label, body) in &blocks { + assert!( + !body + .iter() + .any(|l| l.contains(&flip) || l.contains(&flip_i32)), + "PACKED_SPILL_FLIP must not be emitted at the site (`{label}`): {body:?}" + ); + } + let (_, token) = tower_block(&blocks, "pic.token"); + let (_, _, on_miss) = tower_cond_br(token); + assert!( + on_miss.starts_with("pic.miss.front"), + "the compare's false edge must reach the front, which recognises a \ + spill entry: {token:?}" + ); +} + /// The inherited-read cache (#10834/#10842) is asked on the NEVER-PRIMED edge /// and nowhere else. A read whose key lives on the prototype chain is never an /// own slot on the receiver's shape, so a site that only reads such a key never -/// resolves its per-site cache, and every read of it reaches `pic.token.ways` -/// with `present` false. That edge — which used to go straight to the exit — -/// now asks the cache before calling out. The first placement asked on EVERY -/// path into the exit and charged each own-key miss a declining probe (+88 on -/// a megamorphic site, +89 on a spill read, measured); this one costs every -/// other path zero instructions. +/// resolves its per-site cache. The first placement asked on EVERY path into +/// the exit and charged each own-key miss a declining probe (+88 on a +/// megamorphic site, +89 on a spill read, measured). /// -/// Five things are pinned, each of which would otherwise fail silently (the -/// program still computes the right value through the slow entry): +/// First-read D3: the probe moved into the slow entry +/// (`js_object_get_field_ic_slow`, which asks it only when the site's cache +/// slot is unresolved), behind the leaf front — so an own-key way, spill or +/// latched read never reaches it, and the site expands none of it. Pinned +/// here, each of which would otherwise fail silently (the program still +/// computes the right value through the slow entry): /// -/// 1. the hook call sits in `pic.miss.inherited` and in no other block, in -/// particular NOT on any path to the inline slot load (the CFG-walk test -/// asserts the same from the other side); -/// 2. that block is reached from `pic.token.ways` on the FALSE edge of the -/// cache-present test, and from nowhere else; -/// 3. its result is branched on with the SERVED edge as the true edge, the -/// tower's rule for every guard-passing edge, and the false edge is the -/// one exit; -/// 4. the slow entry is still called from `pic.miss.call` only, with the same -/// four operands; -/// 5. the merge phi takes the served value from `pic.miss.inherited`. +/// 1. no block of the site calls the hook — in particular none on a path to +/// the inline slot load (the CFG-walk test asserts the same from the other +/// side); +/// 2. the slow entry is called from `pic.miss.call` only, with the same four +/// operands (the never-primed test reads the cache slot); +/// 3. `pic.miss.call` is reached from the front only on its `TAG_HOLE` +/// decline, so a front-served read never pays the probe; +/// 4. the merge takes the slow entry's value from `pic.miss.call`. #[test] fn the_inherited_read_cache_is_asked_on_the_never_primed_edge_only() { let ir = emit(false, None); - let func = ir - .split("\ndefine ") - .find(|f| f.contains("\npic.miss.call")) - .unwrap_or_else(|| panic!("no function contains the generic tower:\n{ir}")); - let mut blocks: Vec<(String, Vec)> = Vec::new(); - let mut cur: Option<(String, Vec)> = None; - for line in func.lines() { - if !line.starts_with(' ') && line.ends_with(':') { - if let Some(b) = cur.take() { - blocks.push(b); - } - cur = Some((line.trim_end_matches(':').to_string(), Vec::new())); - continue; - } - if let Some((_, body)) = cur.as_mut() { - body.push(line.trim().to_string()); - } - } - if let Some(b) = cur.take() { - blocks.push(b); - } - // 1. one caller block, and it is the inherited arm. + let blocks = tower_blocks(&ir); + // 1. let holders: Vec<&str> = blocks .iter() .filter(|(_, body)| { body.iter() - .any(|l| l.contains("call double @js_inherited_read_cache_hit_f64(")) + .any(|l| l.contains("@js_inherited_read_cache_hit_f64(")) }) .map(|(l, _)| l.as_str()) .collect(); - assert_eq!( - holders.len(), - 1, - "the inherited hook must be called from exactly one block: {holders:?}\n{func}" - ); - let inh_label = holders[0]; assert!( - inh_label.starts_with("pic.miss.inherited"), - "the hook belongs on the never-primed edge, found it in `{inh_label}`:\n{func}" + holders.is_empty(), + "the inherited hook belongs to the slow entry, not the site: {holders:?}" ); - let (_, inh_body) = blocks.iter().find(|(l, _)| l == inh_label).unwrap(); - let hook_line = inh_body - .iter() - .find(|l| l.contains("@js_inherited_read_cache_hit_f64(")) - .unwrap(); - assert!( - hook_line.contains("(ptr %") && hook_line.matches(", ptr %").count() == 1, - "the hook takes the masked receiver and the interned key as two \ - pointers:\n{hook_line}" - ); - // 2. reached only from `pic.token.ways`, on the FALSE edge of `present`. - let preds: Vec<(&str, &str)> = blocks + // 2. + let slow_callers: Vec<(&str, &String)> = blocks .iter() .flat_map(|(l, body)| { body.iter() - .filter(|t| t.starts_with("br ") && t.contains(&format!("label %{inh_label}"))) - .map(move |t| (l.as_str(), t.as_str())) + .filter(|t| t.contains("@js_object_get_field_ic_slow(")) + .map(move |t| (l.as_str(), t)) }) .collect(); - assert_eq!( - preds.len(), - 1, - "exactly one edge may reach the hook: {preds:?}\n{func}" - ); - let (pred_label, pred_term) = preds[0]; - assert!( - pred_label.starts_with("pic.token.ways"), - "the hook's one predecessor must be the cache-present test: {pred_label}" - ); - let parts: Vec<&str> = pred_term - .strip_prefix("br i1 ") - .unwrap() - .split(", ") - .collect(); - assert!( - parts[1].starts_with("label %pic.miss") && !parts[1].starts_with("label %pic.miss.inh"), - "the TRUE edge of `present` must still be the way compares: {pred_term}" - ); - assert!( - parts[2].starts_with(&format!("label %{inh_label}")), - "the hook must sit on the FALSE (never-primed) edge: {pred_term}" - ); - let present_def = blocks - .iter() - .find(|(l, _)| l == pred_label) - .and_then(|(_, body)| { - body.iter() - .find(|l| l.starts_with(&format!("{} = ", parts[0]))) - }) - .unwrap_or_else(|| { - panic!( - "the branch condition {} must be defined in {pred_label}", - parts[0] - ) - }); - assert!( - present_def.contains("icmp ne ptr ") && present_def.ends_with(", null"), - "`present` is the cache slot's non-null test:\n{present_def}" - ); - // 3. polarity: `icmp ne , TAG_HOLE` is "served", served is the TRUE - // edge and lands on the merge; the false edge is the one exit. - let served = inh_body - .iter() - .find(|l| l.contains("icmp ne i64 ") && l.ends_with(crate::nanbox::TAG_HOLE_I64)) - .unwrap_or_else(|| panic!("the decline compare against TAG_HOLE:\n{func}")); - let cond = served.split_once(" = ").map(|(c, _)| c).unwrap(); - let term = inh_body - .iter() - .rev() - .find(|l| l.starts_with("br ")) - .unwrap(); - let parts: Vec<&str> = term - .strip_prefix("br i1 ") - .unwrap_or_else(|| panic!("the arm must branch on the hook's answer: {term}")) - .split(", ") - .collect(); - assert_eq!( - parts[0], cond, - "the branch must be on the served predicate: {term}" - ); - assert!( - parts[1].starts_with("label %pget.recv_merge"), - "the SERVED edge must be the true edge and land on the merge: {term}" - ); - assert!( - parts[2].starts_with("label %pic.miss.call"), - "the decline must be the false edge into the one exit: {term}" - ); - // 4. the slow entry: one caller, the exit, same operands. - let slow_callers: Vec<&str> = blocks - .iter() - .filter(|(_, body)| { - body.iter() - .any(|l| l.contains("@js_object_get_field_ic_slow(")) - }) - .map(|(l, _)| l.as_str()) - .collect(); assert_eq!(slow_callers.len(), 1, "{slow_callers:?}"); + let (call_label, slow_line) = slow_callers[0]; assert!( - slow_callers[0].starts_with("pic.miss.call"), + call_label.starts_with("pic.miss.call"), "the slow entry must be called from the one exit: {slow_callers:?}" ); - let (_, slow_body) = blocks.iter().find(|(l, _)| l == slow_callers[0]).unwrap(); - let slow_line = slow_body - .iter() - .find(|l| l.contains("@js_object_get_field_ic_slow(")) - .unwrap(); assert!( - slow_line.contains("ptr @perry_ic_") && slow_line.contains("_packed_get"), - "the slow entry must still receive the cache slot and the packed \ - word:\n{slow_line}" + slow_line.contains("(i64 %") + && slow_line.matches(", i64 %").count() == 1 + && slow_line.contains("ptr @perry_ic_") + && slow_line.contains("_packed_get"), + "the slow entry must still receive the receiver, the key, the cache \ + slot and the packed word:\n{slow_line}" ); - // 5. the merge takes the served value from the inherited arm. - let (_, merge_body) = blocks - .iter() - .find(|(l, _)| l.starts_with("pget.recv_merge")) - .unwrap(); - let phi = merge_body - .iter() - .find(|l| l.contains(" = phi double ")) - .unwrap(); - let served_value = hook_line.split_once(" = ").map(|(v, _)| v).unwrap(); + // 3. + let (_, front) = tower_block(&blocks, "pic.miss.front"); + let (cond, served, declined) = tower_cond_br(front); assert!( - phi.contains(&format!("[ {served_value}, %{inh_label} ]")), - "the merge must take the hook's value from `{inh_label}`:\n{phi}" + front + .iter() + .any(|l| l.starts_with(&format!("{cond} = icmp ne i64 ")) + && l.ends_with(crate::nanbox::TAG_HOLE_I64)), + "the front's branch must be on its TAG_HOLE decline: {front:?}" + ); + assert!(served.starts_with("pget.recv_merge"), "{front:?}"); + assert_eq!(declined, call_label, "{front:?}"); + // 4. + let (_, merge) = tower_block(&blocks, "pget.recv_merge"); + let phi = merge.iter().find(|l| l.contains(" = phi double ")).unwrap(); + let value = slow_line.split_once(" = ").map(|(v, _)| v).unwrap(); + assert!( + phi.contains(&format!("[ {value}, %{call_label} ]")), + "the merge must take the slow entry's value from `{call_label}`:\n{phi}" ); } diff --git a/crates/perry-codegen/src/expr/receiver_range.rs b/crates/perry-codegen/src/expr/receiver_range.rs index 47f6536907..84a7b5bd13 100644 --- a/crates/perry-codegen/src/expr/receiver_range.rs +++ b/crates/perry-codegen/src/expr/receiver_range.rs @@ -88,7 +88,10 @@ pub(crate) enum Route { Generic = 0, /// ...and was served by the compact MRU word. GenericMruHit = 1, - /// ...and was served by one of the polymorphic ways. + /// ...and was served by one of the polymorphic ways. No longer emitted: + /// the ways are read by the runtime's miss entry (first-read D3). The + /// number stays reserved: it indexes `RECV_ROUTE_NAMES`. + #[allow(dead_code)] GenericWayHit = 2, /// A read region's receiver test passed (R1 part 1). Region = 3, @@ -101,7 +104,9 @@ pub(crate) enum Route { /// The cached field-index early return's receiver test passed. CachedFieldIndex = 7, /// A generic read served from the receiver's SPILL buffer by the compact - /// word's flipped entry (S5, `pic.spill.hit`). + /// word's flipped entry (S5). No longer emitted: the runtime's miss entry + /// serves it (first-read D3). The number stays reserved, as above. + #[allow(dead_code)] GenericSpillHit = 8, // 9 and 10 are runtime-counted (`hot_diag::RT_ROUTE_*`). /// Step 4b (#10884): a loop/body region's guard ran (loop entry, or one diff --git a/crates/perry-codegen/src/expr/stack_guard.rs b/crates/perry-codegen/src/expr/stack_guard.rs index c8ac09be0a..855e065a3a 100644 --- a/crates/perry-codegen/src/expr/stack_guard.rs +++ b/crates/perry-codegen/src/expr/stack_guard.rs @@ -21,7 +21,7 @@ //! `llvm.frameaddress`, which would force a frame pointer and stop a //! frameless (shrink-wrapped) entry from staying frameless. -use super::agent_ptr::{agent_ptr_access, AgentPtrAccess, AGENT_PTRS_SYMBOL}; +use super::agent_ptr::{agent_ptr_access, AgentPtrAccess}; use super::FnCtx; use crate::types::{I64, I8, PTR}; @@ -65,11 +65,14 @@ fn emit_check(ctx: &mut FnCtx<'_>) { let ok_idx; let limit = match agent_ptr_access(ctx) { AgentPtrAccess::Call => return, - AgentPtrAccess::InitialExec => { + // The runtime publishes no stack limit on Windows (`stack_bounds` is + // `None` there), so the slot stays 0 and a check could never fire: + // emit none, as before the block was reachable inline on Windows. + AgentPtrAccess::WindowsTeb => return, + access @ AgentPtrAccess::InitialExec => { ok_idx = ctx.new_block("stack_guard.ok"); - let blk = ctx.block(); - let at = blk.gep(I8, &format!("@{AGENT_PTRS_SYMBOL}"), &[(I64, &slot_off)]); - blk.load(PTR, &at) + let at = super::agent_ptr::emit_slot_addr(ctx, access, &slot_off); + ctx.block().load(PTR, &at) } AgentPtrAccess::AppleTsd => { let lookup = super::hot_tls::emit_hot_tls_lookup(ctx, "stack_guard"); diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index 8e6a3a060c..460a5143ff 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -761,6 +761,25 @@ mod tests { } } + /// First-read D3: a generic read's miss front must be a proven GC leaf + /// (`read_confirm.rs` says why). If the generated table ever classifies + /// it otherwise, the front grew a collecting path — a design error in the + /// front, not a table update. Its decline continuation still collects. + #[test] + fn the_generic_read_miss_front_is_leaf_and_its_continuation_collects() { + assert_eq!( + runtime_class("js_object_get_field_ic_front"), + RuntimeClass::Leaf + ); + assert!(external_callee_cannot_collect( + "js_object_get_field_ic_front" + )); + assert!(external_callee_cannot_collect("perry_shape_dir_cell")); + assert!(!external_callee_cannot_collect( + "js_object_get_field_ic_slow" + )); + } + #[test] fn register_global_root_tracks_the_barrier_it_wraps() { assert_eq!( diff --git a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv index 914db12021..7f84742d39 100644 --- a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv @@ -2123,6 +2123,7 @@ js_object_get_field_f64 Leaf js_object_get_field_ic Reenters js_object_get_field_ic_fast Leaf js_object_get_field_ic_fast_miss Reenters +js_object_get_field_ic_front Leaf js_object_get_field_ic_miss Reenters js_object_get_field_ic_miss_packed Reenters js_object_get_field_ic_nonptr Reenters @@ -3953,6 +3954,7 @@ perry_resolve_static_plugin Reenters perry_runtime_widget_init Leaf perry_safe_area_insets_make Reenters perry_set_wake_callback Leaf +perry_shape_dir_cell Leaf perry_store_census_arm Leaf perry_string_header_abi_revision Leaf perry_stub_warn_ffi Leaf diff --git a/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv b/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv index 24ca620ba2..56567a8030 100644 --- a/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv +++ b/crates/perry-codegen/src/gc_effects/macos-aarch64.tsv @@ -2123,6 +2123,7 @@ js_object_get_field_f64 Leaf js_object_get_field_ic Reenters js_object_get_field_ic_fast Leaf js_object_get_field_ic_fast_miss Reenters +js_object_get_field_ic_front Leaf js_object_get_field_ic_miss Reenters js_object_get_field_ic_miss_packed Reenters js_object_get_field_ic_nonptr Reenters @@ -3948,6 +3949,7 @@ perry_resolve_static_plugin Leaf perry_runtime_widget_init Reenters perry_safe_area_insets_make Reenters perry_set_wake_callback Leaf +perry_shape_dir_cell Leaf perry_store_census_arm Leaf perry_string_header_abi_revision Leaf perry_stub_warn_ffi Leaf diff --git a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv index e611d78064..fd13150286 100644 --- a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv @@ -2123,6 +2123,7 @@ js_object_get_field_f64 Leaf js_object_get_field_ic Reenters js_object_get_field_ic_fast Leaf js_object_get_field_ic_fast_miss Reenters +js_object_get_field_ic_front Leaf js_object_get_field_ic_miss Reenters js_object_get_field_ic_miss_packed Reenters js_object_get_field_ic_nonptr Reenters @@ -3942,6 +3943,7 @@ perry_resolve_static_plugin Leaf perry_runtime_widget_init Leaf perry_safe_area_insets_make Reenters perry_set_wake_callback Leaf +perry_shape_dir_cell Leaf perry_store_census_arm Leaf perry_string_header_abi_revision Leaf perry_stub_warn_ffi Leaf diff --git a/crates/perry-codegen/src/module/linkage.rs b/crates/perry-codegen/src/module/linkage.rs index 118c04f4ed..dad9da83ee 100644 --- a/crates/perry-codegen/src/module/linkage.rs +++ b/crates/perry-codegen/src/module/linkage.rs @@ -326,6 +326,14 @@ pub(crate) fn helper_decl_attrs(name: &str) -> &'static str { // operands are i64 handles) orders it against every GC-capable call. // js_string_compare_value is NOT eligible: number coercion allocates. "js_string_compare" => " #3", + // First-read D3 (`object/field_get_set/ic_miss/read_confirm.rs`): a + // generic read site's miss front. Loads, compares and at most one + // store (D3b re-aims the site's compact word); no allocation, lock, + // throw, call or unbounded loop (the ways and the second-chance scan + // are bounded). NOT readonly: it writes the site word. Also a proven + // `Leaf` in the generated call-effects table, so the call is + // `"gc-leaf-function"`. + "js_object_get_field_ic_front" => " #4", // NOUNWIND+WILLRETURN only (#4, repsel Phase 4a.0) — each verified // (`typed_feedback.rs` / `array/header.rs`): no `js_throw` (longjmp) // anywhere in the body, every loop bounded by the 16M length/capacity diff --git a/crates/perry-codegen/src/root_reload.rs b/crates/perry-codegen/src/root_reload.rs index 49ea2345a5..fb8e18efa6 100644 --- a/crates/perry-codegen/src/root_reload.rs +++ b/crates/perry-codegen/src/root_reload.rs @@ -211,6 +211,8 @@ const NON_COLLECTING: &[&str] = &[ "js_inherited_read_cache_hit_f64", // S2 GC-leaf IC hits; proven `Leaf` by the generated call-effects table. "js_object_get_field_ic_fast", + // First-read D3: a generic read's miss front, proven `Leaf` likewise. + "js_object_get_field_ic_front", "js_class_field_get_ic_fast", "js_class_field_set_ic_fast", "js_put_value_set_packed_fast", diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index 0fa7b3c24f..23c063f5c9 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -85,6 +85,10 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // by the inline lookup in `expr::hot_tls` (Apple aarch64 targets only; // the declaration is unreferenced, and therefore inert, elsewhere). module.add_external_global("PERRY_HOT_TSD_KEY", I64); + // shapes_store — the never-written empty shape directory, a generic read + // site's front operand for `length` and where the agent's own directory + // is not readable inline (`property_get/generic_dispatch.rs`). + module.add_external_global("PERRY_EMPTY_SHAPE_DIR", I64); // #5525 follow-up: the process-global typed-array kind cache + the // "any exotic views live" guard, exported from perry-runtime so the codegen // can emit a guarded *inline* typed-array element load at the access site @@ -434,6 +438,18 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // Heap-pointer receiver: (masked obj_handle, key_handle, per-site IC cache // SLOT, per-site packed MRU word) -> field value. module.declare_function("js_object_get_field_ic_slow", DOUBLE, &[I64, I64, PTR, PTR]); + // First-read D3: a generic read site's ShapeId miss asks this GC leaf + // first (the agent's shape-directory mirror or null, receiver payload, + // the key as its pool global holds it, the site's cache slot and compact + // word); `TAG_HOLE` = declined, and the site calls the slow entry. + // `perry_shape_dir_cell` is the directory's accessor where emitted code + // cannot read the agent's pointer block inline. Both GC leaves. + module.declare_function( + "js_object_get_field_ic_front", + DOUBLE, + &[PTR, I64, I64, PTR, PTR], + ); + module.declare_function("perry_shape_dir_cell", PTR, &[]); // Object rest destructuring: copy all properties from src except excluded keys. // Takes a src object ptr and an array of NaN-boxed strings (the excluded keys), // returns a new object pointer. @@ -629,6 +645,11 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { &format!("[{} x ptr]", crate::expr::agent_ptr::AGENT_PTR_SLOTS), "initialexec", ); + // Windows x86-64 reaches the same block through the TEB (`agent_ptr.rs`, + // `WindowsTeb`): the image TLS index and the block offset in that image's + // TLS block. Declarations only; no other target references them. + module.add_external_global(crate::expr::agent_ptr::TLS_INDEX_SYMBOL, I32); + module.add_external_global(crate::expr::agent_ptr::AGENT_PTRS_SECREL_SYMBOL, I32); // #10812: the prologue stack check (`expr/stack_guard.rs`). module.declare_function("js_stack_overflow", VOID, &[]); module.declare_function( diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index 8614d19320..dcc2aced8e 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -2588,6 +2588,7 @@ js_object_get_field_f64 f64 ptr,i32u js_object_get_field_ic f64 i64,ptr,i64,ptr js_object_get_field_ic_fast f64 i64,ptr,i64,ptr js_object_get_field_ic_fast_miss f64 i64,ptr,i64,ptr +js_object_get_field_ic_front f64 ptr,i64,i64,ptr,ptr js_object_get_field_ic_miss f64 ptr,ptr,ptr js_object_get_field_ic_miss_packed f64 ptr,ptr,ptr,ptr js_object_get_field_ic_nonptr f64 i64,ptr,i64 @@ -4681,6 +4682,7 @@ perry_resolve_static_plugin f64 ptr perry_runtime_widget_init void perry_safe_area_insets_make f64 f64,f64,f64,f64 perry_set_wake_callback void ptr,ptr +perry_shape_dir_cell ptr perry_store_census_arm void perry_string_header_abi_revision i32u perry_stub_warn_ffi void ptr,ptr,ptr diff --git a/crates/perry-runtime/src/agent_ptrs.rs b/crates/perry-runtime/src/agent_ptrs.rs index 34f4b4a39c..98be52a105 100644 --- a/crates/perry-runtime/src/agent_ptrs.rs +++ b/crates/perry-runtime/src/agent_ptrs.rs @@ -14,7 +14,10 @@ //! `agent_ptrs` field holds this block's address — Mach-O has no //! initial-exec model, so a direct thread-local access would be a TLV thunk //! call; -//! * every other target, and any image that can be `dlopen`ed (a dylib or +//! * Windows x86-64: the native TLS sequence spelled out — `gs:[0x58]` +//! indexed by the image's `_tls_index`, plus [`PERRY_AGENT_PTRS`]'s offset +//! in the image's TLS block, published below as `PERRY_AGENT_PTRS_SECREL`; +//! * every other target, and any ELF image that can be `dlopen`ed (a dylib or //! staticlib output, where initial-exec TLS may not fit the static TLS //! block): the runtime accessor call. //! @@ -24,6 +27,8 @@ use std::cell::Cell; +/// Slot 0: the address of this agent's ordinary shape-directory mirror. +pub use crate::codegen_abi::AGENT_PTR_SHAPE_DIR; /// Slots in the block. **Must equal `AGENT_PTR_SLOTS` in /// `perry-codegen/src/expr/agent_ptr.rs`** (the emitted global's type). pub use crate::codegen_abi::AGENT_PTR_SLOTS; @@ -42,11 +47,38 @@ pub struct AgentPtrs([Cell<*const u8>; AGENT_PTR_SLOTS]); /// rustc emits a thread-local shim for the static under the same symbol name /// and a `#[no_mangle]` static fails to build ("symbol `PERRY_AGENT_PTRS` is /// already defined"). Only ELF executables name it -/// (`perry-codegen/src/expr/agent_ptr.rs`); Windows takes the accessor call. +/// (`perry-codegen/src/expr/agent_ptr.rs`); Windows x86-64 reaches it through +/// `PERRY_AGENT_PTRS_SECREL` instead (below). #[cfg_attr(not(windows), no_mangle)] #[thread_local] -pub static PERRY_AGENT_PTRS: AgentPtrs = - AgentPtrs([const { Cell::new(std::ptr::null()) }; AGENT_PTR_SLOTS]); +pub static PERRY_AGENT_PTRS: AgentPtrs = AgentPtrs({ + let mut slots = [const { Cell::new(std::ptr::null()) }; AGENT_PTR_SLOTS]; + // The shape-directory slot is never null: until this agent publishes its + // own mirror it names the shared empty directory, so the generic-read + // miss front reads it without a null test. + slots[AGENT_PTR_SHAPE_DIR] = + Cell::new(std::ptr::addr_of!(crate::object::shapes::PERRY_EMPTY_SHAPE_DIR) as *const u8); + slots +}); + +// Windows x86-64: `PERRY_AGENT_PTRS`'s section-relative offset in this image's +// TLS block (`.tls$`), as a 4-byte constant under a stable name. The static +// itself cannot be exported there (above), but `sym` names it under whatever +// symbol rustc gave it, and `.secrel32` is exactly the relocation rustc's own +// access to it uses (`mov _tls_index; mov gs:[0x58]; mov [..+idx*8]; +// sym@SECREL32`). Generated code performs that same sequence with this +// constant, so it reads the block without a call +// (`perry-codegen/src/expr/agent_ptr.rs`, `AgentPtrAccess::WindowsTeb`). +#[cfg(all(windows, target_arch = "x86_64"))] +core::arch::global_asm!( + ".section .rdata,\"dr\"", + ".globl PERRY_AGENT_PTRS_SECREL", + ".p2align 2", + "PERRY_AGENT_PTRS_SECREL:", + ".secrel32 {agent_ptrs}", + ".text", + agent_ptrs = sym PERRY_AGENT_PTRS, +); /// Publish (or clear, with null) one of this agent's pointers. #[allow(dead_code)] // no slot is published today (see the module doc) @@ -66,3 +98,19 @@ pub(crate) fn read(slot: usize) -> *const u8 { pub(crate) fn hot_addr() -> *mut u8 { &PERRY_AGENT_PTRS as *const AgentPtrs as *mut u8 } + +/// The address of this agent's ordinary shape-directory mirror +/// (`ShapeSlab::ordinary_dir_addr`), published into slot +/// [`AGENT_PTR_SHAPE_DIR`]. A generic read site passes it to its GC-leaf miss +/// front (`read_confirm::js_object_get_field_ic_front`) so the front reads no +/// thread-local; this is the accessor for targets where emitted code cannot +/// read the block inline. The slab also publishes the slot whenever it +/// publishes its directory, so the inline reads see it once any shape exists +/// on this agent. The address is stable for the thread's life, so the slot is +/// never cleared; the mirror it names is. A leaf: one TLS read and one store. +#[no_mangle] +pub extern "C" fn perry_shape_dir_cell() -> *const u8 { + let dir = crate::object::shapes::ordinary_dir_addr(); + publish(AGENT_PTR_SHAPE_DIR, dir); + dir +} diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index ef51375b2f..120791ca3d 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -347,6 +347,11 @@ pub use ic_slow::{js_object_get_field_ic_nonptr, js_object_get_field_ic_slow}; #[path = "field_get_set/ic_miss/outline_split.rs"] mod outline_split; pub use outline_split::{js_object_get_field_ic_fast, js_object_get_field_ic_fast_miss}; +/// First-read D3: the megamorphic read confirm, a GC-leaf stub a latched +/// generic read site calls before the slow entry. +#[path = "field_get_set/ic_miss/read_confirm.rs"] +mod read_confirm; +pub use read_confirm::js_object_get_field_ic_front; #[cfg(test)] mod buffer_ic_miss_tests { diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 78c7b67b99..e4f119976c 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -216,8 +216,8 @@ unsafe fn overflow_arm( super::ic_miss::get_field_ic_miss_impl(obj, key, cache_slot, std::ptr::null()) } -/// The exit for a receiver that IS a heap pointer — every failing guard on the -/// emitted site's object path lands here. +/// The exit for a receiver that IS a heap pointer — every failing guard and +/// every MRU miss on the emitted site's object path lands here (one call). /// /// * `obj_handle` — the receiver with the NaN-box tag already masked off. The /// caller has established the POINTER/STRING tag; this entry re-establishes @@ -238,62 +238,32 @@ pub extern "C-unwind" fn js_object_get_field_ic_slow( cache_slot: *mut PicCacheSlot, packed: *const AtomicU64, ) -> f64 { - // --- 0. a MEGAMORPHIC site's slot guess, confirmed by the receiver ------ - // - // A site whose way state is latched negative sees more shapes than any - // per-site entry can name, and every read that misses its compact word - // lands here. The site may still hold one thing: a slot GUESS (the compact - // word's high half — the slot the receiver's shape answered last, step 2b), - // which the RECEIVER'S own shape confirms or refutes - // (`shapes::confirm_slot_guess`: the position bound says key position - // `guess` is inline slot `guess`, and the key there IS this key, one - // pointer compare). Everything the guess cannot answer continues in - // `ic_slow_body` unchanged. Asked first, and only at a latched site, so a - // site that can still be primed is primed exactly as before. `length` is - // excluded as in step 2b: an Array-subclass receiver serves it from its - // elements store. Kept in this frameless entry, with the body out of line, - // so the confirmed read pays no prologue for the arms below. - if let Some(value) = unsafe { megamorphic_slot_guess(obj_handle, key, cache_slot, packed) } { - return value; - } - ic_slow_body(obj_handle, key, cache_slot, packed) -} - -/// Step 0 of [`js_object_get_field_ic_slow`]: the latched site's slot guess. -/// -/// # Safety -/// The entry's contract: a POINTER receiver handle, this site's cache slot -/// and packed word. -#[inline(always)] -unsafe fn megamorphic_slot_guess( - obj_handle: i64, - key: *const crate::StringHeader, - cache_slot: *mut PicCacheSlot, - packed: *const AtomicU64, -) -> Option { - let obj = obj_handle as usize as *const ObjectHeader; - if packed.is_null() - || key.is_null() - || !crate::value::addr_class::is_above_handle_band(obj as usize) + // First-read D3: the site asked its GC-leaf front + // (`read_confirm::js_object_get_field_ic_front`) first; what reaches this + // entry is what the front declined. A never-primed site asks the + // inherited-read cache (#10834/#10842) — the one edge an inherited read + // ever takes — and everything else runs the collecting body. + let addr = obj_handle as usize; + if !key.is_null() + && crate::value::addr_class::is_above_handle_band(addr) + // SAFETY: the site passes its own cache slot or null. + && unsafe { crate::object::pic_slot_peek(cache_slot) }.is_null() { - return None; - } - let cache = crate::object::pic_slot_peek(cache_slot); - if cache.is_null() - || (*cache)[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] >= 0 - || key_is_length(key) - { - return None; + // SAFETY: a POINTER-tagged payload above the handle band. + let v = unsafe { + crate::object::inherited_read_cache::js_inherited_read_cache_hit_f64( + addr as *const ObjectHeader, + key, + ) + }; + if v.to_bits() != crate::value::TAG_HOLE { + return v; + } } - let guess = ((*packed).load(Ordering::Relaxed) >> 32) as usize; - let value = crate::object::shapes::confirm_slot_guess(obj, key, guess)?; - #[cfg(test)] - crate::object::shapes::SHAPE_ANSWERED_READS.fetch_add(1, Ordering::Relaxed); - Some(value) + ic_slow_body(obj_handle, key, cache_slot, packed) } -/// Everything after step 0 of [`js_object_get_field_ic_slow`]. -#[inline(never)] +/// The body of [`js_object_get_field_ic_slow`]. fn ic_slow_body( obj_handle: i64, key: *const crate::StringHeader, @@ -434,7 +404,7 @@ fn ic_slow_body( /// `key` spells `length` — six bytes, compared directly (no UTF-8 validation). #[inline] -unsafe fn key_is_length(key: *const crate::StringHeader) -> bool { +pub(super) unsafe fn key_is_length(key: *const crate::StringHeader) -> bool { (*key).byte_len == 6 && std::slice::from_raw_parts(crate::string::string_data(key), 6) == b"length" } @@ -444,6 +414,10 @@ mod tests { use super::*; use crate::object::{PicCache, PIC_CACHE_WORDS}; + /// The read an emitted site performs on its miss edge: the confirm stub + /// first at a latched site, then the slow entry. + use super::super::read_confirm::test_site_miss_read as site_read; + /// The compact word an emitted site is born holding. const PACKED_GET_EMPTY_WORD: u64 = 0xFFFF_FFFF; @@ -503,7 +477,7 @@ mod tests { let packed = AtomicU64::new(0); let read = |o: &crate::gc::RuntimeHandle<'_>, slot: &mut PicCacheSlot| { o.with_mut_ptr(|p: *mut ObjectHeader| { - kind.with_const_ptr(|k| js_object_get_field_ic_slow(handle(p), k, slot, &packed)) + kind.with_const_ptr(|k| unsafe { site_read(handle(p), k, slot, &packed) }) }) }; // Drive the site until it latches. @@ -567,17 +541,13 @@ mod tests { for _ in 0..4 { for o in &objs { o.with_mut_ptr(|p: *mut ObjectHeader| { - kind.with_const_ptr(|k| { - js_object_get_field_ic_slow(handle(p), k, &mut slot, &packed) - }) + kind.with_const_ptr(|k| unsafe { site_read(handle(p), k, &mut slot, &packed) }) }); } } for o in &objs { let v = o.with_mut_ptr(|p: *mut ObjectHeader| { - absent.with_const_ptr(|k| { - js_object_get_field_ic_slow(handle(p), k, &mut slot, &packed) - }) + absent.with_const_ptr(|k| unsafe { site_read(handle(p), k, &mut slot, &packed) }) }); assert_eq!( v.to_bits(), @@ -642,7 +612,7 @@ mod tests { let packed = AtomicU64::new(0); let read = |o: &crate::gc::RuntimeHandle<'_>, slot: &mut PicCacheSlot| { o.with_mut_ptr(|p: *mut ObjectHeader| { - key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(p), k, slot, &packed)) + key.with_const_ptr(|k| unsafe { site_read(handle(p), k, slot, &packed) }) }) }; for round in 0..4 { @@ -874,9 +844,13 @@ mod tests { /// says key position `guess` is inline slot `guess` and holds exactly the /// key `site_key_bits` names, `None` for a decline. unsafe fn guess_walk(shape_id: u32, guess: u64, site_key_bits: u64) -> Option { - let key = (site_key_bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader; - crate::object::shapes::slot_guess_confirmed(shape_id, key, guess as usize) - .then_some(guess as usize) + crate::object::shapes::slot_guess_confirmed( + crate::object::shapes::ordinary_dir_addr(), + shape_id, + site_key_bits, + guess as usize, + ) + .then_some(guess as usize) } fn stamp_of(o: &crate::gc::RuntimeHandle<'_>) -> u32 { @@ -1470,7 +1444,7 @@ mod tests { let packed = AtomicU64::new(PACKED_GET_EMPTY_WORD); let read = |o: &crate::gc::RuntimeHandle<'_>, slot: &mut PicCacheSlot| { o.with_mut_ptr(|p: *mut ObjectHeader| { - key.with_const_ptr(|k| js_object_get_field_ic_slow(handle(p), k, slot, &packed)) + key.with_const_ptr(|k| unsafe { site_read(handle(p), k, slot, &packed) }) }) }; let want = |i: usize| { diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/read_confirm.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/read_confirm.rs new file mode 100644 index 0000000000..6ce271109e --- /dev/null +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/read_confirm.rs @@ -0,0 +1,523 @@ +//! First-read D3: everything a generic read site's miss can answer WITHOUT +//! collecting, in one GC-leaf call. +//! +//! Emitted code keeps exactly one thing inline for a read: the receiver's +//! ShapeId compared against the site's compact word, and the slot load. A +//! ShapeId miss makes ONE call to [`js_object_get_field_ic_front`], which +//! answers, in this order and only from shape facts: +//! +//! 1. a polymorphic way (#7753): `(ShapeId token, slot)` pairs in the site's +//! full cache; +//! 2. a SPILL entry: the compact word holds the receiver's ShapeId flipped by +//! `PACKED_SPILL_FLIP`; the ShapeId fixes the key's index in the spill +//! buffer (`packed_get::prime_get`, `spill_reserve_claimed`); +//! 3. a LATCHED megamorphic site (way state negative): its slot guess (the +//! compact word's high half), confirmed by the receiver's own shape — the +//! shape record's `POSBOUND` and its canonical key list compared with the +//! key atom — and, on a wrong guess, one bounded scan of that key list that +//! re-aims the guess (D3b). +//! +//! Anything else answers `TAG_HOLE`, and only then does the site branch to +//! its cold block and call the collecting `js_object_get_field_ic_slow` with +//! its usual operands (a never-primed site's inherited-read cache is asked +//! there). The front never allocates, collects, enters JS, throws, takes a +//! lock or makes a call — not even a thread-local access: the site passes the +//! agent's shape-directory mirror (`PERRY_AGENT_PTRS` slot 0, never null) as +//! an operand. +//! It is therefore `Leaf` in the generated call-effects table, the site's call +//! is a plain `"gc-leaf-function"` call, and nothing live across it is +//! spilled or relocated: statepoint spills exist only on the cold slow edge. + +use super::ic_miss::{PACKED_GET_EMPTY, PACKED_SPILL_FLIP, PIC_WAYS, PIC_WAY_BASE, PIC_WAY_STATE}; +use crate::object::shapes::{is_shape_id, positional_key_words, PIC_ID_TOKEN_BIT}; +use crate::object::{ObjectHeader, PicCacheSlot}; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// D3b's second chance scans at most this many key positions. +const SECOND_CHANCE_POSITIONS: usize = 32; + +#[inline(always)] +fn hole() -> f64 { + f64::from_bits(crate::value::TAG_HOLE) +} + +/// Inline slot `slot` of `obj`. +#[inline(always)] +unsafe fn inline_slot(obj: *const ObjectHeader, slot: usize) -> f64 { + *((obj as *const u8).add(std::mem::size_of::() + slot * 8) as *const f64) +} + +/// A generic read site's ShapeId miss (module docs): the answer, or +/// `TAG_HOLE` for the site's collecting slow call. +/// +/// * `dir` — this agent's shape-directory mirror +/// (`shapes::ordinary_dir_addr`, published in `PERRY_AGENT_PTRS` slot +/// `AGENT_PTR_SHAPE_DIR`), or `PERRY_EMPTY_SHAPE_DIR`, which confirms +/// nothing; never null. A `length` site passes the empty one on purpose: an +/// Array-subclass receiver serves `length` from its elements store, which no +/// key list names. +/// * `obj_handle` — the receiver's payload. The site calls only on the +/// ShapeId compare's false edge, which its small-handle test dominates, so +/// this is a real object pointer (its `+4` word was just loaded). +/// * `key_bits` — the site's key exactly as its pool global holds it: the +/// interned key, STRING-tagged, which is also how a canonical key list +/// stores it, so the confirm compares one word. +/// * `cache_slot`, `packed` — the site's full-cache slot (its global, never +/// null) and compact word. +/// +/// The ways are asked first (a polymorphic site's common miss), then the +/// spill entry, then a latched site's confirm: each answer is proven on its +/// own, so the order only decides who pays for which test. +/// +/// # Safety +/// The operands as a generic read site passes them (above). +#[no_mangle] +pub unsafe extern "C" fn js_object_get_field_ic_front( + dir: *const u8, + obj_handle: i64, + key_bits: u64, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, +) -> f64 { + let obj = obj_handle as usize as *const ObjectHeader; + let shape_id = (*obj).parent_class_id; + // `pic_slot_peek` without its null test: the slot is the site's global. + let cache = (*(cache_slot as *const std::sync::atomic::AtomicPtr)) + .load(Ordering::Acquire); + // A never-primed site (no cache) can still hold a spill entry in its + // compact word; what else can serve it (the inherited-read cache) is + // asked on the slow edge. + let state = if cache.is_null() { + 0 + } else { + (*cache)[PIC_WAY_STATE] + }; + if state > 0 { + // 1. The ways. A way token is `PIC_ID_TOKEN_BIT | ShapeId`; an empty + // way is 0 and cannot match. + let token = (shape_id as u64 | PIC_ID_TOKEN_BIT) as i64; + for w in 0..PIC_WAYS { + if (*cache)[PIC_WAY_BASE + 2 * w] == token { + return inline_slot(obj, (*cache)[PIC_WAY_BASE + 2 * w + 1] as usize); + } + } + } + let word = (*packed).load(Ordering::Relaxed); + // 2. Spill. Equality with a real ShapeId proves the receiver is an + // ordinary object of that shape (#10828 rule 3); the range test keeps an + // unflipped non-id word (a zeroed word flips to the synthetic-class floor) + // from matching an unstamped receiver. Nested, not `&&`: the common miss + // leaves on the first compare. + let spill_id = (word as u32) ^ PACKED_SPILL_FLIP; + if shape_id == spill_id { + if !is_shape_id(spill_id) { + return hole(); + } + let meta = (*obj).meta; + let spill = (*meta).spill as usize as *const u8; + let index = (word >> 32) as usize; + return *(spill.add(std::mem::size_of::() + index * 8) + as *const f64); + } + if state < 0 { + // 3. Latched. + return confirm_in(dir, obj, shape_id, packed, word, key_bits); + } + hole() +} + +/// The latched site's confirm (module docs, 3.), for tests that drive it +/// with an explicit directory. +#[cfg(test)] +pub(crate) fn read_confirm( + dir: *const u8, + obj_handle: i64, + shape_id: u32, + packed: *const AtomicU64, + key_bits: u64, +) -> f64 { + unsafe { + let word = (*packed).load(Ordering::Relaxed); + confirm_in( + dir, + obj_handle as usize as *const ObjectHeader, + shape_id, + packed, + word, + key_bits, + ) + } +} + +#[inline(always)] +unsafe fn confirm_in( + dir: *const u8, + obj: *const ObjectHeader, + shape_id: u32, + packed: *const AtomicU64, + word: u64, + key_bits: u64, +) -> f64 { + let Some((keys, bound)) = positional_key_words(dir, shape_id) else { + return hole(); + }; + let guess = (word >> 32) as usize; + // `guess < bound` proves POSBOUND nonzero before the keys are touched. + let slot = if guess < bound && *keys.words().add(guess) == key_bits { + guess + } else { + // D3b: the receiver's own key list, bounded; the found position is + // published as the new guess unless the word holds a stamp. + let scan = bound.min(SECOND_CHANCE_POSITIONS); + if scan == 0 { + return hole(); + } + let words = keys.words(); + let Some(found) = (0..scan).find(|&i| *words.add(i) == key_bits) else { + return hole(); + }; + if word as u32 == PACKED_GET_EMPTY as u32 { + (*packed).store( + ((found as u64) << 32) | (PACKED_GET_EMPTY as u32 as u64), + Ordering::Relaxed, + ); + } + found + }; + #[cfg(test)] + crate::object::shapes::SHAPE_ANSWERED_READS.fetch_add(1, Ordering::Relaxed); + inline_slot(obj, slot) +} + +/// A generic read site's miss as emitted code performs it: the leaf front +/// with this agent's directory, then the slow entry on `TAG_HOLE`. +#[cfg(test)] +pub(crate) unsafe fn test_site_miss_read( + obj_handle: i64, + key: *const crate::StringHeader, + cache_slot: *mut PicCacheSlot, + packed: *const AtomicU64, +) -> f64 { + let dir = if super::ic_slow::key_is_length(key) { + std::ptr::addr_of!(crate::object::shapes::PERRY_EMPTY_SHAPE_DIR) as *const u8 + } else { + crate::object::shapes::ordinary_dir_addr() + }; + let key_bits = key as usize as u64 | crate::value::STRING_TAG; + let v = js_object_get_field_ic_front(dir, obj_handle, key_bits, cache_slot, packed); + if v.to_bits() != crate::value::TAG_HOLE { + return v; + } + super::js_object_get_field_ic_slow(obj_handle, key, cache_slot, packed) +} + +#[cfg(test)] +mod tests { + use super::read_confirm as js_object_read_confirm; + use crate::gc::RuntimeHandle; + use crate::object::ObjectHeader; + use std::sync::atomic::{AtomicU64, Ordering}; + + fn atom(text: &[u8]) -> *mut crate::StringHeader { + let hash = crate::object::key_bytes_hash(text.as_ptr(), text.len()); + crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) + } + + /// The confirm's answer bits and the site word after the call. + fn confirm( + obj: &RuntimeHandle<'_>, + key: &RuntimeHandle<'_>, + word: u64, + dir: *const u8, + ) -> (u64, u64) { + let packed = AtomicU64::new(word); + let bits = obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k: *const crate::StringHeader| unsafe { + js_object_read_confirm( + dir, + o as i64, + (*o).parent_class_id, + &packed, + crate::value::js_nanbox_string(k as i64).to_bits(), + ) + .to_bits() + }) + }); + (bits, packed.load(Ordering::Relaxed)) + } + + fn guess(slot: u64) -> u64 { + (slot << 32) | 0xFFFF_FFFF + } + + /// The confirm answers exactly one thing: the receiver's own inline slot at + /// the guessed position, when the receiver's shape names the site's key + /// atom there. Every other input declines with `TAG_HOLE` — including a + /// key of the same TEXT that is not the atom (a pointer mismatch proves + /// nothing, so the slow entry decides), a guess past the position bound, + /// an id that names no ordinary record, and a null directory. + #[test] + fn the_confirm_answers_only_what_the_receivers_shape_names() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let a = scope.root_string_ptr(atom(b"d3_confirm_a")); + let b = scope.root_string_ptr(atom(b"d3_confirm_b")); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + for (k, v) in [(&a, 11.0), (&b, 22.0)] { + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + let dir = crate::object::shapes::ordinary_dir_addr(); + let hole = crate::value::TAG_HOLE; + let (v11, v22) = (11.0f64.to_bits(), 22.0f64.to_bits()); + // The guess, confirmed: answered, the word untouched. + assert_eq!(confirm(&obj, &a, guess(0), dir), (v11, guess(0)), "a at 0"); + assert_eq!(confirm(&obj, &b, guess(1), dir), (v22, guess(1)), "b at 1"); + // A wrong guess, or one past the position bound: the second chance + // finds the atom in the receiver's own key list, answers it, and + // re-aims the guess. + assert_eq!( + confirm(&obj, &a, guess(1), dir), + (v11, guess(0)), + "a is at 0" + ); + assert_eq!( + confirm(&obj, &b, guess(0), dir), + (v22, guess(1)), + "b is at 1" + ); + assert_eq!( + confirm(&obj, &a, guess(9), dir), + (v11, guess(0)), + "past the bound" + ); + // A word whose low half is a matchable stamp is never re-aimed. + let stamped = (1u64 << 32) | 0x8000_0001; + assert_eq!( + confirm(&obj, &a, stamped, dir), + (v11, stamped), + "stamp kept" + ); + // The empty directory confirms nothing: declined, the slow entry + // decides. + let empty = std::ptr::addr_of!(crate::object::shapes::PERRY_EMPTY_SHAPE_DIR) as *const u8; + assert_eq!( + confirm(&obj, &a, guess(0), empty), + (hole, guess(0)), + "empty dir" + ); + let copy_text = b"d3_confirm_a"; + let copy = scope.root_string_ptr(crate::string::js_string_from_bytes( + copy_text.as_ptr(), + copy_text.len() as u32, + )); + assert_ne!( + copy.with_const_ptr(|p: *const crate::StringHeader| p as usize), + a.with_const_ptr(|p: *const crate::StringHeader| p as usize), + "premise: the copy is another string object" + ); + assert_eq!( + confirm(&obj, ©, guess(0), dir), + (hole, guess(0)), + "same text, not the atom: declined, the slow entry decides" + ); + let unrecorded = a.with_const_ptr(|k: *const crate::StringHeader| { + obj.with_mut_ptr(|o: *mut ObjectHeader| { + js_object_read_confirm( + dir, + o as i64, + 7, + &AtomicU64::new(0xFFFF_FFFF), + crate::value::js_nanbox_string(k as i64).to_bits(), + ) + .to_bits() + }) + }); + assert_eq!(unrecorded, hole, "an id below the ShapeId band"); + } + + /// The front as a site calls it: `(answer bits, word after)`. + fn front( + obj: *mut ObjectHeader, + key_bits: u64, + cache_slot: *mut crate::object::PicCacheSlot, + word: u64, + ) -> (u64, u64) { + let packed = AtomicU64::new(word); + let dir = crate::object::shapes::ordinary_dir_addr(); + let bits = unsafe { + super::js_object_get_field_ic_front(dir, obj as i64, key_bits, cache_slot, &packed) + .to_bits() + }; + (bits, packed.load(Ordering::Relaxed)) + } + + /// #7753 in the front: a polymorphic site's way — `(PIC_ID_TOKEN_BIT | + /// ShapeId, slot)` in the full cache — is answered before anything else, + /// from the receiver's own inline slot; a way naming another shape, an + /// unprimed site (null cache: never dereferenced) and a primed site whose + /// ways hold nothing for this shape all decline to the slow call. + #[test] + fn the_front_answers_a_way_and_declines_a_null_cache() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let a = scope.root_string_ptr(atom(b"d3_front_way_a")); + let b = scope.root_string_ptr(atom(b"d3_front_way_b")); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + for (k, v) in [(&a, 11.0), (&b, 22.0)] { + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + let key_bits = b.with_const_ptr(|k: *const crate::StringHeader| { + crate::value::js_nanbox_string(k as i64).to_bits() + }); + let hole = crate::value::TAG_HOLE; + obj.with_mut_ptr(|o: *mut ObjectHeader| { + let shape_id = unsafe { (*o).parent_class_id }; + assert!(super::is_shape_id(shape_id), "premise: a shaped receiver"); + let mut cache: crate::object::PicCache = [0; crate::object::PIC_CACHE_WORDS]; + cache[super::PIC_WAY_STATE] = 1; + cache[super::PIC_WAY_BASE + 2] = (shape_id as u64 | super::PIC_ID_TOKEN_BIT) as i64; + cache[super::PIC_WAY_BASE + 3] = 1; + let mut slot: crate::object::PicCacheSlot = &mut cache; + assert_eq!( + front(o, key_bits, &mut slot, 0xFFFF_FFFF), + (22.0f64.to_bits(), 0xFFFF_FFFF), + "the way names this shape: its slot answers" + ); + cache[super::PIC_WAY_BASE + 2] = + ((shape_id + 1) as u64 | super::PIC_ID_TOKEN_BIT) as i64; + let mut slot: crate::object::PicCacheSlot = &mut cache; + assert_eq!( + front(o, key_bits, &mut slot, 0xFFFF_FFFF).0, + hole, + "a way for another shape" + ); + let mut null_slot: crate::object::PicCacheSlot = std::ptr::null_mut(); + assert_eq!( + front(o, key_bits, &mut null_slot, 0xFFFF_FFFF).0, + hole, + "never primed" + ); + }); + } + + /// S5 in the front: a SPILL entry — the compact word holding the + /// receiver's ShapeId flipped by `PACKED_SPILL_FLIP` — is served from the + /// spill buffer at the word's index; and the un-flipped id must be a REAL + /// ShapeId before it proves anything. A zeroed word un-flips to + /// `PACKED_SPILL_FLIP` itself, the synthetic-class floor, which an + /// unstamped receiver's `+4` word can hold: that receiver is not an + /// ordinary shaped object and its spill buffer answers nothing. + #[test] + fn the_front_serves_a_spill_entry_only_for_a_real_shape_id() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let names: [&[u8]; 4] = [b"d3_sp_a", b"d3_sp_b", b"d3_sp_c", b"d3_sp_d"]; + let keys: Vec<_> = names + .iter() + .map(|n| scope.root_string_ptr(atom(n))) + .collect(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 0)); + for (i, k) in keys.iter().enumerate() { + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| { + crate::object::js_object_set_field_by_name(o, kp, 10.0 + i as f64) + }) + }); + } + let last = &keys[3]; + let key_bits = last.with_const_ptr(|k: *const crate::StringHeader| { + crate::value::js_nanbox_string(k as i64).to_bits() + }); + let packed = AtomicU64::new(0xFFFF_FFFF); + let mut slot: crate::object::PicCacheSlot = std::ptr::null_mut(); + let primed = obj.with_mut_ptr(|o: *mut ObjectHeader| { + last.with_const_ptr(|k| { + super::super::js_object_get_field_ic_slow(o as i64, k, &mut slot, &packed) + }) + }); + assert_eq!(primed, 13.0, "the priming read"); + let word = packed.load(Ordering::Relaxed); + obj.with_mut_ptr(|o: *mut ObjectHeader| { + let shape_id = unsafe { (*o).parent_class_id }; + assert_eq!( + (word as u32) ^ super::PACKED_SPILL_FLIP, + shape_id, + "premise: the site published a spill entry (word {word:#x})" + ); + let mut none: crate::object::PicCacheSlot = std::ptr::null_mut(); + assert_eq!( + front(o, key_bits, &mut none, word), + (13.0f64.to_bits(), word), + "a spill entry for this shape is served by the front" + ); + // The same receiver under a `+4` word that is no ShapeId, read at a + // site whose word's low half is 0 (it un-flips to exactly that + // word) and whose index names the live spill value, so a front + // that skipped the id check would answer it. + assert!(!super::is_shape_id(super::PACKED_SPILL_FLIP), "premise"); + unsafe { (*o).parent_class_id = super::PACKED_SPILL_FLIP }; + let unstamped = front(o, key_bits, &mut none, word & !0xFFFF_FFFF).0; + unsafe { (*o).parent_class_id = shape_id }; + assert_eq!( + unstamped, + crate::value::TAG_HOLE, + "an un-flipped word that is no ShapeId proves nothing" + ); + }); + } + + /// S6: a `length` site is never confirmed from the receiver's shape — its + /// front is handed the EMPTY directory (codegen: + /// `array_length::a_length_read_serves_a_live_plain_array_off_the_shape_compare` + /// pins the emitted operand; `test_site_miss_read` mirrors it). An + /// Array-subclass receiver serves `length` from its elements store, so a + /// latched `length` site gains nothing from a key-list confirm; every such + /// read goes to the slow entry, which answers `length` for every receiver + /// kind. Pinned on the receiver where the confirm WOULD answer — a plain + /// object with an own `length` data key — by the shape-answered counter: + /// the read is right either way, so the value alone cannot see a `length` + /// site that started confirming. + #[test] + fn a_length_site_is_never_confirmed_from_the_shape() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let len = scope.root_string_ptr(atom(b"length")); + let other = scope.root_string_ptr(atom(b"d3_len_other")); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + for (k, v) in [(&len, 3.0), (&other, 4.0)] { + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + let answered = || crate::object::shapes::SHAPE_ANSWERED_READS.load(Ordering::Relaxed); + let mut cache: crate::object::PicCache = [0; crate::object::PIC_CACHE_WORDS]; + cache[super::PIC_WAY_STATE] = -1_000_000; + let mut read = |key: &RuntimeHandle<'_>, word: u64| { + let mut slot: crate::object::PicCacheSlot = &mut cache; + let packed = AtomicU64::new(word); + obj.with_mut_ptr(|o: *mut ObjectHeader| { + key.with_const_ptr(|k| unsafe { + super::test_site_miss_read(o as i64, k, &mut slot, &packed) + }) + }) + }; + // Premise: the same latched site confirms an ordinary key from the + // shape, so the counter can move. + let before = answered(); + assert_eq!(read(&other, guess(1)), 4.0); + assert_eq!(answered(), before + 1, "premise: a latched confirm counts"); + for word in [guess(0), guess(1), 0xFFFF_FFFF] { + let before = answered(); + assert_eq!(read(&len, word), 3.0, "a latched `length` read ({word:#x})"); + assert_eq!( + answered(), + before, + "a `length` read was confirmed from the shape ({word:#x})" + ); + } + } +} diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 9d05408e11..f6f5495fd6 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -52,6 +52,7 @@ pub(crate) use shapes_slot_list::{ shape_index_migrate_after_delete, shape_index_shift_in_place, try_update_stable_tombstone_shape, try_update_stable_tombstone_shape_cached, SlotIndex, }; +pub(crate) use shapes_store::PERRY_EMPTY_SHAPE_DIR; use shapes_store::{ IdList, ShapeRecord, ShapeSlab, RECORD_FLAG_BIRTH_OWNER, RECORD_FLAG_CACHE_CARRIER, RECORD_FLAG_CARRIED_SEEN, RECORD_FLAG_EXTERNAL_CARRIER, RECORD_FLAG_FACTS_INDEXED, @@ -415,8 +416,9 @@ pub(crate) fn test_position_bound_of(obj: *const crate::object::ObjectHeader) -> } /// Walk every present record of this agent's slab: `(records, positional, -/// ordinary records with an accessor key, disagreements)`, where a disagreement is a record whose stored positional -/// bit differs from [`ShapeRecord::positional_by_facts`]. +/// ordinary records with an accessor key, disagreements)`, where a +/// disagreement is a record whose stored POSBOUND differs from +/// [`ShapeRecord::position_bound_by_facts`]. #[cfg(test)] pub(crate) fn test_positional_census() -> (usize, usize, usize, Vec) { let table = &crate::state::state().shapes; @@ -427,7 +429,7 @@ pub(crate) fn test_positional_census() -> (usize, usize, usize, Vec) { return; } n += 1; - if r.positional_bit() { + if r.stored_position_bound() > 0 { positional += 1; } if r.object_kind().is_ordinary_layout() @@ -435,82 +437,107 @@ pub(crate) fn test_positional_census() -> (usize, usize, usize, Vec) { { accessor += 1; } - if r.positional_bit() != r.positional_by_facts() { + if r.stored_position_bound() != r.position_bound_by_facts() { bad.push(id); } }); (n, positional, accessor, bad) } -/// `(stored positional bit, its definition)` for shape `id`. +/// `(stored POSBOUND, its definition)` for shape `id`. #[cfg(test)] -pub(crate) fn test_positional_of_id(id: u32) -> Option<(bool, bool)> { +pub(crate) fn test_positional_of_id(id: u32) -> Option<(u32, u32)> { shape_record_by_id(id).map(|r| unsafe { let r = &*r.0.as_ptr(); - (r.positional_bit(), r.positional_by_facts()) + (r.stored_position_bound(), r.position_bound_by_facts()) }) } +/// The address of this thread's ordinary shape-directory mirror, which +/// [`positional_key_words`] reads through (`agent_ptrs` slot +/// `AGENT_PTR_SHAPE_DIR`). +#[inline] +pub(crate) fn ordinary_dir_addr() -> *const u8 { + ShapeSlab::ordinary_dir_addr() +} + /// The position bound of shape `id` (S3c), or `None` when it names no record. #[cfg(test)] pub(crate) fn test_position_bound_of_id(id: u32) -> Option { shape_record_by_id(id).map(|r| unsafe { (*r.0.as_ptr()).position_bound() }) } -/// The megamorphic read's slot-guess confirm: when `obj` carries an ordinary -/// ShapeId of this agent whose record says key position `guess` is inline slot -/// `guess` (`position_bound`), and the key AT that position is `key` itself -/// (one pointer compare: canonical lists hold their text's atom), the value in -/// the receiver's slot `guess`. `None` for anything else — a wrong or stale -/// guess, another text, a dictionary/class/descriptor/tombstoned shape, an id -/// that names no record — and the caller takes its ordinary path. +/// Shape `shape_id`'s positional key words, for the megamorphic read confirm +/// (`ic_miss::read_confirm::js_object_get_field_ic_front`): `(the keys +/// array, POSBOUND)` when the record answers by position — key position +/// `i < POSBOUND` IS inline slot `i` of every receiver carrying the shape — +/// and `None` otherwise (no ordinary record under that id in this agent, or +/// POSBOUND 0: a dictionary, class, descriptor/prototype-generation, +/// tombstoned or accessor-keyed shape). /// -/// The guess decides nothing: the receiver's own shape confirms it or it is -/// ignored. Allocation-free, no user code. +/// A canonical list holds its text's ATOM, boxed exactly as a site's pool +/// entry holds it, so the caller compares key WORDS: equality is identity, +/// and a mismatch proves nothing (a list written before its atom existed, an +/// SSO slot), which the caller answers by declining to the slow entry. +/// +/// `dir` is this thread's ordinary directory mirror ([`ordinary_dir_addr`]), +/// as the agent's pointer block holds it. +/// +/// Allocation-free, no user code, no formatting path: it is part of a +/// GC-leaf stub. /// /// # Safety -/// `obj` is a heap pointer above the handle band (its `+4` word is read); -/// `key` is a heap `StringHeader`. -#[inline] -pub(crate) unsafe fn confirm_slot_guess( - obj: *const crate::object::ObjectHeader, - key: *const crate::StringHeader, - guess: usize, -) -> Option { - if !slot_guess_confirmed((*obj).parent_class_id, key, guess) { - return None; +/// `dir` is this thread's [`ordinary_dir_addr`] or `PERRY_EMPTY_SHAPE_DIR` (never +/// null); any `shape_id`. +/// The words are valid until the next safepoint. +#[inline(always)] +pub(crate) unsafe fn positional_key_words( + dir: *const u8, + shape_id: u32, +) -> Option<(PositionalKeys, usize)> { + let r = ShapeSlab::ordinary_record_in(dir, shape_id)?; + Some(( + PositionalKeys(r.keys as usize as *const ArrayHeader), + r.position_bound_raw() as usize, + )) +} + +/// A record's canonical keys array, for [`positional_key_words`]: its words +/// are asked only once POSBOUND is known to be nonzero, so the front-offset +/// arithmetic runs only on the path that reads a key. +pub(crate) struct PositionalKeys(*const ArrayHeader); + +impl PositionalKeys { + /// The first logical key word. + /// + /// # Safety + /// Only when the record's POSBOUND is nonzero: then `keys` names a live + /// keys array (the collector marks through and rewrites `keys`) holding + /// at least POSBOUND logical keys. Logical element `i` sits past the + /// array's FRONT OFFSET, which a canonical list can carry without ever + /// being shifted (a size-class round-up alone makes the physical capacity + /// exceed the logical one: `keys_front_offset_tests`), so the accessor is + /// asked, never `+8`. + #[inline(always)] + pub(crate) unsafe fn words(&self) -> *const u64 { + crate::array::array_elements_ptr(self.0) as *const u64 } - Some( - *((obj as *const u8).add(std::mem::size_of::() + guess * 8) - as *const f64), - ) } -/// Does shape `shape_id` of this agent store `key` at inline slot `guess`, -/// by position? See [`confirm_slot_guess`]. +/// Does shape `shape_id` store the key whose NaN-boxed bits are `key_bits` +/// at inline slot `guess`, by position? See [`positional_key_words`]. /// /// # Safety -/// `key` is a heap `StringHeader`. -#[inline] +/// As [`positional_key_words`]. +#[cfg(test)] pub(crate) unsafe fn slot_guess_confirmed( + dir: *const u8, shape_id: u32, - key: *const crate::StringHeader, + key_bits: u64, guess: usize, ) -> bool { - let record = ShapeSlab::ordinary_record(shape_id); - if record.is_null() { - return false; - } - let r = &*record; - if guess >= r.position_bound() as usize { - return false; - } - // A bound > 0 means the record names a live keys array (the collector - // marks through and rewrites `keys`) holding at least `bound` logical - // keys; logical element `i` sits past the array's front offset. - let arr = r.keys as usize as *const ArrayHeader; - let slots = crate::array::array_elements_ptr(arr) as *const u64; - *slots.add(guess) == crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits() + positional_key_words(dir, shape_id) + .is_some_and(|(keys, bound)| guess < bound && *keys.words().add(guess) == key_bits) } /// Byte equality without a libc call for the short keys property names are. diff --git a/crates/perry-runtime/src/object/shapes_store.rs b/crates/perry-runtime/src/object/shapes_store.rs index 43991df124..de4ca0296e 100644 --- a/crates/perry-runtime/src/object/shapes_store.rs +++ b/crates/perry-runtime/src/object/shapes_store.rs @@ -73,9 +73,8 @@ pub(crate) struct ShapeRecord { /// Low 8 bits: the `RECORD_FLAG_*` set. Bits 8-10: the `ShapeObjectKind` /// discriminant (codes 0-6; the store facts F-A/F-B are kinds 5 and 6). /// Bits 11-14: the births a keyless birth shape served while tracking its - /// width (#10905). Bit 15: ANSWERABLE BY POSITION (see - /// [`ShapeRecord::position_bound`]), derived from the record's own facts. - /// Bits 16-23: the + /// width (#10905). Bit 15: reserved (it held the answerable-by-position + /// bit, which is now [`Self::position_bound`]'s zero). Bits 16-23: the /// attribute SUMMARY byte (`key_attrs::SUMMARY_*`), an identity fact. /// Bits 24-31: the inline width a keyless birth shape's descendants grow /// to (#10905). The two #10905 fields are learned facts of the record, @@ -86,6 +85,19 @@ pub(crate) struct ShapeRecord { /// 8-aligned (asserted below) and the slab geometry is unchanged — the /// kind field is free, it lives in padding that was already paid for. flags_and_kind: u32, + /// POSBOUND: how many leading key positions ARE inline slots of every + /// receiver carrying this shape — `min(logical_key_count, + /// live_inline_slot_count)` when the shape answers by position + /// ([`Self::positional_by_facts`]), 0 otherwise. A function of the + /// record's facts, rewritten by [`Self::refresh_positional`] wherever an + /// input changes, read by [`Self::position_bound`]. + /// + /// ONE field so the megamorphic read confirm (`js_object_read_confirm`) + /// answers "is the guess a position of this shape" with one compare — + /// `guess < position_bound` — instead of a flag test and a `min`. + /// Offset 40; `rep` follows at 48 (4 bytes of padding between), so the + /// record is 56 bytes. + position_bound: u32, /// Charter step 5: the per-slot field representation, two bits per inline /// slot 0..32 (`field_rep`). An identity fact under /// [`field_rep::identity`](crate::object::field_rep::identity), folded into the @@ -119,10 +131,6 @@ const RECORD_BIRTHS_MASK: u32 = 0xF << RECORD_BIRTHS_SHIFT; /// #10905 (`shapes_birth_width`): the learned descendant width, bits 24-31. const RECORD_WIDTH_SHIFT: u32 = 24; const RECORD_WIDTH_MASK: u32 = 0xFF << RECORD_WIDTH_SHIFT; -/// Bit 15: the shape answers by position ([`ShapeRecord::position_bound`]). -/// A function of the record's facts, rewritten by -/// [`ShapeRecord::refresh_positional`] wherever an input changes. -const RECORD_POSITIONAL: u32 = 1 << 15; // The fields of `flags_and_kind` are pairwise disjoint. const _: () = { let fields = [ @@ -131,7 +139,6 @@ const _: () = { RECORD_BIRTHS_MASK, RECORD_SUMMARY_MASK, RECORD_WIDTH_MASK, - RECORD_POSITIONAL, ]; let mut i = 0; while i < fields.len() { @@ -147,8 +154,10 @@ const _: () = assert!( super::shapes_birth_width::TRACKING_BIRTHS <= RECORD_BIRTHS_MASK >> RECORD_BIRTHS_SHIFT ); -const _: () = assert!(std::mem::size_of::() == 48); +const _: () = assert!(std::mem::size_of::() == 56); const _: () = assert!(std::mem::align_of::() == 8); +const _: () = assert!(std::mem::offset_of!(ShapeRecord, position_bound) == 40); +const _: () = assert!(std::mem::offset_of!(ShapeRecord, rep) == 48); impl ShapeRecord { const EMPTY: ShapeRecord = ShapeRecord { @@ -159,6 +168,7 @@ impl ShapeRecord { live_inline_slot_count: 0, hole_count: 0, flags_and_kind: 0, + position_bound: 0, rep: 0, }; @@ -277,6 +287,7 @@ impl ShapeRecord { live_inline_slot_count, hole_count, flags_and_kind: u32::from(flags) | kind_bits, + position_bound: 0, rep: 0, }; record.refresh_positional(); @@ -288,25 +299,32 @@ impl ShapeRecord { /// receiver carrying the shape. 0 when the shape cannot answer by position /// at all. /// - /// Whether it can is a FACT OF THE RECORD, stored in bit 15 - /// (`RECORD_POSITIONAL`) and read here with one load: the megamorphic - /// read asks it on every latched miss. [`Self::positional_by_facts`] is - /// its definition; every write of one of its inputs is followed by - /// [`Self::refresh_positional`] (construction, `with_summary`, slab - /// insert, the in-place stable-tombstone update), and debug builds assert - /// the stored bit against the definition on every read. - /// - /// The bound is `min(logical_key_count, live_inline_slot_count)`: a key - /// past the key count is another list's (canonical backings are shared by - /// a growth chain), and one past the live inline count is spilled. + /// It is a FACT OF THE RECORD, stored in the `position_bound` field + /// (POSBOUND) and read here with one load: the megamorphic read confirm + /// compares a site's slot guess against it on every latched read. + /// [`Self::position_bound_by_facts`] is its definition; every write of one + /// of its inputs is followed by [`Self::refresh_positional`] + /// (construction, `with_summary`, slab insert, the in-place + /// stable-tombstone update), and debug builds assert the stored bound + /// against the definition on every read. #[inline] pub(crate) fn position_bound(&self) -> u32 { debug_assert_eq!( - self.flags_and_kind & RECORD_POSITIONAL != 0, - self.positional_by_facts(), - "the positional bit disagrees with the record's facts: {self:?}" + self.position_bound, + self.position_bound_by_facts(), + "the position bound disagrees with the record's facts: {self:?}" ); - if self.flags_and_kind & RECORD_POSITIONAL == 0 { + self.position_bound + } + + /// The definition of POSBOUND: `min(logical_key_count, + /// live_inline_slot_count)` for a shape that answers by position, else 0. + /// A key past the key count is another list's (canonical backings are + /// shared by a growth chain), and one past the live inline count is + /// spilled. + #[inline] + pub(super) fn position_bound_by_facts(&self) -> u32 { + if !self.positional_by_facts() { return 0; } self.logical_key_count.min(self.live_inline_slot_count) @@ -324,6 +342,15 @@ impl ShapeRecord { /// * no ACCESSOR key in the attribute summary — an accessor key's slot /// holds its accessor pair, not a value; /// * a keys array at all. + /// + /// The field representation (`rep`) is deliberately NOT an input: an + /// `F64` slot holds a JS Number as raw IEEE bits outside the tag band, + /// which is itself a valid NaN-boxed value, so key position `i` is inline + /// slot `i` whatever the slot's representation. A shape minted with a + /// non-`Any` rep is its own record and gets its own bound from these + /// facts at construction and slab insert, like every other shape, and + /// deprecating a lane in place (`deprecate_rep_slot`) leaves the bound + /// as it is, correctly. #[inline] pub(super) fn positional_by_facts(&self) -> bool { self.object_kind().is_ordinary_layout() @@ -333,20 +360,25 @@ impl ShapeRecord { && self.summary() & crate::object::key_attrs::SUMMARY_ACCESSOR == 0 } - /// Rewrite the positional bit from the record's facts. + /// Rewrite POSBOUND from the record's facts. #[inline] pub(super) fn refresh_positional(&mut self) { - if self.positional_by_facts() { - self.flags_and_kind |= RECORD_POSITIONAL; - } else { - self.flags_and_kind &= !RECORD_POSITIONAL; - } + self.position_bound = self.position_bound_by_facts(); } - /// The stored positional bit, for the agreement test. + /// The stored POSBOUND without the debug agreement assert, for the + /// agreement test. #[cfg(test)] - pub(super) fn positional_bit(&self) -> bool { - self.flags_and_kind & RECORD_POSITIONAL != 0 + pub(super) fn stored_position_bound(&self) -> u32 { + self.position_bound + } + + /// The stored POSBOUND for the megamorphic read confirm, which must stay + /// a GC leaf with no formatting path: the agreement is asserted by + /// [`Self::position_bound`] everywhere else and by the census test. + #[inline(always)] + pub(crate) fn position_bound_raw(&self) -> u32 { + self.position_bound } /// The same record carrying field representation `rep` (`field_rep`). @@ -559,40 +591,166 @@ const PAGE_MASK: usize = PAGE_LEN - 1; /// the table interior mutability through a shared slab reference: the /// collector writes liveness bits and the `keys` word through raw record /// pointers while other code holds only copies (`ShapeDescriptor`). -type Chunk = Box<[UnsafeCell; CHUNK_LEN]>; +type ChunkCells = [UnsafeCell; CHUNK_LEN]; /// One directory page: `PAGE_LEN` chunk slots. -type Page = Box<[Option; PAGE_LEN]>; - -fn new_chunk() -> Chunk { - let mut v: Vec> = Vec::with_capacity(CHUNK_LEN); - v.resize_with(CHUNK_LEN, || UnsafeCell::new(ShapeRecord::EMPTY)); - // Exact length by construction; the conversion moves the allocation. - v.into_boxed_slice() - .try_into() - .unwrap_or_else(|_| unreachable!("chunk vector has CHUNK_LEN cells")) +type PageSlots = [Slot; PAGE_LEN]; + +/// A directory or page entry: an allocation this slab owns, or the SHARED +/// all-empty one of its level ([`EMPTY_CHUNK`], [`EMPTY_PAGE`]) — never null. +/// An absent run therefore reads exactly like a present run of absent +/// records (`ShapeRecord::EMPTY`: not present, position bound 0), so a +/// reader walks page → chunk → record with no null test at either level +/// (the megamorphic read confirm, `ordinary_record_in`). Nothing is ever +/// written through a shared empty: every writer asks [`Slot::is_shared`] +/// first and allocates. The slab frees what it owns ([`ShapeSlab::free_dir`]). +#[repr(transparent)] +struct Slot(std::ptr::NonNull); + +impl Clone for Slot { + fn clone(&self) -> Self { + *self + } +} +impl Copy for Slot {} + +/// A shared all-empty allocation. Never written (see [`Slot`]). +#[repr(transparent)] +pub struct SharedEmpty(T); +// SAFETY: nothing ever writes a shared empty; every reader only loads. +unsafe impl Sync for SharedEmpty {} + +static EMPTY_CHUNK: SharedEmpty = + SharedEmpty([const { UnsafeCell::new(ShapeRecord::EMPTY) }; CHUNK_LEN]); +static EMPTY_PAGE: SharedEmpty = SharedEmpty( + // SAFETY: the address of a static is never null. + [Slot(unsafe { + std::ptr::NonNull::new_unchecked(std::ptr::addr_of!(EMPTY_CHUNK.0) as *mut ChunkCells) + }); PAGE_LEN], +); + +trait Level: Sized + 'static { + fn shared() -> std::ptr::NonNull; + fn fresh() -> Box; } -fn new_page() -> Page { - let mut v: Vec> = Vec::with_capacity(PAGE_LEN); - v.resize_with(PAGE_LEN, || None); - v.into_boxed_slice() - .try_into() - .unwrap_or_else(|_| unreachable!("page vector has PAGE_LEN slots")) +impl Level for ChunkCells { + fn shared() -> std::ptr::NonNull { + std::ptr::NonNull::from(&EMPTY_CHUNK.0) + } + fn fresh() -> Box { + let mut v: Vec> = Vec::with_capacity(CHUNK_LEN); + v.resize_with(CHUNK_LEN, || UnsafeCell::new(ShapeRecord::EMPTY)); + // Exact length by construction; the conversion moves the allocation. + v.into_boxed_slice() + .try_into() + .unwrap_or_else(|_| unreachable!("chunk vector has CHUNK_LEN cells")) + } } +impl Level for PageSlots { + fn shared() -> std::ptr::NonNull { + std::ptr::NonNull::from(&EMPTY_PAGE.0) + } + fn fresh() -> Box { + let mut v: Vec> = Vec::with_capacity(PAGE_LEN); + v.resize_with(PAGE_LEN, Slot::empty); + v.into_boxed_slice() + .try_into() + .unwrap_or_else(|_| unreachable!("page vector has PAGE_LEN slots")) + } +} + +impl Slot { + #[inline] + fn empty() -> Self { + Slot(T::shared()) + } + #[inline] + fn is_shared(self) -> bool { + self.0 == T::shared() + } + /// The owned allocation, or `None` for the shared empty. + #[inline] + fn owned(&self) -> Option<&T> { + // SAFETY: an owned slot points at a live allocation of this slab. + (!self.is_shared()).then(|| unsafe { self.0.as_ref() }) + } + #[inline] + fn owned_mut(&mut self) -> Option<&mut T> { + // SAFETY: as `owned`; `&mut self` is the slab's exclusive borrow. + (!self.is_shared()).then(|| unsafe { self.0.as_mut() }) + } + /// The owned allocation, allocating it first if this is the shared empty. + #[inline] + fn owned_or_alloc(&mut self) -> &mut T { + if self.is_shared() { + self.0 = std::ptr::NonNull::from(Box::leak(T::fresh())); + } + // SAFETY: owned now. + unsafe { self.0.as_mut() } + } + /// Free an owned allocation (not its children) and become the shared + /// empty. + fn release(&mut self) { + if !self.is_shared() { + // SAFETY: allocated by `owned_or_alloc`, freed once: the slot is + // the shared empty afterwards. + drop(unsafe { Box::from_raw(self.0.as_ptr()) }); + self.0 = T::shared(); + } + } +} + +type Page = Slot; + +/// The ordinary directory mirror's type: `(page pointers, page count)`. +#[repr(C)] +pub(crate) struct OrdinaryDir { + pages: std::cell::Cell<*const Page>, + len: std::cell::Cell, +} + +impl OrdinaryDir { + const fn empty() -> Self { + OrdinaryDir { + pages: std::cell::Cell::new(std::ptr::null()), + len: std::cell::Cell::new(0), + } + } + #[inline(always)] + fn get(&self) -> (*const Page, usize) { + (self.pages.get(), self.len.get()) + } + #[inline] + fn set(&self, (pages, len): (*const Page, usize)) { + self.pages.set(pages); + self.len.set(len); + } +} + +/// A directory of no pages, never written: the value of the agent's +/// shape-directory pointer slot until the agent publishes its own mirror +/// (`agent_ptrs::PERRY_AGENT_PTRS`), and what a `length` read site passes +/// (`perry-codegen` `generic_dispatch.rs`). Every id indexes past its length, +/// so a reader never needs a null test for the directory itself. +#[no_mangle] +pub static PERRY_EMPTY_SHAPE_DIR: SharedEmpty = SharedEmpty(OrdinaryDir::empty()); + /// This thread's ordinary page directory as `(page pointers, page count)`: /// `ShapeSlab::pages`' element pointer and length, republished after every /// change to `pages` (`ShapeSlab::publish_dir`) and cleared before the slab /// is dropped. The megamorphic read's slot-guess confirm -/// ([`ShapeSlab::ordinary_record`]) reads it with one thread-local load -/// instead of resolving the runtime state and walking `record_ptr`. -/// `#[thread_local]` (const, no destructor) rather than `thread_local!`: a -/// late read during thread teardown sees the cleared pair, and the access -/// compiles to one thread-pointer-relative load. +/// ([`ShapeSlab::ordinary_record_in`]) reads it through its ADDRESS, which +/// emitted code passes from the agent's pointer block, instead of resolving +/// the runtime state and walking `record_ptr`: the confirm itself then +/// touches no thread-local (a runtime thread-local access is a +/// `__tls_get_addr` call on ELF and a TLV thunk call on Darwin, which would +/// give the stub a frame). `#[thread_local]` (const, no destructor) rather +/// than `thread_local!`: the address is stable for the thread's life, and a +/// late read during thread teardown sees the cleared pair. #[thread_local] -static ORDINARY_DIR: std::cell::Cell<(*const Option, usize)> = - std::cell::Cell::new((std::ptr::null(), 0)); +static ORDINARY_DIR: OrdinaryDir = OrdinaryDir::empty(); /// The by-id descriptor store. See the module docs. /// Two page directories: ordinary ShapeIds index from `SHAPE_ID_BASE`, and the @@ -607,14 +765,17 @@ impl Drop for ShapeSlab { if ORDINARY_DIR.get().0 == self.pages.as_ptr() { ORDINARY_DIR.set((std::ptr::null(), 0)); } + for band in [0u8, 1, 2] { + Self::free_dir(self.dir_mut(band)); + } } } pub(crate) struct ShapeSlab { - pages: Vec>, - dict_pages: Vec>, + pages: Vec, + dict_pages: Vec, /// The exotic-receiver band (`shapes::EXOTIC_SHAPE_ID_BASE`). - exotic_pages: Vec>, + exotic_pages: Vec, /// Present records. len: usize, } @@ -655,7 +816,7 @@ impl ShapeSlab { } #[inline] - fn dir(&self, band: u8) -> &Vec> { + fn dir(&self, band: u8) -> &Vec { match band { 0 => &self.pages, 1 => &self.dict_pages, @@ -664,7 +825,7 @@ impl ShapeSlab { } #[inline] - fn dir_mut(&mut self, band: u8) -> &mut Vec> { + fn dir_mut(&mut self, band: u8) -> &mut Vec { match band { 0 => &mut self.pages, 1 => &mut self.dict_pages, @@ -695,7 +856,7 @@ impl ShapeSlab { pub(super) fn record_ptr(&self, id: u32) -> Option<*mut ShapeRecord> { let (dict, index) = Self::index_of(id)?; let (page, chunk, slot) = Self::split(index); - let chunk = self.dir(dict).get(page)?.as_ref()?[chunk].as_ref()?; + let chunk = self.dir(dict).get(page)?.owned()?[chunk].owned()?; let cell = chunk[slot].get(); // SAFETY: the cell belongs to a live chunk owned by this slab; reads // and writes are serialized by the single-threaded agent discipline @@ -730,15 +891,15 @@ impl ShapeSlab { let (page, chunk, slot) = Self::split(index); let dir = self.dir_mut(band); if page >= dir.len() { - dir.resize_with(page + 1, || None); + dir.resize_with(page + 1, Slot::empty); // Only the ordinary band is mirrored (`ORDINARY_DIR`). if band == 0 { self.publish_dir(); } } let dir = self.dir_mut(band); - let page = dir[page].get_or_insert_with(new_page); - let chunk = page[chunk].get_or_insert_with(new_chunk); + let page = dir[page].owned_or_alloc(); + let chunk = page[chunk].owned_or_alloc(); let cell = chunk[slot].get_mut(); let previous = cell.present().then_some(*cell); // A retire-and-reinsert edits facts on a removed copy: the positional @@ -755,7 +916,7 @@ impl ShapeSlab { pub(super) fn remove(&mut self, id: u32) -> Option { let (dict, index) = Self::index_of(id)?; let (page, chunk, slot) = Self::split(index); - let chunk = self.dir_mut(dict).get_mut(page)?.as_mut()?[chunk].as_mut()?; + let chunk = self.dir_mut(dict).get_mut(page)?.owned_mut()?[chunk].owned_mut()?; let cell = chunk[slot].get_mut(); if !cell.present() { return None; @@ -771,11 +932,11 @@ impl ShapeSlab { pub(super) fn for_each(&self, mut f: impl FnMut(u32, *mut ShapeRecord)) { for dict in [0u8, 1, 2] { for (page_index, page) in self.dir(dict).iter().enumerate() { - let Some(page) = page else { + let Some(page) = page.owned() else { continue; }; for (chunk_index, chunk) in page.iter().enumerate() { - let Some(chunk) = chunk else { + let Some(chunk) = chunk.owned() else { continue; }; let base = ((page_index << PAGE_SHIFT) | chunk_index) << CHUNK_SHIFT; @@ -807,26 +968,26 @@ impl ShapeSlab { for dict in [0u8, 1, 2] { let dir = self.dir_mut(dict); for page in dir.iter_mut() { - let Some(chunks) = page.as_mut() else { + let Some(chunks) = page.owned_mut() else { continue; }; let mut live_chunks = 0usize; for chunk in chunks.iter_mut() { let empty = chunk - .as_ref() + .owned() .is_some_and(|c| c.iter().all(|cell| !unsafe { (*cell.get()).present() })); if empty { - *chunk = None; + chunk.release(); } - if chunk.is_some() { + if !chunk.is_shared() { live_chunks += 1; } } if live_chunks == 0 { - *page = None; + page.release(); } } - while dir.last().is_some_and(Option::is_none) { + while dir.last().is_some_and(|p| p.is_shared()) { dir.pop(); } dir.shrink_to_fit(); @@ -834,43 +995,64 @@ impl ShapeSlab { self.publish_dir(); } - /// Publish `pages` for [`Self::ordinary_record`] (see [`ORDINARY_DIR`]). + /// Publish `pages` for [`Self::ordinary_record_in`] (see [`ORDINARY_DIR`]). fn publish_dir(&self) { ORDINARY_DIR.set((self.pages.as_ptr(), self.pages.len())); + // Emitted read sites hand the mirror's address to the miss front + // from the agent's pointer block; publish it with the directory. + crate::agent_ptrs::publish( + crate::agent_ptrs::AGENT_PTR_SHAPE_DIR, + Self::ordinary_dir_addr(), + ); + } + + /// The address of THIS thread's [`ORDINARY_DIR`] mirror, as an opaque + /// pointer for [`Self::ordinary_record_in`]. Stable for the thread's + /// life (a const-initialised `#[thread_local]` with no destructor), so an + /// agent publishes it once into its `PERRY_AGENT_PTRS` slot + /// (`agent_ptrs::perry_shape_dir_cell`) and emitted code hands it to the + /// megamorphic read confirm, which then reads no thread-local at all. + #[inline] + pub(crate) fn ordinary_dir_addr() -> *const u8 { + &ORDINARY_DIR as *const OrdinaryDir as *const u8 } - /// The record of ordinary ShapeId `id` in THIS thread's slab, or null — - /// the fast twin of [`Self::record_ptr`] for the megamorphic read: one - /// thread-local load, two dependent directory loads, no `state()`. A - /// dictionary- or exotic-band id indexes past the ordinary directory's - /// length. The - /// record may be absent (`EMPTY`): its position bound is 0. + /// The record of ordinary ShapeId `id` in the slab whose [`ORDINARY_DIR`] + /// mirror is at `dir` (an [`Self::ordinary_dir_addr`] of this thread, or + /// `PERRY_EMPTY_SHAPE_DIR`), or `None` — the fast twin of [`Self::record_ptr`] for the + /// megamorphic read: two dependent directory loads, no `state()`, no + /// thread-local access. A dictionary- or exotic-band id indexes past the + /// ordinary directory's length. The record may be absent (`EMPTY`): its + /// position bound is 0. + /// + /// # Safety + /// `dir` is this thread's [`Self::ordinary_dir_addr`] or + /// `PERRY_EMPTY_SHAPE_DIR`; never null. #[inline(always)] - pub(super) fn ordinary_record(id: u32) -> *const ShapeRecord { - let (pages, len) = ORDINARY_DIR.get(); + pub(super) unsafe fn ordinary_record_in<'a>( + dir: *const u8, + id: u32, + ) -> Option<&'a ShapeRecord> { + let (pages, len) = (*(dir as *const OrdinaryDir)).get(); let index = id.wrapping_sub(SHAPE_ID_BASE) as usize; let (page, chunk, slot) = Self::split(index); if page >= len { - return std::ptr::null(); + return None; } // SAFETY: `pages` holds `len` entries of this thread's slab, current - // as of the last change to it; nothing here can change it. - unsafe { - let Some(page) = (*pages.add(page)).as_ref() else { - return std::ptr::null(); - }; - match page[chunk].as_ref() { - Some(chunk) => chunk[slot].get(), - None => std::ptr::null(), - } - } + // as of the last change to it; nothing here can change it. An absent + // page or chunk is the shared empty one (`Slot`), never null, so + // both levels are plain loads and the record is never null. + let page = (*pages.add(page)).0.as_ref(); + let chunk = page[chunk].0.as_ref(); + Some(&*chunk[slot].get()) } #[cfg(test)] pub(super) fn clear(&mut self) { - self.pages.clear(); - self.dict_pages.clear(); - self.exotic_pages.clear(); + for band in [0u8, 1, 2] { + Self::free_dir(self.dir_mut(band)); + } self.publish_dir(); self.len = 0; } @@ -885,14 +1067,14 @@ impl ShapeSlab { .iter() .chain(self.dict_pages.iter()) .chain(self.exotic_pages.iter()) - .flatten() + .filter_map(Slot::owned) { pages += 1; - chunks += page.iter().filter(|c| c.is_some()).count(); + chunks += page.iter().filter(|c| !c.is_shared()).count(); } (self.pages.capacity() + self.dict_pages.capacity() + self.exotic_pages.capacity()) - * std::mem::size_of::>() - + pages * PAGE_LEN * std::mem::size_of::>() + * std::mem::size_of::() + + pages * PAGE_LEN * std::mem::size_of::>() + chunks * CHUNK_LEN * std::mem::size_of::() } @@ -903,10 +1085,23 @@ impl ShapeSlab { .iter() .chain(self.dict_pages.iter()) .chain(self.exotic_pages.iter()) - .flatten() - .map(|page| page.iter().filter(|c| c.is_some()).count()) + .filter_map(Slot::owned) + .map(|page| page.iter().filter(|c| !c.is_shared()).count()) .sum() } + + /// Free every page and chunk a directory owns, and empty it. + fn free_dir(dir: &mut Vec) { + for page in dir.iter_mut() { + if let Some(chunks) = page.owned_mut() { + for chunk in chunks.iter_mut() { + chunk.release(); + } + } + page.release(); + } + dir.clear(); + } } /// MEASUREMENT that this structure is judged on, and the test's instrument. @@ -1433,9 +1628,12 @@ mod tests { /// (`proto_id`), and a 64-bit prototype identity does not fit the padding. /// 40 -> 48 is deliberate too: the per-slot field representation (charter /// step 5, `rep`) is a shape fact with no free bits left to live in. + /// 48 -> 56 is deliberate: POSBOUND (`position_bound`, offset 40) is the + /// one-compare position fact the megamorphic read confirm needs; `rep` + /// moves to offset 48 behind it. #[test] fn the_record_geometry_is_free_and_facts_key_is_o1() { - assert_eq!(std::mem::size_of::(), 48, "record grew"); + assert_eq!(std::mem::size_of::(), 56, "record grew"); assert_eq!(std::mem::align_of::(), 8, "record realigned"); // `facts_key` folds the keys ADDRESS; it must never dereference it. diff --git a/crates/perry-runtime/src/object/shapes_tests.rs b/crates/perry-runtime/src/object/shapes_tests.rs index 77792d6780..aed11794ba 100644 --- a/crates/perry-runtime/src/object/shapes_tests.rs +++ b/crates/perry-runtime/src/object/shapes_tests.rs @@ -1480,6 +1480,44 @@ mod field_rep_identity_tests { assert_eq!(shape_descriptor_by_id(any).map(|d| d.rep), Some(REP_ANY)); } + /// POSBOUND is a fact of every record, a rep-typed one included: a shape + /// minted with an `F64` slot through the rep-aware entry carries the + /// bound its facts define, the same bound as its all-`Any` sibling + /// (`rep` is not an input of the definition), and its stored bound agrees + /// with the definition. + #[test] + fn a_rep_typed_shape_carries_its_own_position_bound() { + let _lock = crate::gc::global_side_table_test_lock(); + let keys = crate::array::js_array_alloc_with_length(3); + let mint_keys = |rep: u64| { + publish_shape_result(shape_descriptor_ensure_with_rep( + keys, + 3, + 3, + 0, + ShapeObjectKind::Ordinary, + 0, + PROTO, + 0, + rep, + None, + )) + }; + let any = mint_keys(REP_ANY); + let typed = mint_keys(with_slot_rep(0, 1, REP_F64)); + assert_ne!(any, typed, "premise: the rep makes a different shape"); + assert_eq!( + crate::object::shapes::test_positional_of_id(any), + Some((3, 3)), + "premise: the all-Any shape answers three positions" + ); + assert_eq!( + crate::object::shapes::test_positional_of_id(typed), + Some((3, 3)), + "the rep-typed shape must carry its own, agreeing POSBOUND" + ); + } + /// P1 is inert: the all-`Any` entry points mint the same id as an explicit /// `REP_ANY` request, so no existing caller's shape changes. #[test] diff --git a/crates/perry-runtime/src/object/static_shapes_tests.rs b/crates/perry-runtime/src/object/static_shapes_tests.rs index 1f76a671df..8be10eef27 100644 --- a/crates/perry-runtime/src/object/static_shapes_tests.rs +++ b/crates/perry-runtime/src/object/static_shapes_tests.rs @@ -151,6 +151,17 @@ fn pool_atom(text: &str) -> *const crate::StringHeader { crate::string::js_string_pool_atom(text.as_ptr(), text.len() as u32, hash, 0) } +/// The megamorphic confirm as the miss entry asks it: this agent's directory, +/// the site's key as NaN-boxed bits. +unsafe fn confirmed(id: u32, key: *const crate::StringHeader, guess: usize) -> bool { + shapes::slot_guess_confirmed( + shapes::ordinary_dir_addr(), + id, + crate::JSValue::string_ptr(key as *mut _).bits(), + guess, + ) +} + /// A seeded literal shape answers the megamorphic read's slot-guess confirm /// (`shapes::slot_guess_confirmed`: the position bound, then ONE pointer /// compare of the listed key against the site's key atom) exactly as a shape @@ -181,17 +192,20 @@ fn a_seeded_literal_shape_answers_the_megamorphic_confirm_like_a_minted_one() { (minted, a, 0, "minted `a` at 0"), (minted, k1, 1, "minted `k1` at 1"), ] { - assert_eq!( - shapes::test_positional_of_id(id), - Some((true, true)), - "{what}: the record must answer by position" + // POSBOUND is a fact of the record: a seeded shape (built by the + // slab insert, like every other) carries it, nonzero, equal to its + // definition. + let (stored, by_facts) = shapes::test_positional_of_id(id).expect("a record"); + assert!( + stored > 0 && stored == by_facts, + "{what}: the record must answer by position (POSBOUND {stored}, by facts {by_facts})" ); assert!( - unsafe { shapes::slot_guess_confirmed(id, key, guess) }, + unsafe { confirmed(id, key, guess) }, "{what}: the megamorphic confirm must accept the key atom" ); } // And refutes a wrong guess or another key. - assert!(!unsafe { shapes::slot_guess_confirmed(seeded, a, 1) }); - assert!(!unsafe { shapes::slot_guess_confirmed(seeded, k1, 1) }); + assert!(!unsafe { confirmed(seeded, a, 1) }); + assert!(!unsafe { confirmed(seeded, k1, 1) }); } diff --git a/scripts/shape_descriptor_census.py b/scripts/shape_descriptor_census.py index 394836e105..a4cecd0307 100644 --- a/scripts/shape_descriptor_census.py +++ b/scripts/shape_descriptor_census.py @@ -407,13 +407,16 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: # ordinary GC slot, and a budgeted dirty scan can hold that address # across mutator resumptions that insert descriptors — so a record's # address must never move for its lifetime. Chunks are individually - # boxed and never reallocated; only the directory of chunk pointers - # grows. Putting records into one flat `Vec` (or back into a rehashing - # bucket) moves them under the collector's feet. + # allocated (a `Slot`: an owned pointer, or the shared all-empty chunk + # that is never written) and never reallocated; only the directory of + # chunk pointers grows. Putting records into one flat `Vec` (or back + # into a rehashing bucket) moves them under the collector's feet. (r"slab\s*:\s*(?:std::cell::)?UnsafeCell\s*<\s*ShapeSlab\s*>", "by-id descriptor slab with stable record addresses"), - (r"type\s+Chunk\s*=\s*Box\s*<\s*\[\s*UnsafeCell\s*<\s*ShapeRecord\s*>\s*;\s*CHUNK_LEN\s*\]\s*>", "slab chunks individually boxed, never reallocated"), - (r"type\s+Page\s*=\s*Box\s*<\s*\[\s*Option\s*<\s*Chunk\s*>\s*;\s*PAGE_LEN\s*\]\s*>", "slab directory pages hold chunk pointers, not records"), - (r"pages\s*:\s*Vec\s*<\s*Option\s*<\s*Page\s*>\s*>", "slab directory is a vector of page pointers"), + (r"struct\s+Slot\s*<\s*T\s*>\s*\(\s*std::ptr::NonNull\s*<\s*T\s*>\s*\)", "a slab slot is one pointer to its own allocation"), + (r"type\s+ChunkCells\s*=\s*\[\s*UnsafeCell\s*<\s*ShapeRecord\s*>\s*;\s*CHUNK_LEN\s*\]", "slab chunks individually allocated, never reallocated"), + (r"type\s+PageSlots\s*=\s*\[\s*Slot\s*<\s*ChunkCells\s*>\s*;\s*PAGE_LEN\s*\]", "slab pages hold chunk pointers, not records"), + (r"type\s+Page\s*=\s*Slot\s*<\s*PageSlots\s*>", "slab directory entries are page pointers"), + (r"pages\s*:\s*Vec\s*<\s*Page\s*>", "slab directory is a vector of page pointers"), # `keys` must stay the FIRST field of the `#[repr(C)]` record: the # record address IS the rewritable keys slot (`keys_slot`). (r"#\[repr\(C\)\]\s*(?:#\[[^\]]*\]\s*)*pub\(crate\)\s+struct\s+ShapeRecord\s*\{\s*(?://[^\n]*\n\s*)*pub\(super\)\s+keys\s*:\s*u64", "slab record is repr(C) with the keys word first"), @@ -1131,8 +1134,8 @@ def run_sabotage_selftests(sources: dict[str, str], baseline: dict[str, object]) store_path = "crates/perry-runtime/src/object/shapes_store.rs" flat_slab = dict(sources) flat_slab[store_path] = flat_slab[store_path].replace( - "type Chunk = Box<[UnsafeCell; CHUNK_LEN]>;", - "type Chunk = Vec>;", + "type ChunkCells = [UnsafeCell; CHUNK_LEN];", + "type ChunkCells = Vec>;", 1, ) expect_rejected( diff --git a/scripts/thread_exit_address_globals.json b/scripts/thread_exit_address_globals.json index 7baa7ac1f8..a1878f4280 100644 --- a/scripts/thread_exit_address_globals.json +++ b/scripts/thread_exit_address_globals.json @@ -4091,6 +4091,16 @@ "verdict": "per_thread", "why": "`#[thread_local]` static (not `thread_local!`, so the megamorphic read reaches it with one thread-pointer-relative load): each thread mirrors ITS OWN ShapeSlab page directory (a Rust-heap Vec pointer and length, never an arena address), republished on every change to `pages` and cleared by ShapeSlab::drop before the Vec is freed; const-initialised to (null, 0) with no drop glue." }, + { + "file": "crates/perry-runtime/src/object/shapes_store.rs", + "names": [ + "EMPTY_CHUNK", + "EMPTY_PAGE", + "PERRY_EMPTY_SHAPE_DIR" + ], + "verdict": "no_heap_address", + "why": "Shared all-empty shape-slab structure (a chunk of EMPTY records, a page of pointers to that chunk, an empty page directory). Every pointer inside is the address of another immutable static in the program image, never an arena or heap address; nothing ever writes them, so an exited thread's Arena::drop cannot leave them dangling." + }, { "file": "crates/perry-runtime/src/object/static_shapes.rs", "names": [