From fc2ee36d491f85be971c7aeab4dc396251310354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 3 Oct 2026 10:15:08 +0200 Subject: [PATCH 1/2] perf(runtime,codegen): class accessor sites on shape facts (#10498) A read or store site whose key the receiver inherits as a compiled class accessor answers it from shape facts: the receiver ShapeId (key not own, prototype identity names the holder), the holder ShapeId (the key is an accessor lane) and the lane value against the primed pair. The emitted read and store towers call the getter/setter directly; the collecting miss entries ask the same entry first. The class -> declared-prototype registry link is write-once: a replacement or generic-origin redirect retires the displaced prototype ShapeId, so the accessor sites drop the global class_lookup_surface_generation, proto_validity and vtable_generation compares. --- .../PENDING-10498-accessor-shape-facts.md | 13 ++ crates/perry-abi/src/lib.rs | 30 +++ crates/perry-codegen/src/expr/body_call.rs | 24 ++ crates/perry-codegen/src/expr/property_get.rs | 1 + .../src/expr/property_get/accessor_arm.rs | 165 ++++++++++++++ .../expr/property_get/front_contract_tests.rs | 117 +++++++++- .../src/expr/property_get/generic_dispatch.rs | 44 +++- .../src/expr/property_get/tests.rs | 57 ++++- .../src/expr/put_value_store_ic.rs | 53 ++++- .../src/expr/put_value_store_ic/setter_arm.rs | 173 ++++++++++++++ .../src/gc_effects/linux-x86_64.tsv | 2 +- .../src/gc_effects/windows-x86_64.tsv | 2 +- .../perry-runtime/src/object/accessor_pair.rs | 10 +- .../src/object/class_meta_registry.rs | 8 + .../src/object/class_registry.rs | 1 + .../src/object/class_registry/state.rs | 36 ++- .../object/field_get_set/ic_miss/ic_slow.rs | 17 ++ .../src/object/method_site/read_holder.rs | 186 ++++++++------- .../method_site/read_holder/class_read.rs | 12 + .../src/proxy/put_value/packed_set.rs | 7 + .../src/proxy/put_value/setter_site.rs | 199 +++++++++------- .../test_gap_class_accessor_shape_facts.ts | 213 ++++++++++++++++++ 22 files changed, 1156 insertions(+), 214 deletions(-) create mode 100644 changelog.d/PENDING-10498-accessor-shape-facts.md create mode 100644 crates/perry-codegen/src/expr/property_get/accessor_arm.rs create mode 100644 crates/perry-codegen/src/expr/put_value_store_ic/setter_arm.rs create mode 100644 test-files/test_gap_class_accessor_shape_facts.ts diff --git a/changelog.d/PENDING-10498-accessor-shape-facts.md b/changelog.d/PENDING-10498-accessor-shape-facts.md new file mode 100644 index 0000000000..8a96dc02a0 --- /dev/null +++ b/changelog.d/PENDING-10498-accessor-shape-facts.md @@ -0,0 +1,13 @@ +Class getters and setters are answered from shape facts at the read and +store sites (#10498). A site that inherits a compiled class accessor checks +the receiver's ShapeId (the key is not own, and the prototype identity names +the holder), the holder's ShapeId (the key is still an accessor lane) and the +lane's value against the accessor pair it primed, then calls the compiled +getter or setter directly: inline in the emitted read and store towers, and +first thing in the collecting miss entries. The class-registry link from a +class to its declared prototype is written once; a replacement or a +generic-origin redirect retires the displaced prototype's ShapeId, so the +accessor read and setter sites no longer compare the global +`class_lookup_surface_generation`, `proto_validity` or `vtable_generation` +words. getter_read2 1,540 -> 223 instructions per iteration, setter_write2 +1,207 -> 313, setter_ctor 3.6x -> 1.5x field_ctor. diff --git a/crates/perry-abi/src/lib.rs b/crates/perry-abi/src/lib.rs index 351bf74fd9..40292555a8 100644 --- a/crates/perry-abi/src/lib.rs +++ b/crates/perry-abi/src/lib.rs @@ -538,6 +538,36 @@ pub const PIC_HOLDER_RECV_WORD: usize = 12; pub const PIC_HOLDER_OBJ_WORD: usize = 13; pub const PIC_HOLDER_SHAPE_WORD: usize = 14; pub const PIC_HOLDER_KIND_WORD: usize = 15; +/// A class-accessor entry (#10498): its kind word carries +/// [`PIC_HOLDER_ACCESSOR_BIT`] over the holder's inline slot (low 32 bits); +/// [`PIC_HOLDER_PAIR_WORD`] holds the raw address of the accessor pair that +/// slot held when the site primed (a strong root the collector rewrites), and +/// [`PIC_HOLDER_GETTER_WORD`] the compiled getter that pair names +/// (`double get(double this)`; 0 for a setter-only pair). A hit is the +/// receiver token, the holder's ShapeId and the slot's value equal to the +/// pair: then the getter is called with the receiver as `this`. +pub const PIC_HOLDER_ACCESSOR_BIT: i64 = 1 << 61; +pub const PIC_HOLDER_PAIR_WORD: usize = 16; +pub const PIC_HOLDER_GETTER_WORD: usize = 19; + +/// `proxy::put_value::setter_site` (#10498): the word of a static-key store +/// site's ways cache that names the site's compiled-setter entry, as +/// [`SETTER_SITE_TAG`] over the entry's address ([`SETTER_SITE_ADDRESS_MASK`]). +/// The entry is a `#[repr(C)]` record the emitted store tower reads +/// (`perry-codegen/src/expr/put_value_store_ic/setter_arm.rs`): the receiver +/// ShapeId and the holder ShapeId (u32 each), the holder's raw address, the +/// holder's inline slot (u32), the raw address of the accessor pair that slot +/// held at prime time, and the compiled setter it names +/// (`double set(double this, double v)`). +pub const PACKED_SET_SETTER_WORD: usize = 9; +pub const SETTER_SITE_TAG: u64 = 0xA2C2_0000_0000_0000; +pub const SETTER_SITE_ADDRESS_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; +pub const SETTER_SITE_RECV_SHAPE_OFFSET: usize = 0; +pub const SETTER_SITE_HOLDER_SHAPE_OFFSET: usize = 4; +pub const SETTER_SITE_HOLDER_OFFSET: usize = 8; +pub const SETTER_SITE_SLOT_OFFSET: usize = 16; +pub const SETTER_SITE_PAIR_OFFSET: usize = 24; +pub const SETTER_SITE_CODE_OFFSET: usize = 32; /// The site's holder state word, and its bit for a LATCHED site: one that /// refused, or whose non-own receivers took several shapes. Its misses ask the /// inherited-read hook, as a never-primed site's do. diff --git a/crates/perry-codegen/src/expr/body_call.rs b/crates/perry-codegen/src/expr/body_call.rs index be64271585..9118cf8369 100644 --- a/crates/perry-codegen/src/expr/body_call.rs +++ b/crates/perry-codegen/src/expr/body_call.rs @@ -107,6 +107,30 @@ pub(crate) fn emit_js_body_call_gc_leaf( blk.call_indirect_gc_leaf(DOUBLE, code_ptr, &native) } +/// Call a compiled class INSTANCE getter through its code address (a site's +/// class-accessor entry, `perry_abi::PIC_HOLDER_GETTER_WORD`): a method body +/// `double get(double this)`, the convention the runtime calls the same entry +/// with (`perry-runtime/src/closure/body_call.rs`, `js_method_body_fn!`). +/// `code_ptr` is a `ptr`, `this_box` the NaN-boxed receiver as a `double`. +/// The getter can run any JS: this is a collecting, possibly throwing call. +pub(crate) fn emit_class_getter_call(blk: &mut LlBlock, code_ptr: &str, this_box: &str) -> String { + blk.call_indirect(DOUBLE, code_ptr, &[(DOUBLE, this_box)]) +} + +/// Call a compiled class INSTANCE setter through its code address (a store +/// site's compiled-setter entry, `perry_abi::SETTER_SITE_CODE_OFFSET`): a +/// method body `double set(double this, double v)`, the convention the runtime +/// calls the same entry with. Its result is ignored: the assignment's value +/// is `v`. A collecting, possibly throwing call. +pub(crate) fn emit_class_setter_call( + blk: &mut LlBlock, + code_ptr: &str, + this_box: &str, + value: &str, +) -> String { + blk.call_indirect(DOUBLE, code_ptr, &[(DOUBLE, this_box), (DOUBLE, value)]) +} + /// The receiver bits for a call whose callee binds `this` to `undefined` /// (or whose callee is an arrow and never reads it). pub(crate) const JS_THIS_UNDEFINED: &str = crate::nanbox::TAG_UNDEFINED_I64; diff --git a/crates/perry-codegen/src/expr/property_get.rs b/crates/perry-codegen/src/expr/property_get.rs index 41645ac524..e9e6fa445f 100644 --- a/crates/perry-codegen/src/expr/property_get.rs +++ b/crates/perry-codegen/src/expr/property_get.rs @@ -44,6 +44,7 @@ use super::property_get_names::{ is_net_native_method_value, is_url_pattern_data_property, }; +mod accessor_arm; pub(crate) mod generic_dispatch; pub(crate) mod globalget; mod helpers; diff --git a/crates/perry-codegen/src/expr/property_get/accessor_arm.rs b/crates/perry-codegen/src/expr/property_get/accessor_arm.rs new file mode 100644 index 0000000000..e1b2729bb7 --- /dev/null +++ b/crates/perry-codegen/src/expr/property_get/accessor_arm.rs @@ -0,0 +1,165 @@ +//! The read site's class-accessor arm (#10498): `recv.k` where `k` is a +//! compiled class getter the receiver inherits, answered by two ShapeId +//! compares, one lane load and a direct call. +//! +//! The runtime primes the entry in the site's own cache +//! (`perry-runtime/src/object/method_site/read_holder.rs`, kind +//! `PIC_HOLDER_ACCESSOR_BIT`). Every fact the hit uses is a shape fact or the +//! lane's own value: +//! +//! * the receiver's ShapeId (the token) proves `k` is not own and names the +//! receiver's prototype identity, hence the holder: a recorded serial, or a +//! bare class whose registry link retires the holder's ShapeId if it is +//! ever replaced (`class_registry::retire_displaced_decl_prototype`); +//! * the holder's ShapeId proves `k`'s slot is still an accessor lane; +//! * the lane still holds the primed pair, which names the compiled getter. +//! +//! Emitted on the MRU compare's false edge, ahead of the GC-leaf front: +//! +//! ```text +//! packed == PACKED_GET_EMPTY else FRONT +//! c = @site ; c != null else FRONT +//! (u32)c[RECV] == [recv+4] else FRONT +//! c[KIND] & ACCESSOR else FRONT +//! [c[OBJ]+4] == (u32)c[SHAPE] else FRONT +//! [c[OBJ] + HDR + 8*(u32)c[KIND]] == POINTER_TAG | c[PAIR] +//! && no worker && c[GETTER] != 0 else FRONT +//! r = c[GETTER](recv) +//! ``` +//! +//! Only a site whose MRU word was never primed takes the arm: a site that +//! also reads own data keeps its misses on the front, which declines the +//! accessor kind, and the collecting slow call asks the same entry first. A +//! worker's start (`PERRY_METHOD_SITE_WORKERS_PRESENT`) sends every read to +//! the front: holder entries belong to the primary heap. + +use super::super::FnCtx; +use crate::runtime_abi as abi; +use crate::types::{I1, I32, I64, I8, PTR}; + +/// Emit the arm starting at `entry_idx` (a fresh block the MRU compare's false +/// edge targets). Every failing test branches to `miss_label` (the front); the +/// call's result reaches `merge_label`. Returns `(value, end label)` for the +/// tower's merge phi. +#[allow(clippy::too_many_arguments)] +pub(super) fn emit_class_accessor_arm( + ctx: &mut FnCtx<'_>, + entry_idx: usize, + packed_word: &str, + packed_empty: i64, + cache_slot_ref: &str, + recv_biased: &str, + recv_box: &str, + header_bytes: i64, + miss_label: &str, + merge_label: &str, +) -> (String, String) { + let cache_idx = ctx.new_block("pic.acc.cache"); + let recv_idx = ctx.new_block("pic.acc.recv"); + let kind_idx = ctx.new_block("pic.acc.kind"); + let holder_idx = ctx.new_block("pic.acc.holder"); + let lane_idx = ctx.new_block("pic.acc.lane"); + let call_idx = ctx.new_block("pic.acc.call"); + let cache_l = ctx.block_label(cache_idx); + let recv_l = ctx.block_label(recv_idx); + let kind_l = ctx.block_label(kind_idx); + let holder_l = ctx.block_label(holder_idx); + let lane_l = ctx.block_label(lane_idx); + let call_l = ctx.block_label(call_idx); + + // A site that reads own data has a primed MRU word; its accessor reads + // stay on the front and the slow call. + ctx.current_block = entry_idx; + { + let blk = ctx.block(); + let empty = blk.icmp_eq(I64, packed_word, &packed_empty.to_string()); + blk.cond_br(&empty, &cache_l, miss_label); + } + + // The full cache is allocated lazily; it is published with release order. + ctx.current_block = cache_idx; + let cache = { + let blk = ctx.block(); + let cache = blk.load_atomic_acquire(PTR, cache_slot_ref, 8); + let present = blk.icmp_ne(PTR, &cache, "null"); + blk.cond_br(&present, &recv_l, miss_label); + cache + }; + let word = |ctx: &mut FnCtx<'_>, index: usize| -> String { + let blk = ctx.block(); + let p = blk.gep(I64, &cache, &[(I64, &index.to_string())]); + blk.load(I64, &p) + }; + + // The receiver token against the receiver's ShapeId, re-read here so the + // hot compare's load keeps its single use. A token is + // `PIC_ID_TOKEN_BIT | ShapeId` and an empty entry is 0, so its low half + // is a ShapeId or 0; a receiver word equal to a ShapeId proves a live + // ordinary object of that shape (#10828 rule 3), and 0 is never one. + ctx.current_block = recv_idx; + let recv_word = word(ctx, abi::PIC_HOLDER_RECV_WORD); + { + let blk = ctx.block(); + let sid_ptr = crate::expr::receiver_range::emit_field_ptr(blk, recv_biased, 4); + let sid = blk.load(I32, &sid_ptr); + let primed = blk.trunc(I64, &recv_word, I32); + let same = blk.icmp_eq(I32, &sid, &primed); + blk.cond_br(&same, &kind_l, miss_label); + } + + ctx.current_block = kind_idx; + let kind = word(ctx, abi::PIC_HOLDER_KIND_WORD); + { + let blk = ctx.block(); + let bit = blk.and(I64, &kind, &abi::PIC_HOLDER_ACCESSOR_BIT.to_string()); + let accessor = blk.icmp_ne(I64, &bit, "0"); + blk.cond_br(&accessor, &holder_l, miss_label); + } + + // The holder's ShapeId against the primed one. + ctx.current_block = holder_idx; + let holder = word(ctx, abi::PIC_HOLDER_OBJ_WORD); + let holder_shape = word(ctx, abi::PIC_HOLDER_SHAPE_WORD); + { + let blk = ctx.block(); + let sid_addr = blk.add(I64, &holder, "4"); + let sid_ptr = blk.inttoptr(I64, &sid_addr); + let sid = blk.load(I32, &sid_ptr); + let primed = blk.trunc(I64, &holder_shape, I32); + let same = blk.icmp_eq(I32, &sid, &primed); + blk.cond_br(&same, &lane_l, miss_label); + } + + // The lane still holds the primed pair; no worker; a getter to call. + ctx.current_block = lane_idx; + let pair = word(ctx, abi::PIC_HOLDER_PAIR_WORD); + let getter = word(ctx, abi::PIC_HOLDER_GETTER_WORD); + { + let blk = ctx.block(); + let slot = blk.and(I64, &kind, "4294967295"); + let base_addr = blk.add(I64, &holder, &header_bytes.to_string()); + let base = blk.inttoptr(I64, &base_addr); + let lane_ptr = blk.gep(I64, &base, &[(I64, &slot)]); + let lane = blk.load(I64, &lane_ptr); + let tagged = blk.or(I64, &pair, crate::nanbox::POINTER_TAG_I64); + let same = blk.icmp_eq(I64, &lane, &tagged); + let workers = blk.load_atomic_seq_cst(I8, "@PERRY_METHOD_SITE_WORKERS_PRESENT", 1); + let workers = blk.zext(I8, &workers, I32); + let no_workers = blk.icmp_eq(I32, &workers, "0"); + let has_getter = blk.icmp_ne(I64, &getter, "0"); + let ok = blk.and(I1, &same, &no_workers); + let ok = blk.and(I1, &ok, &has_getter); + blk.cond_br(&ok, &call_l, miss_label); + } + + // The getter runs user code: a versioned loop records its bailout here, + // as on the collecting slow call. + ctx.current_block = call_idx; + crate::expr::emit_versioned_loop_callback_deopt(ctx); + let blk = ctx.block(); + let code = blk.inttoptr(I64, &getter); + let value = crate::expr::body_call::emit_class_getter_call(blk, &code, recv_box); + let end = blk.label.clone(); + blk.br(merge_label); + (value, end) +} diff --git a/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs b/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs index 9cf4706637..01da21a416 100644 --- a/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs +++ b/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs @@ -34,7 +34,18 @@ fn verify_front_flow(blocks: &Blocks) -> Result { }; let (entry, _) = tower_block(blocks, "pic.miss.front"); let (token, token_body) = tower_block(blocks, "pic.token"); - if predecessors(blocks, entry) != [token] || tower_cond_br(token_body).2 != entry { + // The front is reached from the token compare's false edge, directly or + // (#10498) through the class-accessor arm, whose every guard declines to + // it and each of which the token compare dominates. + let front_preds: Vec<&str> = if blocks.iter().any(|(l, _)| l.starts_with("pic.acc.")) { + verify_accessor_arm(blocks)? + } else { + if tower_cond_br(token_body).2 != entry { + return Err("the front entry must be dominated by the token compare".into()); + } + vec![token] + }; + if predecessors(blocks, entry) != front_preds { return Err("the front entry must be dominated by the token compare".into()); } // Every branch between the token miss and the front call resolves the @@ -51,7 +62,7 @@ fn verify_front_flow(blocks: &Blocks) -> Result { )); } for (label, expected_targets, expected_preds) in [ - (entry, vec![tsd, slow], vec![token]), + (entry, vec![tsd, slow], front_preds.clone()), (tsd, vec![fast, slow], vec![entry]), (fast, vec![join], vec![tsd]), (slow, vec![join], vec![entry, tsd]), @@ -135,6 +146,93 @@ fn verify_front_flow(blocks: &Blocks) -> Result { Ok(*index) } +/// The class-accessor arm's guards, in chain order (#10498, `accessor_arm.rs`). +pub(super) const ACCESSOR_ARM_GUARDS: [&str; 6] = [ + "pic.acc.empty", + "pic.acc.cache", + "pic.acc.recv", + "pic.acc.kind", + "pic.acc.holder", + "pic.acc.lane", +]; + +/// #10498: the class-accessor arm on `pic.token`'s false edge, ahead of the +/// front. Returns its guard blocks in chain order, after proving that +/// +/// * the token compare's false edge is the first guard; +/// * every guard ends in a live conditional branch whose true edge is the +/// next guard (the last guard's is the call block) and whose false edge is +/// the front's entry; +/// * every guard and the call block has exactly one predecessor (the token +/// compare for the first, the previous guard otherwise), so every guard +/// dominates the call; +/// * the call block makes exactly one call, an indirect +/// `call double %code(double %recv)` (the compiled getter), and continues +/// only to the tower's merge. +pub(super) fn verify_accessor_arm(blocks: &Blocks) -> Result, String> { + let find = |prefix: &str| -> Result<(&str, &[String]), String> { + let found: Vec<_> = blocks + .iter() + .filter(|(l, _)| l.starts_with(prefix)) + .collect(); + match found.as_slice() { + [(l, b)] => Ok((l.as_str(), b.as_slice())), + _ => Err(format!("expected one `{prefix}` block: {found:?}")), + } + }; + let (token, token_body) = find("pic.token")?; + let (front, _) = find("pic.miss.front")?; + let (call, call_body) = find("pic.acc.call")?; + let mut guards = Vec::new(); + for prefix in ACCESSOR_ARM_GUARDS { + guards.push(find(prefix)?); + } + if tower_cond_br(token_body).2 != guards[0].0 { + return Err("the token compare's false edge must enter the accessor arm".into()); + } + for (i, (label, body)) in guards.iter().enumerate() { + if !body.last().is_some_and(|l| l.starts_with("br i1 %")) { + return Err(format!( + "accessor guard {label} must branch on a live predicate" + )); + } + let (_, on_true, on_false) = tower_cond_br(body); + let next = guards.get(i + 1).map(|(l, _)| *l).unwrap_or(call); + if on_true != next || on_false != front { + return Err(format!( + "accessor guard {label} must continue to {next} and decline to the front: {body:?}" + )); + } + let pred = if i == 0 { token } else { guards[i - 1].0 }; + if predecessors(blocks, label) != [pred] { + return Err(format!( + "accessor guard {label} must be reached only from {pred}" + )); + } + } + if predecessors(blocks, call) != [guards[guards.len() - 1].0] { + return Err("the getter call must be reached only through every guard".into()); + } + let calls: Vec<&String> = call_body + .iter() + .filter(|l| l.contains(" call ") || l.contains(" invoke ") || l.starts_with("call ")) + .collect(); + if calls.len() != 1 || !calls[0].contains(" = call double %") { + return Err(format!( + "the accessor arm makes exactly one indirect getter call: {call_body:?}" + )); + } + if !call_body + .last() + .is_some_and(|l| l.starts_with("br label %pget.recv_merge.")) + { + return Err(format!( + "the getter's answer must reach the merge: {call_body:?}" + )); + } + Ok(guards.iter().map(|(l, _)| *l).collect()) +} + pub(super) fn front_call_block(blocks: &Blocks) -> (&str, &[String]) { let index = verify_front_flow(blocks).unwrap_or_else(|e| panic!("{e}: {blocks:?}")); (&blocks[index].0, &blocks[index].1) @@ -361,5 +459,20 @@ fn front_contract_rejects_lookup_bypasses_wrong_slots_and_wrong_declines() { let body = &mut wrong.iter_mut().find(|(l, _)| l == entry).unwrap().1; *body.last_mut().unwrap() = format!("br label %{slow}"); assert!(verify_front_flow(&wrong).is_err(), "{target}: front bypass"); + // #10498: an accessor guard that stops declining, or the receiver + // compare jumped over, must be caught. + if blocks.iter().any(|(l, _)| l.starts_with("pic.acc.")) { + for guard in ACCESSOR_ARM_GUARDS { + let mut wrong = blocks.clone(); + let (label, body) = tower_block(&blocks, guard); + let (_, on_true, _) = tower_cond_br(body); + let body = &mut wrong.iter_mut().find(|(l, _)| l == label).unwrap().1; + *body.last_mut().unwrap() = format!("br label %{on_true}"); + assert!( + verify_front_flow(&wrong).is_err(), + "{target}: accessor guard {guard} skipped" + ); + } + } } } diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 618f85bfd3..d6f2b7dfe5 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -107,6 +107,12 @@ fn emit_key_handle(ctx: &mut FnCtx<'_>, key_handle_global: &str) -> String { blk.and(I64, &key_bits, POINTER_MASK_I64) } +/// Does `triple` take the class-accessor arm? 64-bit targets only. +fn accessor_arm_target(triple: &str) -> bool { + (triple.starts_with("x86_64") || triple.starts_with("aarch64") || triple.starts_with("arm64")) + && !triple.contains("32") +} + /// The receiver's handle (its 48-bit payload), materialised in the CURRENT /// block: re-derived from the fused receiver test's biased value for every key /// but `.length`, or the entry-block mask for `.length`. Only ever called on @@ -951,10 +957,41 @@ pub(crate) fn lower_generic_property_get( let handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); emit_plain_array_length_arm(ctx, &handle, &token_miss_label, &merge_label, true) }); + // #10498: a site whose reads inherit a compiled class getter answers them + // inline, ahead of the front (`accessor_arm`). 64-bit targets only, as the + // method site: the entry's words address 8-byte slots behind the header. + let accessor_entry = match fused_recv.as_ref() { + Some(f) + if array_length_arm.is_none() + && front_idx.is_some() + && accessor_arm_target(ctx.target_triple) => + { + Some((ctx.new_block("pic.acc.empty"), f.biased.clone())) + } + _ => None, + }; if array_length_arm.is_none() { - ctx.block() - .cond_br(&token_eq, &hit_label, &token_miss_label); + let miss = accessor_entry + .as_ref() + .map(|(idx, _)| ctx.block_label(*idx)) + .unwrap_or_else(|| token_miss_label.clone()); + ctx.block().cond_br(&token_eq, &hit_label, &miss); } + let accessor_arm = accessor_entry.map(|(entry_idx, biased)| { + let header_bytes = crate::target_layout::object_header_size_bytes(ctx.target_triple) as i64; + super::accessor_arm::emit_class_accessor_arm( + ctx, + entry_idx, + &packed_word, + PACKED_GET_EMPTY, + &cache_slot_ref, + &biased, + &obj_box, + header_bytes, + &token_miss_label, + &merge_label, + ) + }); // `js_object_get_field_ic_miss` primes only slots below the descriptor's // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact @@ -1172,6 +1209,9 @@ pub(crate) fn lower_generic_property_get( if let Some((answered, front_end_label)) = front_arm.as_ref() { incoming.push((answered, front_end_label)); } + if let Some((value, accessor_end_label)) = accessor_arm.as_ref() { + incoming.push((value, accessor_end_label)); + } if let Some((sso_val, sso_end_label)) = sso_arm.as_ref() { incoming.push((sso_val, sso_end_label)); } diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 6f5c9520c6..60c0fb6829 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -16,7 +16,7 @@ use perry_hir::{Expr, Module, ModuleInitKind, Stmt}; #[path = "front_contract_tests.rs"] mod front_contract; -use front_contract::{front_call_block, verify_front_directory}; +use front_contract::{front_call_block, verify_accessor_arm, verify_front_directory}; fn ir_opts(debug_locations: bool, module_source: Option<&str>) -> CompileOptions { CompileOptions { @@ -571,9 +571,13 @@ fn generic_property_get_tries_ways_before_calling_the_miss_handler() { let (_, token) = tower_block(&blocks, "pic.token"); let (_, on_hit, on_miss) = tower_cond_br(token); assert!(on_hit.starts_with("pic.hit"), "{token:?}"); + // #10498: the class-accessor arm sits on the miss edge; every one of its + // guards declines to the front, so the front (the ways) is still asked + // before anything that collects except a proven accessor hit. + let arm = verify_accessor_arm(&blocks).unwrap_or_else(|e| panic!("{e}: {blocks:?}")); assert!( - on_miss.starts_with("pic.miss.front"), - "the compare's miss edge must reach the front (the ways) first: {token:?}" + on_miss == arm[0], + "the compare's miss edge must reach the accessor arm, then the front (the ways): {token:?}" ); let (front_label, front) = front_call_block(&blocks); assert!( @@ -648,13 +652,28 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { }) .map(|(l, _)| l.as_str()) .collect(); + // #10498: the front's predecessors are the class-accessor arm's guards, + // a chain the token compare's false edge enters and dominates. + let arm = verify_accessor_arm(&blocks).unwrap_or_else(|e| panic!("{e}: {blocks:?}")); assert_eq!( - preds, - vec![token_label], - "the front must have exactly one predecessor (pic.token), or it is no \ - longer dominated by it: {blocks:?}" + tower_cond_br(token).2, + arm[0], + "the token compare's false edge must enter the arm: {token:?}" + ); + assert_eq!( + preds, arm, + "the front must be reached only through the accessor arm's guards, \ + or it is no longer dominated by pic.token: {blocks:?}" ); let all: Vec<&String> = blocks.iter().flat_map(|(_, b)| b.iter()).collect(); + // The arm re-reads the receiver's ShapeId on the miss edge on purpose + // (pinned by `verify_accessor_arm`); the predicate counts below are about + // the token path. + let token_path: Vec<&String> = blocks + .iter() + .filter(|(l, _)| !l.starts_with("pic.acc.")) + .flat_map(|(_, b)| b.iter()) + .collect(); assert!( !all.iter().any(|l| l.contains("@PERRY_IC_EPOCH")), "the removed keys-pointer epoch global must not appear" @@ -670,7 +689,7 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { ("icmp eq i8 ", "the GC_TYPE_OBJECT compare", 0), ("icmp eq i32 %", "the ShapeId identity compare", 1), ] { - let n = all.iter().filter(|l| l.contains(needle)).count(); + let n = token_path.iter().filter(|l| l.contains(needle)).count(); assert_eq!( n, expect, "{what} appears {n} times, expected {expect} — a receiver \ @@ -1598,6 +1617,11 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { /// false edge makes ONE plain call to the GC-leaf front /// (`js_object_get_field_ic_front`), whose `TAG_HOLE` decline continues to the /// collecting slow call. +/// +/// #10498: ahead of the front, the class-accessor arm may make one more call, +/// an indirect call of a compiled getter it has proved (`verify_accessor_arm`); +/// it calls no runtime property entry, so the property-GET family below is +/// unchanged. #[test] fn the_generic_tower_is_one_leaf_call_two_exits_and_a_bounded_number_of_blocks() { let ir = emit(false, None); @@ -1684,6 +1708,16 @@ fn the_generic_tower_is_one_leaf_call_two_exits_and_a_bounded_number_of_blocks() // the one exit, and the join "pic.miss.call", "pget.recv_merge", + // #10498: the class-accessor arm on the compare's false edge, ahead of + // the front: six guards that decline to the front, and the direct + // getter call (`verify_accessor_arm` pins the chain). + "pic.acc.empty", + "pic.acc.cache", + "pic.acc.recv", + "pic.acc.kind", + "pic.acc.holder", + "pic.acc.lane", + "pic.acc.call", ]; // Labels carry a numeric suffix (`pic.token.6`); strip it for comparison. let mut normalized: Vec = blocks @@ -1734,10 +1768,13 @@ fn a_spill_entry_is_recognised_by_the_front_and_nowhere_at_the_site() { } let (_, token) = tower_block(&blocks, "pic.token"); let (_, _, on_miss) = tower_cond_br(token); + // #10498: through the class-accessor arm, every guard of which declines + // to the front. + let arm = verify_accessor_arm(&blocks).unwrap_or_else(|e| panic!("{e}: {blocks:?}")); assert!( - on_miss.starts_with("pic.miss.front"), + on_miss == arm[0], "the compare's false edge must reach the front, which recognises a \ - spill entry: {token:?}" + spill entry, through the accessor arm: {token:?}" ); } diff --git a/crates/perry-codegen/src/expr/put_value_store_ic.rs b/crates/perry-codegen/src/expr/put_value_store_ic.rs index 4e768a604b..178338fca8 100644 --- a/crates/perry-codegen/src/expr/put_value_store_ic.rs +++ b/crates/perry-codegen/src/expr/put_value_store_ic.rs @@ -298,7 +298,32 @@ pub(crate) fn emit_static_store_ic( let sid = ctx.block().load(I32, &sid_ptr); let stamp = ctx.block().trunc(I64, &word, I32); let shape_eq = ctx.block().icmp_eq(I32, &sid, &stamp); - ctx.block().cond_br(&shape_eq, &kind_label, &add_label); + // #10498: a store to a key the receiver inherits as a compiled class + // setter calls it inline (`setter_arm`), ahead of the key-add memo and the + // ways. 64-bit targets only: the entry is a record of 8-byte words. + let setter_entry = setter_arm_target(ctx.target_triple) + .then(|| ctx.new_block(&format!("{STORE_IC_STEM}.acc"))); + let shape_miss = setter_entry + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| add_label.clone()); + ctx.block().cond_br(&shape_eq, &kind_label, &shape_miss); + let setter_end = setter_entry.map(|entry_idx| { + let header_bytes = crate::target_layout::object_header_size_bytes(ctx.target_triple) as i64; + setter_arm::emit_setter_arm( + ctx, + entry_idx, + &word, + PACKED_SET_EMPTY, + &cache_slot_ref, + &sid, + obj_box, + value_double, + value_bits, + header_bytes, + &add_label, + &merge_label, + ) + }); let mut word_incoming: Vec<(String, String)> = vec![(word, tok_label.clone())]; // A word miss: the key-add memo's primary pre-shape next, BEFORE the @@ -503,16 +528,26 @@ pub(crate) fn emit_static_store_ic( ctx.block().br(&merge_label); ctx.current_block = merge_idx; - ctx.block().phi( - DOUBLE, - &[ - (value_double, &hit_end_label), - (value_double, &add_end_label), - (&miss_value, &miss_end_label), - ], - ) + let mut incoming: Vec<(&str, &str)> = vec![ + (value_double, &hit_end_label), + (value_double, &add_end_label), + (&miss_value, &miss_end_label), + ]; + if let Some(setter_end) = setter_end.as_ref() { + incoming.push((value_double, setter_end)); + } + ctx.block().phi(DOUBLE, &incoming) } +/// Does `triple` take the compiled-setter arm? 64-bit targets only. +fn setter_arm_target(triple: &str) -> bool { + (triple.starts_with("x86_64") || triple.starts_with("aarch64") || triple.starts_with("arm64")) + && !triple.contains("32") +} + +#[path = "put_value_store_ic/setter_arm.rs"] +mod setter_arm; + /// The key-add hit: `k` is not own on the receiver, and one of the site's /// add memos (`perry_runtime::proxy::put_value::packed_add`) names the /// receiver's PRE-shape. Entered from the pre-shape compare that matched, diff --git a/crates/perry-codegen/src/expr/put_value_store_ic/setter_arm.rs b/crates/perry-codegen/src/expr/put_value_store_ic/setter_arm.rs new file mode 100644 index 0000000000..7c5d395ea7 --- /dev/null +++ b/crates/perry-codegen/src/expr/put_value_store_ic/setter_arm.rs @@ -0,0 +1,173 @@ +//! The store site's compiled-setter arm (#10498): `recv.k = v` where `k` is a +//! compiled class setter the receiver inherits, answered by two ShapeId +//! compares, one lane load and a direct call. +//! +//! The runtime primes the entry (`perry-runtime/src/proxy/put_value/ +//! setter_site.rs`) and publishes it in the ways cache's setter word as +//! `SETTER_SITE_TAG | address` of a `#[repr(C)]` record. Every fact the hit +//! uses is a shape fact or the lane's own value: +//! +//! * the receiver's ShapeId proves `k` is not own and names the receiver's +//! prototype identity, hence the holder (a recorded serial, or a bare class +//! whose registry link retires the holder's ShapeId if it is ever replaced); +//! * the holder's ShapeId proves `k`'s slot is still an accessor lane; +//! * the lane still holds the primed pair, which names the compiled setter. +//! +//! Emitted on the shape compare's false edge, ahead of the key-add memo and +//! the existing-key ways: +//! +//! ```text +//! packed == PACKED_SET_EMPTY else ADD +//! c = @site ; c != null else ADD +//! w = c[SETTER] ; w & ~MASK == SETTER_SITE_TAG else ADD +//! e = w & MASK ; e.recv_shape == [recv+4] else ADD +//! [e.holder+4] == e.holder_shape else ADD +//! [e.holder + HDR + 8*e.slot] == POINTER_TAG | e.pair +//! && no worker && v is a Number else ADD +//! e.code(recv, v) ; the store's value is v +//! ``` +//! +//! A non-Number value takes the miss entry, which roots it across the setter +//! (the assignment's value is the stored value, and a heap value can move). + +use super::super::FnCtx; +use crate::runtime_abi as abi; +use crate::types::{I1, I32, I64, I8, PTR}; + +/// Emit the arm starting at `entry_idx` (a fresh block the shape compare's +/// false edge targets). Every failing test branches to `next_label` (the +/// key-add check); the call reaches `merge_label`. Returns the label of the +/// block that branches to the merge (its value is `value_double`). +#[allow(clippy::too_many_arguments)] +pub(super) fn emit_setter_arm( + ctx: &mut FnCtx<'_>, + entry_idx: usize, + packed_word: &str, + packed_empty: i64, + cache_slot_ref: &str, + sid: &str, + obj_box: &str, + value_double: &str, + value_bits: &str, + header_bytes: i64, + next_label: &str, + merge_label: &str, +) -> String { + let cache_idx = ctx.new_block("put.acc.cache"); + let tag_idx = ctx.new_block("put.acc.entry"); + let recv_idx = ctx.new_block("put.acc.recv"); + let holder_idx = ctx.new_block("put.acc.holder"); + let lane_idx = ctx.new_block("put.acc.lane"); + let call_idx = ctx.new_block("put.acc.call"); + let cache_l = ctx.block_label(cache_idx); + let tag_l = ctx.block_label(tag_idx); + let recv_l = ctx.block_label(recv_idx); + let holder_l = ctx.block_label(holder_idx); + let lane_l = ctx.block_label(lane_idx); + let call_l = ctx.block_label(call_idx); + + // A site with own-data or key-add traffic keeps its misses on the ways + // and the miss entry, which asks the same entry first. + ctx.current_block = entry_idx; + { + let blk = ctx.block(); + let empty = blk.icmp_eq(I64, packed_word, &packed_empty.to_string()); + blk.cond_br(&empty, &cache_l, next_label); + } + + ctx.current_block = cache_idx; + let cache = { + let blk = ctx.block(); + let cache = blk.load_atomic_acquire(PTR, cache_slot_ref, 8); + let present = blk.icmp_ne(PTR, &cache, "null"); + blk.cond_br(&present, &tag_l, next_label); + cache + }; + + ctx.current_block = tag_idx; + let entry = { + let blk = ctx.block(); + let word_ptr = blk.gep( + I64, + &cache, + &[(I64, &abi::PACKED_SET_SETTER_WORD.to_string())], + ); + let word = blk.load_atomic_monotonic(I64, &word_ptr, 8); + let tag = blk.and( + I64, + &word, + &(!abi::SETTER_SITE_ADDRESS_MASK as i64).to_string(), + ); + let tagged = blk.icmp_eq(I64, &tag, &(abi::SETTER_SITE_TAG as i64).to_string()); + let addr = blk.and( + I64, + &word, + &(abi::SETTER_SITE_ADDRESS_MASK as i64).to_string(), + ); + blk.cond_br(&tagged, &recv_l, next_label); + addr + }; + let field = |ctx: &mut FnCtx<'_>, offset: usize, ty| -> String { + let blk = ctx.block(); + let a = blk.add(I64, &entry, &offset.to_string()); + let p = blk.inttoptr(I64, &a); + blk.load(ty, &p) + }; + + ctx.current_block = recv_idx; + let recv_shape = field(ctx, abi::SETTER_SITE_RECV_SHAPE_OFFSET, I32); + { + let blk = ctx.block(); + let same = blk.icmp_eq(I32, &recv_shape, sid); + blk.cond_br(&same, &holder_l, next_label); + } + + ctx.current_block = holder_idx; + let holder = field(ctx, abi::SETTER_SITE_HOLDER_OFFSET, I64); + let holder_shape = field(ctx, abi::SETTER_SITE_HOLDER_SHAPE_OFFSET, I32); + { + let blk = ctx.block(); + let sid_addr = blk.add(I64, &holder, "4"); + let sid_ptr = blk.inttoptr(I64, &sid_addr); + let live = blk.load(I32, &sid_ptr); + let same = blk.icmp_eq(I32, &live, &holder_shape); + blk.cond_br(&same, &lane_l, next_label); + } + + ctx.current_block = lane_idx; + let slot = field(ctx, abi::SETTER_SITE_SLOT_OFFSET, I32); + let pair = field(ctx, abi::SETTER_SITE_PAIR_OFFSET, I64); + let code = field(ctx, abi::SETTER_SITE_CODE_OFFSET, I64); + { + let blk = ctx.block(); + let slot64 = blk.zext(I32, &slot, I64); + let base_addr = blk.add(I64, &holder, &header_bytes.to_string()); + let base = blk.inttoptr(I64, &base_addr); + let lane_ptr = blk.gep(I64, &base, &[(I64, &slot64)]); + let lane = blk.load(I64, &lane_ptr); + let tagged = blk.or(I64, &pair, crate::nanbox::POINTER_TAG_I64); + let same = blk.icmp_eq(I64, &lane, &tagged); + let workers = blk.load_atomic_seq_cst(I8, "@PERRY_METHOD_SITE_WORKERS_PRESENT", 1); + let workers = blk.zext(I8, &workers, I32); + let no_workers = blk.icmp_eq(I32, &workers, "0"); + // A Number: its top sixteen bits are outside Perry's tag band + // 0x7FF9..=0x7FFF (`JSValue::is_number`). + let top = blk.lshr(I64, value_bits, "48"); + let band = blk.sub(I64, &top, "32761"); + let number = blk.icmp_uge(I64, &band, "7"); + let ok = blk.and(I1, &same, &no_workers); + let ok = blk.and(I1, &ok, &number); + blk.cond_br(&ok, &call_l, next_label); + } + + // The setter runs user code: a versioned loop records its bailout here, + // as on the collecting miss entry. + ctx.current_block = call_idx; + crate::expr::emit_versioned_loop_callback_deopt(ctx); + let blk = ctx.block(); + let code_ptr = blk.inttoptr(I64, &code); + crate::expr::body_call::emit_class_setter_call(blk, &code_ptr, obj_box, value_double); + let end = blk.label.clone(); + blk.br(merge_label); + end +} diff --git a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv index 78dbfb296a..d2475415a9 100644 --- a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv @@ -2730,7 +2730,7 @@ js_register_class_constructor_flags Leaf js_register_class_extends_data_view Leaf js_register_class_extends_error Leaf js_register_class_extends_typed_array Leaf -js_register_class_generic_origin Leaf +js_register_class_generic_origin Reenters js_register_class_getter Reenters js_register_class_has_instance Leaf js_register_class_id Leaf diff --git a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv index dbd5c305e9..7119859491 100644 --- a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv @@ -2728,7 +2728,7 @@ js_register_class_constructor_flags Leaf js_register_class_extends_data_view Leaf js_register_class_extends_error Leaf js_register_class_extends_typed_array Leaf -js_register_class_generic_origin Leaf +js_register_class_generic_origin Reenters js_register_class_getter Reenters js_register_class_has_instance Leaf js_register_class_id Leaf diff --git a/crates/perry-runtime/src/object/accessor_pair.rs b/crates/perry-runtime/src/object/accessor_pair.rs index 7bc8f92140..b7ffaf4361 100644 --- a/crates/perry-runtime/src/object/accessor_pair.rs +++ b/crates/perry-runtime/src/object/accessor_pair.rs @@ -192,10 +192,12 @@ pub(crate) unsafe fn pair_of_value(value: u64) -> Option { } /// The compiled INSTANCE getter at an already-proved accessor slot, or -/// `Some(0)` for a setter-only pair (whose read is `undefined`). A class -/// accessor site's hit needs neither closure nor static-entry decoding. -/// The pair's tag, GC kind and length are still checked on every hit because -/// the slot's value may be replaced without a holder ShapeId transition. +/// `Some(0)` for a setter-only pair (whose read is `undefined`): what a class +/// accessor read site may call with the receiver as `this`. Asked when the +/// site primes. A pair is immutable once published, so the site keeps the +/// answer with the pair it came from, and each hit compares the slot's value +/// with that pair (the value may be replaced without a holder ShapeId +/// transition). /// /// # Safety /// `value` is the slot value of a key proved to carry `ENTRY_ACCESSOR`. diff --git a/crates/perry-runtime/src/object/class_meta_registry.rs b/crates/perry-runtime/src/object/class_meta_registry.rs index 76798e09f1..4455e677c8 100644 --- a/crates/perry-runtime/src/object/class_meta_registry.rs +++ b/crates/perry-runtime/src/object/class_meta_registry.rs @@ -231,6 +231,9 @@ pub extern "C" fn js_register_class_generic_origin(class_id: u32, generic_id: u3 if class_id == 0 || generic_id == 0 || class_id == generic_id { return; } + // The declared prototype `class_id`'s bare CLASS identity named before + // the redirect: a site may hold it as a shape-pinned holder. + let before = crate::object::class_decl_prototype_object(class_id); GENERIC_ORIGIN_LATCH.arm(); { let mut g = CLASS_GENERIC_ORIGIN.write().unwrap(); @@ -244,6 +247,11 @@ pub extern "C" fn js_register_class_generic_origin(class_id: u32, generic_id: u3 // `lookup_prototype_method`'s chain hop to the generic's id, so a cached // per-class-id chain verdict must retire (#10696). crate::object::class_lookup_surface_gen_bump(); + // A redirect that changes the holder retires the old one's ShapeId, as a + // registry replacement does (`retire_displaced_decl_prototype`). + if !before.is_null() && crate::object::class_decl_prototype_object(class_id) != before { + crate::object::class_registry::retire_displaced_decl_prototype(before); + } } /// Keepalive anchor: emitted only from generated module-init code, so the diff --git a/crates/perry-runtime/src/object/class_registry.rs b/crates/perry-runtime/src/object/class_registry.rs index 4824891ea0..06abd4744e 100644 --- a/crates/perry-runtime/src/object/class_registry.rs +++ b/crates/perry-runtime/src/object/class_registry.rs @@ -80,6 +80,7 @@ pub(crate) use crate::object::class_value::CLASS_ACCESSOR_DEFAULT_ATTRS; pub(crate) use state::async_resource_prototype_value; #[cfg(test)] pub(crate) use state::class_decl_prototype_object_root_store; +pub(crate) use state::retire_displaced_decl_prototype; pub(crate) use state::{ builtin_parent_ctor_in_chain, class_decl_prototype_method_names, class_decl_prototype_object, class_decl_prototype_value, class_decl_prototype_value_for_instance_class, diff --git a/crates/perry-runtime/src/object/class_registry/state.rs b/crates/perry-runtime/src/object/class_registry/state.rs index e65b367613..a48f51a2e7 100644 --- a/crates/perry-runtime/src/object/class_registry/state.rs +++ b/crates/perry-runtime/src/object/class_registry/state.rs @@ -799,11 +799,12 @@ pub(crate) fn class_decl_prototype_object_root_store(class_id: u32, proto_ptr: * if class_id == 0 || proto_ptr.is_null() { return; } - CLASS_DECL_PROTOTYPE_OBJECTS.with(|table| { + let displaced = CLASS_DECL_PROTOTYPE_OBJECTS.with(|table| { let mut guard = table.write().unwrap(); - guard - .get_or_insert_with(DeclPrototypeTable::default) - .insert(class_id, proto_ptr as usize); + let table = guard.get_or_insert_with(DeclPrototypeTable::default); + let previous = table.get(class_id).unwrap_or(0); + table.insert(class_id, proto_ptr as usize); + previous }); crate::gc::runtime_write_barrier_root_raw_ptr(proto_ptr); // Its sole caller, `class_decl_prototype_value`, argues at length against @@ -811,6 +812,33 @@ pub(crate) fn class_decl_prototype_object_root_store(class_id: u32, proto_ptr: * // whole class hierarchy). The lookup-surface generation is the separate // counter that exists for exactly this store (#10696). super::class_lookup_surface_gen_bump(); + if displaced != 0 && displaced != proto_ptr as usize { + retire_displaced_decl_prototype(displaced as *mut ObjectHeader); + } +} + +/// A bare CLASS prototype identity (`shapes::PROTO_ID_CLASS | class`) names +/// its holder through this registry, so the link `class -> C.prototype` is a +/// fact of every receiver ShapeId that carries the identity. The link is +/// written once per class identity; a write that REPLACES it (or a +/// generic-origin redirect that changes what it answers) must leave no site +/// trusting the old holder. The displaced prototype takes a semantic shape +/// transition: a process-unique ShapeId no site was trained on. Every site +/// that names that holder compares its ShapeId on each hit, so the relink is +/// seen through shapes alone, without a global generation word. +pub(crate) fn retire_displaced_decl_prototype(old: *mut ObjectHeader) { + if old.is_null() { + return; + } + // The mint is a no-move window here: `old` is a raw registry address. + let _no_move = crate::gc::GcSuppressScope::new(); + // SAFETY: `old` was a registered (rooted) prototype object until the + // store above, and nothing between that read and here can collect. + unsafe { + if crate::object::shapes::object_shape_stamp(old) != 0 { + crate::object::shapes::transition_object_shape_semantics(old); + } + } } pub(crate) fn class_parent_closure_root_store(class_id: u32, closure_addr: usize) { diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 7330fc6ce0..c2784bfc4b 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -261,6 +261,23 @@ fn ic_slow_body( unsafe { let header = &*((addr - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader); if header.obj_type == crate::gc::GC_TYPE_OBJECT { + // --- 1. the site's class-accessor entry --------------------- + // An inherited compiled getter: the emitted compare and the + // leaf front both miss on it by construction (the key is not + // own, and a getter can collect). Its hit is two ShapeId + // compares and one lane load (`read_holder`), so it is asked + // before anything else this entry would re-derive. An + // explicit-this native alias (#11725) keeps the miss + // handler's order: its alias read comes first. + if !crate::object::native_this_alias::alias_active() { + if let Some(value) = + crate::object::method_site::read_holder::try_cached_class_accessor( + obj, cache_slot, + ) + { + return f64::from_bits(value.bits()); + } + } let plain = header._reserved & crate::gc::OBJ_FLAG_HAS_DESCRIPTORS == 0; // --- 2. the MRU token hit the emitted hit path declined ----- if plain && !packed.is_null() { diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 2adebfc79e..7cc163658e 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -5,8 +5,11 @@ //! is an ordinary object, and its [[Prototype]] identity. Only a serial //! identity or `PROTO_ID_DEFAULT` (the realm's `Object.prototype`) pins ONE //! object for the GC-leaf data/absent path. A collecting accessor entry may -//! also use a declared class identity: a registry generation check proves -//! its rooted holder is still the registered prototype on every hit. +//! also use a declared class identity: the registry link from a class to +//! its declared prototype is written once, and a write that replaces or +//! redirects it retires the displaced prototype's ShapeId +//! (`class_registry::retire_displaced_decl_prototype`), so the holder-shape +//! compare below sees the relink. //! * The holder's ShapeId `SH` vouches that `k` is an own inline data slot of //! the holder `H` — or, for an ABSENT entry, that the terminal object lacks //! `k` and has a null [[Prototype]]. @@ -19,8 +22,10 @@ //! holder's slot is seen because the hit LOADS the slot. A delete is a shape //! transition (#10826), so a holder whose ShapeId matches still has the slot: //! the hit needs no `TAG_HOLE` test, as the emitted MRU hit needs none. The -//! collecting class-accessor route additionally checks the class registry's -//! lookup-surface generation for a bare declared-prototype link. +//! collecting class-accessor route is the same two compares: the holder's +//! ShapeId declares the key an accessor lane (`ENTRY_ACCESSOR` in its key +//! list), and the hit loads the lane and compares it with the pair it was +//! primed with, which names the compiled getter it then calls. //! //! The entry lives in the read site's own cache (`PicCache` words //! [`HOLDER_RECV`]..=[`HOLDER_REGISTERED`]). The holder and the hops are @@ -83,8 +88,11 @@ pub const HOLDER_SHAPE: usize = crate::codegen_abi::PIC_HOLDER_SHAPE_WORD; pub const HOLDER_KIND: usize = crate::codegen_abi::PIC_HOLDER_KIND_WORD; /// First of three intermediate hop addresses (depth 2..=4). pub const HOLDER_HOPS: usize = HOLDER_KIND + 1; -/// Data/absence: first and second hop ShapeIds. Class accessor: the class -/// lookup-surface generation at prime time (no hop pointer uses this word). +/// Data/absence: first and second hop ShapeIds. Class accessor: the compiled +/// getter entry the primed pair names (0 for a setter-only pair); a code +/// address, never a GC pointer, and the root scan never visits this word. +/// The accessor's pair itself (its raw address) is in [`HOLDER_HOPS`], a +/// strong root rewritten on move like the hop words it replaces. pub const HOLDER_HOP_SHAPES: usize = HOLDER_HOPS + 3; /// The site's holder state: [`STATE_REGISTERED`], [`STATE_LATCHED`] and the /// count of re-primes for a different receiver shape. @@ -103,8 +111,13 @@ pub const HOLDER_ABSENT_DEPTH1: i64 = crate::codegen_abi::PIC_HOLDER_ABSENT_DEPT pub const HOLDER_STUB: u64 = 1 << 63; const HOLDER_ABSENT_BIT: u64 = 1 << 62; /// A direct class-prototype accessor. It can collect and therefore never -/// answers from the GC-leaf front call. -const HOLDER_ACCESSOR: u64 = 1 << 61; +/// answers from the GC-leaf front call: the emitted arm calls the getter, and +/// the collecting slow call asks [`try_cached_class_accessor`] first. +const HOLDER_ACCESSOR: u64 = crate::codegen_abi::PIC_HOLDER_ACCESSOR_BIT as u64; +// The emitted class-accessor arm (`perry-codegen/src/expr/property_get/ +// accessor_arm.rs`) reads the pair and the getter from these words. +const _: () = assert!(HOLDER_HOPS == crate::codegen_abi::PIC_HOLDER_PAIR_WORD); +const _: () = assert!(HOLDER_HOP_SHAPES == crate::codegen_abi::PIC_HOLDER_GETTER_WORD); /// Depth-1 ABSENT entries can share one terminal holder across several /// receiver shapes. The spare hop words hold ShapeIds, never GC pointers. const HOLDER_MULTI_ABSENT: u64 = 1 << 60; @@ -357,6 +370,9 @@ struct Walk { slot: Option, hops: [(usize, u32); HOLDER_MAX_DEPTH - 1], depth: usize, + /// A class accessor entry's compiled getter (0 for data/absence and for + /// a setter-only pair). Its pair is `hops[0].0`. + raw_get: usize, } /// A hop the entry may name: an ordinary, shaped, non-exotic object whose @@ -401,8 +417,9 @@ pub(super) unsafe fn admitted_proto_id(obj: *const ObjectHeader) -> Option } /// A MIXED identity records an explicit serial link. A bare CLASS identity -/// does not pin its registry-resolved prototype, so priming resolves the live -/// declared-prototype pointer and the hit checks the registry's generation. +/// names its registry-resolved prototype: priming resolves the live +/// declared-prototype pointer, and a later relink retires that pointer's +/// ShapeId (see the module docs), which the hit's holder compare sees. unsafe fn class_link(recv: *const ObjectHeader) -> Option<*const ObjectHeader> { let pid = shape_proto_id(object_shape_stamp(recv))?; if object_proto_id(recv) != pid { @@ -418,31 +435,12 @@ unsafe fn class_link(recv: *const ObjectHeader) -> Option<*const ObjectHeader> { (!holder.is_null() && holder != recv).then_some(holder) } -/// The accessor's prime-time holder is still the direct prototype. Explicit -/// MIXED links are checked by pointer. Every writer that can replace a bare -/// CLASS registry link bumps the lookup-surface generation; GC rewrites both -/// this site's rooted holder and the registry root without changing it. -#[inline] -unsafe fn accessor_link_still_current(recv: *const ObjectHeader, stamp: u32, c: &PicCache) -> bool { - let Some(pid) = shape_proto_id(stamp) else { - return false; - }; - if object_proto_id(recv) != pid || c[HOLDER_OBJ] as usize == recv as usize { - return false; - } - if (PROTO_ID_CLASS..PROTO_ID_MIXED).contains(&pid) { - c[HOLDER_HOP_SHAPES] as u64 == crate::object::class_lookup_surface_generation() - } else if (PROTO_ID_MIXED..PROTO_ID_UNIQUE).contains(&pid) { - next_prototype(recv) as usize == c[HOLDER_OBJ] as usize - } else { - false - } -} - struct ClassAccessor { holder: usize, shape: u32, slot: u32, + /// The pair's raw address: the value the holder's lane holds. + pair: usize, raw_get: usize, } @@ -485,30 +483,28 @@ unsafe fn class_accessor_walk(recv: *const ObjectHeader, name: &[u8]) -> Option< { return None; } - let acc = crate::object::accessor_pair::pair_of_value(slot_bits(holder, slot))?; - if acc.raw_get == 0 && (acc.get != 0 || acc.raw_set == 0) { - return None; - } + let lane = slot_bits(holder, slot); + let raw_get = crate::object::accessor_pair::raw_instance_getter_of_value(lane)?; Some(ClassAccessor { holder, shape: object_shape_stamp(holder as *const ObjectHeader), slot, - raw_get: acc.raw_get, + pair: (lane & crate::value::POINTER_MASK) as usize, + raw_get, }) } +/// Call a compiled class getter with `recv` as `this`. The receiver is the +/// call's argument and nothing reads it afterwards, so nothing is rooted +/// here: the getter's own frame roots its parameter. +#[inline] unsafe fn invoke_class_getter(recv: *const ObjectHeader, raw_get: usize) -> crate::value::JSValue { if raw_get == 0 { return crate::value::JSValue::from_bits(crate::value::TAG_UNDEFINED); } - let scope = crate::gc::RuntimeHandleScope::new(); - let receiver = scope.root_raw_mut_ptr(recv as *mut ObjectHeader); let f = crate::closure::body_call::js_method_body_fn!(raw_get as *const u8;); - let bits = receiver.with_mut_ptr::(|ptr| { - let this = crate::value::js_nanbox_pointer(ptr as i64); - f(f64::from_bits(this.to_bits())).to_bits() - }); - crate::value::JSValue::from_bits(bits) + let this = crate::value::js_nanbox_pointer(recv as i64); + crate::value::JSValue::from_bits(f(this).to_bits()) } /// Collecting-path class data/absence memo; the leaf front never consults it. @@ -520,48 +516,43 @@ pub(crate) unsafe fn try_cached_class_read( } /// Collecting read-miss arm. The GC-leaf front always declines this kind. -/// Every hit confirms the receiver's shape and live link, the rooted holder's -/// shape, and the current accessor pair before invoking with the ORIGINAL receiver. +/// +/// Every fact is a shape fact or the lane's own value: +/// * the receiver's ShapeId (the token) proves the key is not own and names +/// the receiver's prototype identity, hence the holder (a serial, or a +/// bare class whose registry link retires the holder's ShapeId if it ever +/// changes); +/// * the holder's ShapeId proves the slot is still an accessor lane; +/// * the lane's value is the primed pair, so the cached compiled getter is +/// the one `[[Get]]` would call. +/// +/// The getter is called with the ORIGINAL receiver as `this`. +#[inline] pub(crate) unsafe fn try_cached_class_accessor( recv: *const ObjectHeader, cache_slot: *mut PicCacheSlot, ) -> Option { - if cache_slot.is_null() || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { - return None; - } let cache = crate::object::field_get_set::pic_slot_peek::(cache_slot); if cache.is_null() { return None; } let c = &*cache; - if c[HOLDER_KIND] as u64 & HOLDER_ACCESSOR == 0 - || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT - || crate::object::field_get_set::accessor_receiver_override_armed() - || crate::object::prototype_chain::resolution_stack_savepoint() != 0 - { + let kind = c[HOLDER_KIND] as u64; + if kind & HOLDER_ACCESSOR == 0 || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return None; } let stamp = object_shape_stamp(recv); - if stamp == 0 - || c[HOLDER_RECV] != (u64::from(stamp) | PIC_ID_TOKEN_BIT) as i64 - || !accessor_link_still_current(recv, stamp, c) - { + if stamp == 0 || c[HOLDER_RECV] != (u64::from(stamp) | PIC_ID_TOKEN_BIT) as i64 { return None; } let holder = c[HOLDER_OBJ] as usize; - if shape_word(holder) != c[HOLDER_SHAPE] as u32 { + if shape_word(holder) != c[HOLDER_SHAPE] as u32 + || slot_bits(holder, kind as u32) != crate::value::POINTER_TAG | c[HOLDER_HOPS] as u64 + { return None; } - // The holder's unchanged ShapeId carries the prime-time proof that this - // inline slot exists and is an accessor. Key/attribute changes transition - // the ShapeId; a raw-only pair replacement may not, so reread the pair - // itself on every hit before invoking. - let slot = c[HOLDER_KIND] as u32; - let raw_get = - crate::object::accessor_pair::raw_instance_getter_of_value(slot_bits(holder, slot))?; HITS_ACCESSOR.fetch_add(1, Ordering::Relaxed); - super::stats_report_enabled(); - Some(invoke_class_getter(recv, raw_get)) + Some(invoke_class_getter(recv, c[HOLDER_HOP_SHAPES] as usize)) } unsafe fn walk(recv: *const ObjectHeader, name: &[u8], class_first: bool) -> Option { @@ -571,6 +562,7 @@ unsafe fn walk(recv: *const ObjectHeader, name: &[u8], class_first: bool) -> Opt slot: None, hops: [(0, 0); HOLDER_MAX_DEPTH - 1], depth: 0, + raw_get: 0, }; let object_prototype = crate::array::object_prototype_addr_if_resolved(); let mut current = recv; @@ -693,12 +685,15 @@ pub(crate) unsafe fn prime_read_holder( if let Some(acc) = class_accessor_walk(recv, name) { let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); if !cache.is_null() { + let mut hops = [(0, 0); HOLDER_MAX_DEPTH - 1]; + hops[0].0 = acc.pair; let w = Walk { holder: acc.holder, holder_shape: acc.shape, slot: Some(acc.slot), - hops: [(0, 0); HOLDER_MAX_DEPTH - 1], + hops, depth: 1, + raw_get: acc.raw_get, }; publish(cache, recv, &w, true); } @@ -866,7 +861,7 @@ unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk, acc c[HOLDER_HOPS + i] = w.hops[i].0 as i64; } c[HOLDER_HOP_SHAPES] = if accessor { - crate::object::class_lookup_surface_generation() as i64 + w.raw_get as i64 } else { (u64::from(w.hops[0].1) | u64::from(w.hops[1].1) << 32) as i64 }; @@ -937,10 +932,25 @@ mod tests { 8.0 } + /// The entry a class accessor prime publishes for `acc`. + fn accessor_entry(acc: &ClassAccessor) -> Walk { + let mut hops = [(0, 0); HOLDER_MAX_DEPTH - 1]; + hops[0].0 = acc.pair; + Walk { + holder: acc.holder, + holder_shape: acc.shape, + slot: Some(acc.slot), + hops, + depth: 1, + raw_get: acc.raw_get, + } + } + /// Two holders with exactly one ShapeId but different compiled getters. /// Replacing a declared class's registry pointer leaves the receiver's - /// bare CLASS ShapeId unchanged; the collecting hit must compare the live - /// link, rather than trust receiver and holder shapes alone. + /// bare CLASS ShapeId unchanged. The replacement must retire the old + /// holder's ShapeId, so the hit's holder compare refuses the stale entry + /// with no global word to consult. #[test] fn class_accessor_rechecks_same_shape_holder_link() { if !crate::object::method_site::run_with_fresh_worker_gate( @@ -1009,19 +1019,12 @@ mod tests { let cache: &'static mut PicCache = Box::leak(Box::new([0; crate::codegen_abi::PIC_CACHE_WORDS])); let mut slot: PicCacheSlot = cache; - let w = Walk { - holder: first.holder, - holder_shape: first.shape, - slot: Some(first.slot), - hops: [(0, 0); HOLDER_MAX_DEPTH - 1], - depth: 1, - }; - unsafe { publish(cache, receiver, &w, true) }; - let first_generation = cache[HOLDER_HOP_SHAPES]; + unsafe { publish(cache, receiver, &accessor_entry(&first), true) }; assert_eq!( - first_generation as u64, - crate::object::class_lookup_surface_generation() + cache[HOLDER_HOP_SHAPES] as usize, + getter_two as *const () as usize ); + assert_eq!(cache[HOLDER_HOPS] as usize, first.pair); assert_eq!( unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), Some(2.0) @@ -1031,25 +1034,19 @@ mod tests { p2.with_const_ptr::(|ptr| { crate::object::test_seed_class_decl_prototype_object_root(CID, ptr as usize); }); - assert_ne!( - cache[HOLDER_HOP_SHAPES] as u64, - crate::object::class_lookup_surface_generation() - ); assert_eq!(unsafe { object_shape_stamp(receiver) }, recv_shape); + // The relink is seen through the old holder's ShapeId alone. + p1.with_const_ptr::(|old| { + assert_ne!(unsafe { object_shape_stamp(old) }, first.shape); + assert_ne!(unsafe { object_shape_stamp(old) }, 0); + }); assert!( unsafe { try_cached_class_accessor(receiver, &mut slot) }.is_none(), "stale getter was served after registry replacement" ); let second = unsafe { class_accessor_walk(receiver, b"path") }.expect("second accessor"); - let w = Walk { - holder: second.holder, - holder_shape: second.shape, - slot: Some(second.slot), - hops: [(0, 0); HOLDER_MAX_DEPTH - 1], - depth: 1, - }; - unsafe { publish(cache, receiver, &w, true) }; + unsafe { publish(cache, receiver, &accessor_entry(&second), true) }; assert!(read_accessor_same_shape_relinks() > old_relinks); assert_eq!( unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), @@ -1120,6 +1117,7 @@ mod tests { slot: None, hops: [(0, 0); HOLDER_MAX_DEPTH - 1], depth: 1, + raw_get: 0, }; for i in 0..11 { recv.parent_class_id = base + i; diff --git a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs index 03e02ab227..4f9c68236f 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs @@ -381,8 +381,20 @@ mod tests { } crate::object::class_decl_prototype_object_root_store(CID, b); assert_eq!(unsafe { answer(&entry, recv) }, None); + // The displaced holder's ShapeId was retired: an entry naming it can + // never answer again, whatever the registry says later. + assert_ne!(unsafe { object_shape_stamp(a) }, proto_shape); crate::object::class_decl_prototype_object_root_store(CID, a); + assert_eq!(unsafe { answer(&entry, recv) }, None); + let entry = Entry { + holder_shape: unsafe { object_shape_stamp(a) }, + ..entry + }; + assert_eq!( + unsafe { answer(&entry, recv) }, + Some(crate::value::TAG_UNDEFINED) + ); let record = Box::into_raw(Box::new(Site { entries: [entry; WAYS], next: 0, diff --git a/crates/perry-runtime/src/proxy/put_value/packed_set.rs b/crates/perry-runtime/src/proxy/put_value/packed_set.rs index 543d136bf5..377661312e 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_set.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_set.rs @@ -145,6 +145,13 @@ pub extern "C" fn js_put_value_set_packed_miss( packed: *const AtomicU64, ) -> f64 { let site = packed as *const super::packed_add::PackedSetSite; + // The site's compiled-setter entry: a store whose key the receiver + // inherits as a class accessor misses the emitted ways by construction. + // Its hit is two ShapeId compares and one lane load (`setter_site`), so it + // is asked before any other miss work re-derives what it already proves. + if let Some(stored) = unsafe { setter_site::try_hit(cache_slot, target, key, value) } { + return stored; + } // Charter step 5: migrate a receiver whose shape the lineage generalized // before the key-add memo or a way is keyed by it. let target_bits = target.to_bits(); diff --git a/crates/perry-runtime/src/proxy/put_value/setter_site.rs b/crates/perry-runtime/src/proxy/put_value/setter_site.rs index 549b00c0ca..90e5a34b0f 100644 --- a/crates/perry-runtime/src/proxy/put_value/setter_site.rs +++ b/crates/perry-runtime/src/proxy/put_value/setter_site.rs @@ -2,28 +2,51 @@ //! //! The packed store's first eight words are own-data ways and word eight is //! the key-add chain verdict. Word nine names this bounded, collecting-path -//! setter entry. The emitted leaf never reads it. A miss validates the live -//! class-prototype link, receiver and holder shapes, inline accessor slot and -//! raw setter before calling with the original receiver. Any uncertainty -//! falls through to ordinary `[[Set]]`. +//! setter entry. The emitted leaf never reads it. +//! +//! A hit is shape facts and the lane's own value, nothing global: +//! * the receiver's ShapeId proves the key is not own and names the +//! receiver's prototype identity, hence the holder (a recorded serial, or a +//! bare class whose registry link retires the holder's ShapeId if it is ever +//! replaced: `class_registry::retire_displaced_decl_prototype`); +//! * the holder's ShapeId proves the key's slot is still an accessor lane; +//! * the lane still holds the primed pair, which names the compiled setter. +//! +//! Any uncertainty falls through to ordinary `[[Set]]`, which re-primes. use super::*; use std::sync::atomic::{AtomicU64, Ordering}; -const SITE_TAG: u64 = 0xA2C2_0000_0000_0000; +const SITE_TAG: u64 = crate::codegen_abi::SETTER_SITE_TAG; +const _: () = assert!(PACKED_SET_SETTER_WORD == crate::codegen_abi::PACKED_SET_SETTER_WORD); +const _: () = assert!(crate::value::POINTER_MASK == crate::codegen_abi::SETTER_SITE_ADDRESS_MASK); +/// The site's entry. The emitted store tower reads it at the offsets +/// `perry_abi::SETTER_SITE_*_OFFSET` pin (`setter_arm.rs` in codegen). +#[repr(C)] struct Entry { - key: usize, - holder: usize, - class_id: u32, receiver_shape: u32, - bare_class_link: bool, holder_shape: u32, + /// The holder (a strong root, rewritten on move). + holder: usize, slot: u32, + /// The pair the holder's lane held at prime time: its raw address, a + /// strong root rewritten on move, so a hit compares one loaded word. + pair: usize, + /// The compiled setter the pair names. raw_set: usize, - validity: u64, - vtable_gen: u64, + /// The interned key (a strong root). + key: usize, } +const _: () = { + use crate::codegen_abi as abi; + assert!(std::mem::offset_of!(Entry, receiver_shape) == abi::SETTER_SITE_RECV_SHAPE_OFFSET); + assert!(std::mem::offset_of!(Entry, holder_shape) == abi::SETTER_SITE_HOLDER_SHAPE_OFFSET); + assert!(std::mem::offset_of!(Entry, holder) == abi::SETTER_SITE_HOLDER_OFFSET); + assert!(std::mem::offset_of!(Entry, slot) == abi::SETTER_SITE_SLOT_OFFSET); + assert!(std::mem::offset_of!(Entry, pair) == abi::SETTER_SITE_PAIR_OFFSET); + assert!(std::mem::offset_of!(Entry, raw_set) == abi::SETTER_SITE_CODE_OFFSET); +}; crate::perry_thread_local! { static ENTRIES: std::cell::UnsafeCell> = @@ -171,9 +194,6 @@ unsafe fn candidate( return None; } - let bare_class_link = (crate::object::shapes::PROTO_ID_CLASS - ..crate::object::shapes::PROTO_ID_MIXED) - .contains(&recv_shape.proto_id); let holder = class_link(recv)?; let holder_gc = crate::value::addr_class::try_read_gc_header(holder as usize)?; if holder_gc.obj_type != crate::gc::GC_TYPE_OBJECT @@ -199,10 +219,8 @@ unsafe fn candidate( { return None; } - let field = (holder as *const u8) - .add(std::mem::size_of::() + slot as usize * 8) - as *const u64; - let acc = crate::object::accessor_pair::pair_of_value(*field)?; + let lane = lane_bits(holder as usize, slot); + let acc = crate::object::accessor_pair::pair_of_value(lane)?; if acc.raw_set == 0 { return None; } @@ -215,104 +233,105 @@ unsafe fn candidate( Some(Entry { key: key as usize, holder: holder as usize, - class_id, receiver_shape: crate::object::shapes::object_shape_stamp(recv), - bare_class_link, holder_shape: crate::object::shapes::object_shape_stamp(holder), slot, + pair: (lane & crate::value::POINTER_MASK) as usize, raw_set: acc.raw_set, - validity: crate::object::proto_validity::proto_validity(), - vtable_gen: crate::object::vtable_generation(), }) } +/// The value of `holder`'s inline slot `slot`. +#[inline] +unsafe fn lane_bits(holder: usize, slot: u32) -> u64 { + std::ptr::read( + (holder as *const u8).add(std::mem::size_of::() + slot as usize * 8) + as *const u64, + ) +} + +/// The compiled setter `e` names for `recv`, when every fact still holds: the +/// receiver's ShapeId (key not own, prototype identity, hence the holder), +/// the holder's ShapeId (the slot is an accessor lane) and the lane's pair. +/// A ShapeId match also proves a live, non-forwarded ordinary object (#10828 +/// rule 3), so nothing per-object is re-read. +#[inline] unsafe fn validated_raw_set( e: &Entry, recv: *const crate::ObjectHeader, key: *const crate::StringHeader, ) -> Option { - let gc = crate::value::addr_class::try_read_gc_header(recv as usize)?; - if gc.obj_type != crate::gc::GC_TYPE_OBJECT - || gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 - || gc._reserved & crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO != 0 - || crate::object::dictionary::is_dictionary(recv) - { - return None; - } - let meta = (*recv).meta; - if !meta.is_null() - && ((*meta).elements != 0 - || (*meta).flags & crate::object::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER != 0) - { - return None; - } - if e.key != key as usize - || e.class_id != (*recv).class_id - || e.receiver_shape != crate::object::shapes::object_shape_stamp(recv) - || e.validity != crate::object::proto_validity::proto_validity() - || e.vtable_gen != crate::object::vtable_generation() - || crate::object::shapes::object_shape_stamp(e.holder as *const crate::ObjectHeader) - != e.holder_shape - { - return None; - } - if e.bare_class_link { - // ShapeId proves the class-link mode. Every declared-prototype root - // replacement (including generic-origin redirects) bumps the validity - // word checked above. GC moves both the registry root and this rooted - // holder entry together. A per-instance prototype change must either - // restamp the ShapeId or leave an explicit meta link, rejected here. - if gc._reserved & crate::gc::OBJ_FLAG_NULL_PROTO != 0 - || (!meta.is_null() && (*meta).prototype != 0) - || e.holder == recv as usize - { - return None; - } - } else if class_link(recv)? as usize != e.holder { - // MIXED receivers keep the full live per-object link comparison. - return None; - } - let holder_gc = crate::value::addr_class::try_read_gc_header(e.holder)?; - if holder_gc.obj_type != crate::gc::GC_TYPE_OBJECT - || holder_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 - { - return None; - } - let field = (e.holder as *const u8) - .add(std::mem::size_of::() + e.slot as usize * 8) - as *const u64; - let acc = crate::object::accessor_pair::pair_of_value(*field)?; - (acc.raw_set == e.raw_set && acc.raw_set != 0).then_some(acc.raw_set) + let stamp = crate::object::shapes::object_shape_stamp(recv); + (stamp != 0 + && e.receiver_shape == stamp + && e.key == key as usize + && crate::object::shapes::object_shape_stamp(e.holder as *const crate::ObjectHeader) + == e.holder_shape + && lane_bits(e.holder, e.slot) == crate::value::POINTER_TAG | e.pair as u64) + .then_some(e.raw_set) } +/// Call the compiled setter with `target` as `this`. The store's result is +/// `value`; a Number needs no root across the call, anything else is rooted +/// (a heap value can move while the setter runs). +#[inline] unsafe fn invoke(raw_set: usize, target: f64, value: f64) -> f64 { + let f = crate::closure::body_call::js_method_body_fn!(raw_set as *const u8; value); + if crate::value::JSValue::from_bits(value.to_bits()).is_number() { + let _ = f(target, value); + return value; + } let scope = crate::gc::RuntimeHandleScope::new(); - let recv_h = scope.root_nanbox_f64(target); let value_h = scope.root_nanbox_f64(value); - let f = crate::closure::body_call::js_method_body_fn!(raw_set as *const u8; value); - let _ = f(recv_h.get_nanbox_f64(), value_h.get_nanbox_f64()); + let _ = f(target, value); value_h.get_nanbox_f64() } -/// Collecting miss only; the emitted GC-leaf store never consults this word. -pub(super) unsafe fn try_set( +/// The receiver `target` names, when it is a heap pointer the entry could +/// describe. +#[inline] +fn receiver_of(target: f64) -> Option<*const crate::ObjectHeader> { + let bits = target.to_bits(); + if bits & !crate::value::POINTER_MASK != crate::value::POINTER_TAG { + return None; + } + let addr = (bits & crate::value::POINTER_MASK) as usize; + crate::value::addr_class::is_above_handle_band(addr) + .then_some(addr as *const crate::ObjectHeader) +} + +/// The entry's hit and nothing else: allocation-free until the setter runs, +/// and never a candidate walk, so the store-miss entry asks it before any +/// other miss work. +#[inline] +pub(super) unsafe fn try_hit( slot: *mut PackedSetWaysSlot, target: f64, key: *const crate::StringHeader, value: f64, ) -> Option { - if !primary_only() || slot.is_null() || key.is_null() { + let e = entry(slot)?; + if crate::object::method_site::WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return None; } - let bits = target.to_bits(); - if bits & !crate::value::POINTER_MASK != crate::value::POINTER_TAG { - return None; + let raw_set = validated_raw_set(e, receiver_of(target)?, key)?; + if stats_enabled() { + HITS.fetch_add(1, Ordering::Relaxed); } - let addr = (bits & crate::value::POINTER_MASK) as usize; - if !crate::value::addr_class::is_above_handle_band(addr) { + Some(invoke(raw_set, target, value)) +} + +/// Collecting miss only; the emitted GC-leaf store never consults this word. +pub(super) unsafe fn try_set( + slot: *mut PackedSetWaysSlot, + target: f64, + key: *const crate::StringHeader, + value: f64, +) -> Option { + if !primary_only() || slot.is_null() || key.is_null() { return None; } - let recv = addr as *const crate::ObjectHeader; + let recv = receiver_of(target)?; if let Some(e) = entry(slot) { if let Some(raw_set) = validated_raw_set(e, recv, key) { if stats_enabled() { @@ -354,6 +373,9 @@ pub(crate) fn scan_roots(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { if visitor.visit_tagged_usize_slot(&mut e.holder, crate::value::POINTER_TAG) { ROOT_REWRITES.fetch_add(1, Ordering::Relaxed); } + if visitor.visit_tagged_usize_slot(&mut e.pair, crate::value::POINTER_TAG) { + ROOT_REWRITES.fetch_add(1, Ordering::Relaxed); + } } }); } @@ -480,7 +502,10 @@ mod tests { } assert_eq!(call_set!(5.0), Some(5.0)); assert_eq!(call_set!(6.0), Some(6.0)); - assert!(unsafe { entry(&mut slot).unwrap().bare_class_link }); + assert_eq!( + unsafe { entry(&mut slot).unwrap().raw_set }, + first as *const () as usize + ); assert_eq!(FIRST.load(Ordering::Relaxed), 2); p2.with_const_ptr::(|p| { crate::object::test_seed_class_decl_prototype_object_root(CID, p as usize) diff --git a/test-files/test_gap_class_accessor_shape_facts.ts b/test-files/test_gap_class_accessor_shape_facts.ts new file mode 100644 index 0000000000..a2e72d05b9 --- /dev/null +++ b/test-files/test_gap_class_accessor_shape_facts.ts @@ -0,0 +1,213 @@ +// #10498: class getters/setters answered from shape facts at the read and +// store sites (receiver ShapeId -> holder, holder ShapeId -> accessor lane, +// the lane's pair -> the compiled getter/setter). Every scenario runs a site +// hot first, then changes one fact the site relies on, and keeps reading. + +class Point { + _x = 0; + _y = 0; + constructor(x: number, y: number) { + this.x = x; + this.y = y; + } + get x() { return this._x; } + set x(v: number) { this._x = v; } + get y() { return this._y; } + set y(v: number) { this._y = v * 2; } +} + +function sumXY(ps: any[]): number { + let s = 0; + for (let i = 0; i < ps.length; i++) s += ps[i].x + ps[i].y; + return s; +} +function writeX(p: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { p.x = i; s += p._x; } + return s; +} + +const pts: Point[] = []; +for (let i = 0; i < 40; i++) pts.push(new Point(i, i + 1)); +console.log("basic", sumXY(pts), writeX(pts[3], 50), pts[3].x, pts[3].y); + +// Inherited through two levels, and overridden in a subclass. +class A { + _v = 1; + get v() { return this._v * 10; } + set v(n: number) { this._v = n; } +} +class B extends A { tag = "b"; } +class C extends B { more = 3; } +class D extends B { + get v() { return -this._v; } + set v(n: number) { this._v = n + 100; } +} +function readV(o: any): number { return o.v; } +function writeV(o: any, n: number): void { o.v = n; } +const c = new C(); +const d = new D(); +let acc = 0; +for (let i = 0; i < 60; i++) { + writeV(c, i); + acc += readV(c); +} +console.log("two-levels", acc, c.v, (c as any)._v); +acc = 0; +for (let i = 0; i < 60; i++) { + const o: any = i % 3 === 0 ? d : c; + writeV(o, i); + acc += readV(o); +} +console.log("override", acc, d.v, (d as any)._v, c.v); + +// Redefined at runtime on the prototype: hot sites must see the new pair. +class R { + _n = 5; + get n() { return this._n; } + set n(v: number) { this._n = v; } +} +const r = new R(); +function readN(o: any): number { return o.n; } +function writeN(o: any, v: number): void { o.n = v; } +let before = 0; +for (let i = 0; i < 50; i++) { writeN(r, i); before += readN(r); } +Object.defineProperty(R.prototype, "n", { + get() { return this._n + 1000; }, + set(v: number) { this._n = v * 3; }, + configurable: true, +}); +let after = 0; +for (let i = 0; i < 50; i++) { writeN(r, i); after += readN(r); } +console.log("redefine", before, after, r.n, (r as any)._n); +// Getter-only replacement: the store has no setter now (sloppy: ignored). +Object.defineProperty(R.prototype, "n", { get() { return 7; }, configurable: true }); +writeN(r, 99); +console.log("getter-only-redefine", readN(r), (r as any)._n); +// Back to a data property on the prototype. +Object.defineProperty(R.prototype, "n", { value: 42, writable: true, configurable: true }); +console.log("data-redefine", readN(r), Object.prototype.hasOwnProperty.call(r, "n")); +writeN(r, 8); +console.log("own-after-data", readN(r), Object.prototype.hasOwnProperty.call(r, "n")); +// Deleted from the prototype: reads fall to undefined. +class Del { get g() { return 1; } } +const del = new Del(); +function readG(o: any): any { return o.g; } +let gs = 0; +for (let i = 0; i < 30; i++) gs += readG(del); +delete (Del.prototype as any).g; +console.log("delete", gs, readG(del)); + +// An own property shadowing the accessor on one instance. +const sh1 = new Point(1, 2); +const sh2 = new Point(3, 4); +Object.defineProperty(sh2, "x", { value: 77, writable: true }); +let shs = 0; +for (let i = 0; i < 30; i++) shs += (i % 2 ? sh1 : sh2).x; +console.log("shadow", shs); + +// setPrototypeOf on an instance moves it off the class. +const moved = new Point(5, 6); +let ms = 0; +for (let i = 0; i < 20; i++) ms += readX(moved); +Object.setPrototypeOf(moved, { get x() { return -1; } }); +for (let i = 0; i < 20; i++) ms += readX(moved); +function readX(o: any): number { return o.x; } +console.log("setPrototypeOf", ms); + +// Static accessors. +class S { + static _count = 0; + static get count() { return S._count; } + static set count(v: number) { S._count = v + 1; } +} +let ss = 0; +for (let i = 0; i < 30; i++) { S.count = i; ss += S.count; } +console.log("static", ss, S.count); + +// Accessors on object literals, several objects of one shape. +function mk(k: number) { + return { + _k: k, + get k2() { return this._k * 2; }, + set k2(v: number) { this._k = v; }, + }; +} +const lits = [mk(1), mk(2), mk(3)]; +let ls = 0; +for (let i = 0; i < 30; i++) { + const o: any = lits[i % 3]; + ls += o.k2; + o.k2 = i; +} +console.log("literal", ls, lits.map((o) => o.k2).join(",")); + +// Setter-only and getter-only class accessors. +class OnlyOne { + _w = 0; + set w(v: number) { this._w = v; } + get r() { return this._w + 1; } +} +const oo: any = new OnlyOne(); +let os = 0; +let refused = 0; +for (let i = 0; i < 30; i++) { + oo.w = i; + os += oo.r; + os += oo.w === undefined ? 1 : 0; + // Module code is strict: a store to a getter-only accessor throws. + try { oo.r = 5; } catch (e) { refused += (e as Error) instanceof TypeError ? 1 : 0; } +} +console.log("one-sided", os, refused, oo._w, oo.r); + +// Non-Number values through a setter (strings and objects) and back. +class Box { + _v: any = null; + get v() { return this._v; } + set v(x: any) { this._v = x; } +} +const bx: any = new Box(); +const kept: any[] = []; +for (let i = 0; i < 40; i++) { + bx.v = i % 2 ? "s" + i : { i }; + kept.push(bx.v); +} +const lastAssign = (bx.v = { done: true }); +console.log("values", kept.length, JSON.stringify(kept[38]), kept[39], JSON.stringify(lastAssign), JSON.stringify(bx.v)); + +// A throwing getter and setter, from hot sites inside try/catch. +class T { + _t = 0; + get t() { if (this._t > 25) throw new Error("get " + this._t); return this._t; } + set t(v: number) { if (v === 30) throw new Error("set " + v); this._t = v; } +} +const tt: any = new T(); +let caught = ""; +let ts = 0; +for (let i = 0; i < 35; i++) { + try { tt.t = i; ts += tt.t; } catch (e) { caught += (e as Error).message + ";"; } +} +console.log("throws", ts, caught); + +// The getter sees the original receiver as `this`. +class Self { get me() { return this; } } +class Sub2 extends Self { id = 9; } +const s2 = new Sub2(); +let same = 0; +for (let i = 0; i < 20; i++) if ((s2 as any).me === s2) same++; +console.log("this", same, (s2 as any).me.id); + +// Values allocated inside getters, many collections while sites are hot. +class Alloc { + n = 0; + get fresh() { return { n: this.n++, pad: [1, 2, 3, 4, 5, 6, 7, 8] }; } + set slot(v: any) { this.n += v.pad.length; } +} +const al: any = new Alloc(); +let as = 0; +for (let i = 0; i < 20000; i++) { + const f = al.fresh; + as += f.pad[i & 7]; + al.slot = f; +} +console.log("alloc", as, al.n); From 7819523fa58110ec2a58c160280d3f7c6844c99e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 3 Oct 2026 10:15:37 +0200 Subject: [PATCH 2/2] changelog: key the accessor fragment to PR 11784 --- ...0498-accessor-shape-facts.md => 11784-accessor-shape-facts.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{PENDING-10498-accessor-shape-facts.md => 11784-accessor-shape-facts.md} (100%) diff --git a/changelog.d/PENDING-10498-accessor-shape-facts.md b/changelog.d/11784-accessor-shape-facts.md similarity index 100% rename from changelog.d/PENDING-10498-accessor-shape-facts.md rename to changelog.d/11784-accessor-shape-facts.md