diff --git a/changelog.d/11784-accessor-shape-facts.md b/changelog.d/11784-accessor-shape-facts.md new file mode 100644 index 0000000000..8a96dc02a0 --- /dev/null +++ b/changelog.d/11784-accessor-shape-facts.md @@ -0,0 +1,13 @@ +Class getters and setters are answered from shape facts at the read and +store sites (#10498). A site that inherits a compiled class accessor checks +the receiver's ShapeId (the key is not own, and the prototype identity names +the holder), the holder's ShapeId (the key is still an accessor lane) and the +lane's value against the accessor pair it primed, then calls the compiled +getter or setter directly: inline in the emitted read and store towers, and +first thing in the collecting miss entries. The class-registry link from a +class to its declared prototype is written once; a replacement or a +generic-origin redirect retires the displaced prototype's ShapeId, so the +accessor read and setter sites no longer compare the global +`class_lookup_surface_generation`, `proto_validity` or `vtable_generation` +words. getter_read2 1,540 -> 223 instructions per iteration, setter_write2 +1,207 -> 313, setter_ctor 3.6x -> 1.5x field_ctor. diff --git a/crates/perry-abi/src/lib.rs b/crates/perry-abi/src/lib.rs index 351bf74fd9..40292555a8 100644 --- a/crates/perry-abi/src/lib.rs +++ b/crates/perry-abi/src/lib.rs @@ -538,6 +538,36 @@ pub const PIC_HOLDER_RECV_WORD: usize = 12; pub const PIC_HOLDER_OBJ_WORD: usize = 13; pub const PIC_HOLDER_SHAPE_WORD: usize = 14; pub const PIC_HOLDER_KIND_WORD: usize = 15; +/// A class-accessor entry (#10498): its kind word carries +/// [`PIC_HOLDER_ACCESSOR_BIT`] over the holder's inline slot (low 32 bits); +/// [`PIC_HOLDER_PAIR_WORD`] holds the raw address of the accessor pair that +/// slot held when the site primed (a strong root the collector rewrites), and +/// [`PIC_HOLDER_GETTER_WORD`] the compiled getter that pair names +/// (`double get(double this)`; 0 for a setter-only pair). A hit is the +/// receiver token, the holder's ShapeId and the slot's value equal to the +/// pair: then the getter is called with the receiver as `this`. +pub const PIC_HOLDER_ACCESSOR_BIT: i64 = 1 << 61; +pub const PIC_HOLDER_PAIR_WORD: usize = 16; +pub const PIC_HOLDER_GETTER_WORD: usize = 19; + +/// `proxy::put_value::setter_site` (#10498): the word of a static-key store +/// site's ways cache that names the site's compiled-setter entry, as +/// [`SETTER_SITE_TAG`] over the entry's address ([`SETTER_SITE_ADDRESS_MASK`]). +/// The entry is a `#[repr(C)]` record the emitted store tower reads +/// (`perry-codegen/src/expr/put_value_store_ic/setter_arm.rs`): the receiver +/// ShapeId and the holder ShapeId (u32 each), the holder's raw address, the +/// holder's inline slot (u32), the raw address of the accessor pair that slot +/// held at prime time, and the compiled setter it names +/// (`double set(double this, double v)`). +pub const PACKED_SET_SETTER_WORD: usize = 9; +pub const SETTER_SITE_TAG: u64 = 0xA2C2_0000_0000_0000; +pub const SETTER_SITE_ADDRESS_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; +pub const SETTER_SITE_RECV_SHAPE_OFFSET: usize = 0; +pub const SETTER_SITE_HOLDER_SHAPE_OFFSET: usize = 4; +pub const SETTER_SITE_HOLDER_OFFSET: usize = 8; +pub const SETTER_SITE_SLOT_OFFSET: usize = 16; +pub const SETTER_SITE_PAIR_OFFSET: usize = 24; +pub const SETTER_SITE_CODE_OFFSET: usize = 32; /// The site's holder state word, and its bit for a LATCHED site: one that /// refused, or whose non-own receivers took several shapes. Its misses ask the /// inherited-read hook, as a never-primed site's do. diff --git a/crates/perry-codegen/src/expr/body_call.rs b/crates/perry-codegen/src/expr/body_call.rs index be64271585..9118cf8369 100644 --- a/crates/perry-codegen/src/expr/body_call.rs +++ b/crates/perry-codegen/src/expr/body_call.rs @@ -107,6 +107,30 @@ pub(crate) fn emit_js_body_call_gc_leaf( blk.call_indirect_gc_leaf(DOUBLE, code_ptr, &native) } +/// Call a compiled class INSTANCE getter through its code address (a site's +/// class-accessor entry, `perry_abi::PIC_HOLDER_GETTER_WORD`): a method body +/// `double get(double this)`, the convention the runtime calls the same entry +/// with (`perry-runtime/src/closure/body_call.rs`, `js_method_body_fn!`). +/// `code_ptr` is a `ptr`, `this_box` the NaN-boxed receiver as a `double`. +/// The getter can run any JS: this is a collecting, possibly throwing call. +pub(crate) fn emit_class_getter_call(blk: &mut LlBlock, code_ptr: &str, this_box: &str) -> String { + blk.call_indirect(DOUBLE, code_ptr, &[(DOUBLE, this_box)]) +} + +/// Call a compiled class INSTANCE setter through its code address (a store +/// site's compiled-setter entry, `perry_abi::SETTER_SITE_CODE_OFFSET`): a +/// method body `double set(double this, double v)`, the convention the runtime +/// calls the same entry with. Its result is ignored: the assignment's value +/// is `v`. A collecting, possibly throwing call. +pub(crate) fn emit_class_setter_call( + blk: &mut LlBlock, + code_ptr: &str, + this_box: &str, + value: &str, +) -> String { + blk.call_indirect(DOUBLE, code_ptr, &[(DOUBLE, this_box), (DOUBLE, value)]) +} + /// The receiver bits for a call whose callee binds `this` to `undefined` /// (or whose callee is an arrow and never reads it). pub(crate) const JS_THIS_UNDEFINED: &str = crate::nanbox::TAG_UNDEFINED_I64; diff --git a/crates/perry-codegen/src/expr/property_get.rs b/crates/perry-codegen/src/expr/property_get.rs index 41645ac524..e9e6fa445f 100644 --- a/crates/perry-codegen/src/expr/property_get.rs +++ b/crates/perry-codegen/src/expr/property_get.rs @@ -44,6 +44,7 @@ use super::property_get_names::{ is_net_native_method_value, is_url_pattern_data_property, }; +mod accessor_arm; pub(crate) mod generic_dispatch; pub(crate) mod globalget; mod helpers; diff --git a/crates/perry-codegen/src/expr/property_get/accessor_arm.rs b/crates/perry-codegen/src/expr/property_get/accessor_arm.rs new file mode 100644 index 0000000000..e1b2729bb7 --- /dev/null +++ b/crates/perry-codegen/src/expr/property_get/accessor_arm.rs @@ -0,0 +1,165 @@ +//! The read site's class-accessor arm (#10498): `recv.k` where `k` is a +//! compiled class getter the receiver inherits, answered by two ShapeId +//! compares, one lane load and a direct call. +//! +//! The runtime primes the entry in the site's own cache +//! (`perry-runtime/src/object/method_site/read_holder.rs`, kind +//! `PIC_HOLDER_ACCESSOR_BIT`). Every fact the hit uses is a shape fact or the +//! lane's own value: +//! +//! * the receiver's ShapeId (the token) proves `k` is not own and names the +//! receiver's prototype identity, hence the holder: a recorded serial, or a +//! bare class whose registry link retires the holder's ShapeId if it is +//! ever replaced (`class_registry::retire_displaced_decl_prototype`); +//! * the holder's ShapeId proves `k`'s slot is still an accessor lane; +//! * the lane still holds the primed pair, which names the compiled getter. +//! +//! Emitted on the MRU compare's false edge, ahead of the GC-leaf front: +//! +//! ```text +//! packed == PACKED_GET_EMPTY else FRONT +//! c = @site ; c != null else FRONT +//! (u32)c[RECV] == [recv+4] else FRONT +//! c[KIND] & ACCESSOR else FRONT +//! [c[OBJ]+4] == (u32)c[SHAPE] else FRONT +//! [c[OBJ] + HDR + 8*(u32)c[KIND]] == POINTER_TAG | c[PAIR] +//! && no worker && c[GETTER] != 0 else FRONT +//! r = c[GETTER](recv) +//! ``` +//! +//! Only a site whose MRU word was never primed takes the arm: a site that +//! also reads own data keeps its misses on the front, which declines the +//! accessor kind, and the collecting slow call asks the same entry first. A +//! worker's start (`PERRY_METHOD_SITE_WORKERS_PRESENT`) sends every read to +//! the front: holder entries belong to the primary heap. + +use super::super::FnCtx; +use crate::runtime_abi as abi; +use crate::types::{I1, I32, I64, I8, PTR}; + +/// Emit the arm starting at `entry_idx` (a fresh block the MRU compare's false +/// edge targets). Every failing test branches to `miss_label` (the front); the +/// call's result reaches `merge_label`. Returns `(value, end label)` for the +/// tower's merge phi. +#[allow(clippy::too_many_arguments)] +pub(super) fn emit_class_accessor_arm( + ctx: &mut FnCtx<'_>, + entry_idx: usize, + packed_word: &str, + packed_empty: i64, + cache_slot_ref: &str, + recv_biased: &str, + recv_box: &str, + header_bytes: i64, + miss_label: &str, + merge_label: &str, +) -> (String, String) { + let cache_idx = ctx.new_block("pic.acc.cache"); + let recv_idx = ctx.new_block("pic.acc.recv"); + let kind_idx = ctx.new_block("pic.acc.kind"); + let holder_idx = ctx.new_block("pic.acc.holder"); + let lane_idx = ctx.new_block("pic.acc.lane"); + let call_idx = ctx.new_block("pic.acc.call"); + let cache_l = ctx.block_label(cache_idx); + let recv_l = ctx.block_label(recv_idx); + let kind_l = ctx.block_label(kind_idx); + let holder_l = ctx.block_label(holder_idx); + let lane_l = ctx.block_label(lane_idx); + let call_l = ctx.block_label(call_idx); + + // A site that reads own data has a primed MRU word; its accessor reads + // stay on the front and the slow call. + ctx.current_block = entry_idx; + { + let blk = ctx.block(); + let empty = blk.icmp_eq(I64, packed_word, &packed_empty.to_string()); + blk.cond_br(&empty, &cache_l, miss_label); + } + + // The full cache is allocated lazily; it is published with release order. + ctx.current_block = cache_idx; + let cache = { + let blk = ctx.block(); + let cache = blk.load_atomic_acquire(PTR, cache_slot_ref, 8); + let present = blk.icmp_ne(PTR, &cache, "null"); + blk.cond_br(&present, &recv_l, miss_label); + cache + }; + let word = |ctx: &mut FnCtx<'_>, index: usize| -> String { + let blk = ctx.block(); + let p = blk.gep(I64, &cache, &[(I64, &index.to_string())]); + blk.load(I64, &p) + }; + + // The receiver token against the receiver's ShapeId, re-read here so the + // hot compare's load keeps its single use. A token is + // `PIC_ID_TOKEN_BIT | ShapeId` and an empty entry is 0, so its low half + // is a ShapeId or 0; a receiver word equal to a ShapeId proves a live + // ordinary object of that shape (#10828 rule 3), and 0 is never one. + ctx.current_block = recv_idx; + let recv_word = word(ctx, abi::PIC_HOLDER_RECV_WORD); + { + let blk = ctx.block(); + let sid_ptr = crate::expr::receiver_range::emit_field_ptr(blk, recv_biased, 4); + let sid = blk.load(I32, &sid_ptr); + let primed = blk.trunc(I64, &recv_word, I32); + let same = blk.icmp_eq(I32, &sid, &primed); + blk.cond_br(&same, &kind_l, miss_label); + } + + ctx.current_block = kind_idx; + let kind = word(ctx, abi::PIC_HOLDER_KIND_WORD); + { + let blk = ctx.block(); + let bit = blk.and(I64, &kind, &abi::PIC_HOLDER_ACCESSOR_BIT.to_string()); + let accessor = blk.icmp_ne(I64, &bit, "0"); + blk.cond_br(&accessor, &holder_l, miss_label); + } + + // The holder's ShapeId against the primed one. + ctx.current_block = holder_idx; + let holder = word(ctx, abi::PIC_HOLDER_OBJ_WORD); + let holder_shape = word(ctx, abi::PIC_HOLDER_SHAPE_WORD); + { + let blk = ctx.block(); + let sid_addr = blk.add(I64, &holder, "4"); + let sid_ptr = blk.inttoptr(I64, &sid_addr); + let sid = blk.load(I32, &sid_ptr); + let primed = blk.trunc(I64, &holder_shape, I32); + let same = blk.icmp_eq(I32, &sid, &primed); + blk.cond_br(&same, &lane_l, miss_label); + } + + // The lane still holds the primed pair; no worker; a getter to call. + ctx.current_block = lane_idx; + let pair = word(ctx, abi::PIC_HOLDER_PAIR_WORD); + let getter = word(ctx, abi::PIC_HOLDER_GETTER_WORD); + { + let blk = ctx.block(); + let slot = blk.and(I64, &kind, "4294967295"); + let base_addr = blk.add(I64, &holder, &header_bytes.to_string()); + let base = blk.inttoptr(I64, &base_addr); + let lane_ptr = blk.gep(I64, &base, &[(I64, &slot)]); + let lane = blk.load(I64, &lane_ptr); + let tagged = blk.or(I64, &pair, crate::nanbox::POINTER_TAG_I64); + let same = blk.icmp_eq(I64, &lane, &tagged); + let workers = blk.load_atomic_seq_cst(I8, "@PERRY_METHOD_SITE_WORKERS_PRESENT", 1); + let workers = blk.zext(I8, &workers, I32); + let no_workers = blk.icmp_eq(I32, &workers, "0"); + let has_getter = blk.icmp_ne(I64, &getter, "0"); + let ok = blk.and(I1, &same, &no_workers); + let ok = blk.and(I1, &ok, &has_getter); + blk.cond_br(&ok, &call_l, miss_label); + } + + // The getter runs user code: a versioned loop records its bailout here, + // as on the collecting slow call. + ctx.current_block = call_idx; + crate::expr::emit_versioned_loop_callback_deopt(ctx); + let blk = ctx.block(); + let code = blk.inttoptr(I64, &getter); + let value = crate::expr::body_call::emit_class_getter_call(blk, &code, recv_box); + let end = blk.label.clone(); + blk.br(merge_label); + (value, end) +} diff --git a/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs b/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs index 9cf4706637..01da21a416 100644 --- a/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs +++ b/crates/perry-codegen/src/expr/property_get/front_contract_tests.rs @@ -34,7 +34,18 @@ fn verify_front_flow(blocks: &Blocks) -> Result { }; let (entry, _) = tower_block(blocks, "pic.miss.front"); let (token, token_body) = tower_block(blocks, "pic.token"); - if predecessors(blocks, entry) != [token] || tower_cond_br(token_body).2 != entry { + // The front is reached from the token compare's false edge, directly or + // (#10498) through the class-accessor arm, whose every guard declines to + // it and each of which the token compare dominates. + let front_preds: Vec<&str> = if blocks.iter().any(|(l, _)| l.starts_with("pic.acc.")) { + verify_accessor_arm(blocks)? + } else { + if tower_cond_br(token_body).2 != entry { + return Err("the front entry must be dominated by the token compare".into()); + } + vec![token] + }; + if predecessors(blocks, entry) != front_preds { return Err("the front entry must be dominated by the token compare".into()); } // Every branch between the token miss and the front call resolves the @@ -51,7 +62,7 @@ fn verify_front_flow(blocks: &Blocks) -> Result { )); } for (label, expected_targets, expected_preds) in [ - (entry, vec![tsd, slow], vec![token]), + (entry, vec![tsd, slow], front_preds.clone()), (tsd, vec![fast, slow], vec![entry]), (fast, vec![join], vec![tsd]), (slow, vec![join], vec![entry, tsd]), @@ -135,6 +146,93 @@ fn verify_front_flow(blocks: &Blocks) -> Result { Ok(*index) } +/// The class-accessor arm's guards, in chain order (#10498, `accessor_arm.rs`). +pub(super) const ACCESSOR_ARM_GUARDS: [&str; 6] = [ + "pic.acc.empty", + "pic.acc.cache", + "pic.acc.recv", + "pic.acc.kind", + "pic.acc.holder", + "pic.acc.lane", +]; + +/// #10498: the class-accessor arm on `pic.token`'s false edge, ahead of the +/// front. Returns its guard blocks in chain order, after proving that +/// +/// * the token compare's false edge is the first guard; +/// * every guard ends in a live conditional branch whose true edge is the +/// next guard (the last guard's is the call block) and whose false edge is +/// the front's entry; +/// * every guard and the call block has exactly one predecessor (the token +/// compare for the first, the previous guard otherwise), so every guard +/// dominates the call; +/// * the call block makes exactly one call, an indirect +/// `call double %code(double %recv)` (the compiled getter), and continues +/// only to the tower's merge. +pub(super) fn verify_accessor_arm(blocks: &Blocks) -> Result, String> { + let find = |prefix: &str| -> Result<(&str, &[String]), String> { + let found: Vec<_> = blocks + .iter() + .filter(|(l, _)| l.starts_with(prefix)) + .collect(); + match found.as_slice() { + [(l, b)] => Ok((l.as_str(), b.as_slice())), + _ => Err(format!("expected one `{prefix}` block: {found:?}")), + } + }; + let (token, token_body) = find("pic.token")?; + let (front, _) = find("pic.miss.front")?; + let (call, call_body) = find("pic.acc.call")?; + let mut guards = Vec::new(); + for prefix in ACCESSOR_ARM_GUARDS { + guards.push(find(prefix)?); + } + if tower_cond_br(token_body).2 != guards[0].0 { + return Err("the token compare's false edge must enter the accessor arm".into()); + } + for (i, (label, body)) in guards.iter().enumerate() { + if !body.last().is_some_and(|l| l.starts_with("br i1 %")) { + return Err(format!( + "accessor guard {label} must branch on a live predicate" + )); + } + let (_, on_true, on_false) = tower_cond_br(body); + let next = guards.get(i + 1).map(|(l, _)| *l).unwrap_or(call); + if on_true != next || on_false != front { + return Err(format!( + "accessor guard {label} must continue to {next} and decline to the front: {body:?}" + )); + } + let pred = if i == 0 { token } else { guards[i - 1].0 }; + if predecessors(blocks, label) != [pred] { + return Err(format!( + "accessor guard {label} must be reached only from {pred}" + )); + } + } + if predecessors(blocks, call) != [guards[guards.len() - 1].0] { + return Err("the getter call must be reached only through every guard".into()); + } + let calls: Vec<&String> = call_body + .iter() + .filter(|l| l.contains(" call ") || l.contains(" invoke ") || l.starts_with("call ")) + .collect(); + if calls.len() != 1 || !calls[0].contains(" = call double %") { + return Err(format!( + "the accessor arm makes exactly one indirect getter call: {call_body:?}" + )); + } + if !call_body + .last() + .is_some_and(|l| l.starts_with("br label %pget.recv_merge.")) + { + return Err(format!( + "the getter's answer must reach the merge: {call_body:?}" + )); + } + Ok(guards.iter().map(|(l, _)| *l).collect()) +} + pub(super) fn front_call_block(blocks: &Blocks) -> (&str, &[String]) { let index = verify_front_flow(blocks).unwrap_or_else(|e| panic!("{e}: {blocks:?}")); (&blocks[index].0, &blocks[index].1) @@ -361,5 +459,20 @@ fn front_contract_rejects_lookup_bypasses_wrong_slots_and_wrong_declines() { let body = &mut wrong.iter_mut().find(|(l, _)| l == entry).unwrap().1; *body.last_mut().unwrap() = format!("br label %{slow}"); assert!(verify_front_flow(&wrong).is_err(), "{target}: front bypass"); + // #10498: an accessor guard that stops declining, or the receiver + // compare jumped over, must be caught. + if blocks.iter().any(|(l, _)| l.starts_with("pic.acc.")) { + for guard in ACCESSOR_ARM_GUARDS { + let mut wrong = blocks.clone(); + let (label, body) = tower_block(&blocks, guard); + let (_, on_true, _) = tower_cond_br(body); + let body = &mut wrong.iter_mut().find(|(l, _)| l == label).unwrap().1; + *body.last_mut().unwrap() = format!("br label %{on_true}"); + assert!( + verify_front_flow(&wrong).is_err(), + "{target}: accessor guard {guard} skipped" + ); + } + } } } diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 618f85bfd3..d6f2b7dfe5 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -107,6 +107,12 @@ fn emit_key_handle(ctx: &mut FnCtx<'_>, key_handle_global: &str) -> String { blk.and(I64, &key_bits, POINTER_MASK_I64) } +/// Does `triple` take the class-accessor arm? 64-bit targets only. +fn accessor_arm_target(triple: &str) -> bool { + (triple.starts_with("x86_64") || triple.starts_with("aarch64") || triple.starts_with("arm64")) + && !triple.contains("32") +} + /// The receiver's handle (its 48-bit payload), materialised in the CURRENT /// block: re-derived from the fused receiver test's biased value for every key /// but `.length`, or the entry-block mask for `.length`. Only ever called on @@ -951,10 +957,41 @@ pub(crate) fn lower_generic_property_get( let handle = recv_handle(ctx, fused_recv.as_ref(), &entry_handle); emit_plain_array_length_arm(ctx, &handle, &token_miss_label, &merge_label, true) }); + // #10498: a site whose reads inherit a compiled class getter answers them + // inline, ahead of the front (`accessor_arm`). 64-bit targets only, as the + // method site: the entry's words address 8-byte slots behind the header. + let accessor_entry = match fused_recv.as_ref() { + Some(f) + if array_length_arm.is_none() + && front_idx.is_some() + && accessor_arm_target(ctx.target_triple) => + { + Some((ctx.new_block("pic.acc.empty"), f.biased.clone())) + } + _ => None, + }; if array_length_arm.is_none() { - ctx.block() - .cond_br(&token_eq, &hit_label, &token_miss_label); + let miss = accessor_entry + .as_ref() + .map(|(idx, _)| ctx.block_label(*idx)) + .unwrap_or_else(|| token_miss_label.clone()); + ctx.block().cond_br(&token_eq, &hit_label, &miss); } + let accessor_arm = accessor_entry.map(|(entry_idx, biased)| { + let header_bytes = crate::target_layout::object_header_size_bytes(ctx.target_triple) as i64; + super::accessor_arm::emit_class_accessor_arm( + ctx, + entry_idx, + &packed_word, + PACKED_GET_EMPTY, + &cache_slot_ref, + &biased, + &obj_box, + header_bytes, + &token_miss_label, + &merge_label, + ) + }); // `js_object_get_field_ic_miss` primes only slots below the descriptor's // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact @@ -1172,6 +1209,9 @@ pub(crate) fn lower_generic_property_get( if let Some((answered, front_end_label)) = front_arm.as_ref() { incoming.push((answered, front_end_label)); } + if let Some((value, accessor_end_label)) = accessor_arm.as_ref() { + incoming.push((value, accessor_end_label)); + } if let Some((sso_val, sso_end_label)) = sso_arm.as_ref() { incoming.push((sso_val, sso_end_label)); } diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 6f5c9520c6..60c0fb6829 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -16,7 +16,7 @@ use perry_hir::{Expr, Module, ModuleInitKind, Stmt}; #[path = "front_contract_tests.rs"] mod front_contract; -use front_contract::{front_call_block, verify_front_directory}; +use front_contract::{front_call_block, verify_accessor_arm, verify_front_directory}; fn ir_opts(debug_locations: bool, module_source: Option<&str>) -> CompileOptions { CompileOptions { @@ -571,9 +571,13 @@ fn generic_property_get_tries_ways_before_calling_the_miss_handler() { let (_, token) = tower_block(&blocks, "pic.token"); let (_, on_hit, on_miss) = tower_cond_br(token); assert!(on_hit.starts_with("pic.hit"), "{token:?}"); + // #10498: the class-accessor arm sits on the miss edge; every one of its + // guards declines to the front, so the front (the ways) is still asked + // before anything that collects except a proven accessor hit. + let arm = verify_accessor_arm(&blocks).unwrap_or_else(|e| panic!("{e}: {blocks:?}")); assert!( - on_miss.starts_with("pic.miss.front"), - "the compare's miss edge must reach the front (the ways) first: {token:?}" + on_miss == arm[0], + "the compare's miss edge must reach the accessor arm, then the front (the ways): {token:?}" ); let (front_label, front) = front_call_block(&blocks); assert!( @@ -648,13 +652,28 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { }) .map(|(l, _)| l.as_str()) .collect(); + // #10498: the front's predecessors are the class-accessor arm's guards, + // a chain the token compare's false edge enters and dominates. + let arm = verify_accessor_arm(&blocks).unwrap_or_else(|e| panic!("{e}: {blocks:?}")); assert_eq!( - preds, - vec![token_label], - "the front must have exactly one predecessor (pic.token), or it is no \ - longer dominated by it: {blocks:?}" + tower_cond_br(token).2, + arm[0], + "the token compare's false edge must enter the arm: {token:?}" + ); + assert_eq!( + preds, arm, + "the front must be reached only through the accessor arm's guards, \ + or it is no longer dominated by pic.token: {blocks:?}" ); let all: Vec<&String> = blocks.iter().flat_map(|(_, b)| b.iter()).collect(); + // The arm re-reads the receiver's ShapeId on the miss edge on purpose + // (pinned by `verify_accessor_arm`); the predicate counts below are about + // the token path. + let token_path: Vec<&String> = blocks + .iter() + .filter(|(l, _)| !l.starts_with("pic.acc.")) + .flat_map(|(_, b)| b.iter()) + .collect(); assert!( !all.iter().any(|l| l.contains("@PERRY_IC_EPOCH")), "the removed keys-pointer epoch global must not appear" @@ -670,7 +689,7 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { ("icmp eq i8 ", "the GC_TYPE_OBJECT compare", 0), ("icmp eq i32 %", "the ShapeId identity compare", 1), ] { - let n = all.iter().filter(|l| l.contains(needle)).count(); + let n = token_path.iter().filter(|l| l.contains(needle)).count(); assert_eq!( n, expect, "{what} appears {n} times, expected {expect} — a receiver \ @@ -1598,6 +1617,11 @@ fn compact_get_mru_is_atomic_and_full_cache_remains_lazy() { /// false edge makes ONE plain call to the GC-leaf front /// (`js_object_get_field_ic_front`), whose `TAG_HOLE` decline continues to the /// collecting slow call. +/// +/// #10498: ahead of the front, the class-accessor arm may make one more call, +/// an indirect call of a compiled getter it has proved (`verify_accessor_arm`); +/// it calls no runtime property entry, so the property-GET family below is +/// unchanged. #[test] fn the_generic_tower_is_one_leaf_call_two_exits_and_a_bounded_number_of_blocks() { let ir = emit(false, None); @@ -1684,6 +1708,16 @@ fn the_generic_tower_is_one_leaf_call_two_exits_and_a_bounded_number_of_blocks() // the one exit, and the join "pic.miss.call", "pget.recv_merge", + // #10498: the class-accessor arm on the compare's false edge, ahead of + // the front: six guards that decline to the front, and the direct + // getter call (`verify_accessor_arm` pins the chain). + "pic.acc.empty", + "pic.acc.cache", + "pic.acc.recv", + "pic.acc.kind", + "pic.acc.holder", + "pic.acc.lane", + "pic.acc.call", ]; // Labels carry a numeric suffix (`pic.token.6`); strip it for comparison. let mut normalized: Vec = blocks @@ -1734,10 +1768,13 @@ fn a_spill_entry_is_recognised_by_the_front_and_nowhere_at_the_site() { } let (_, token) = tower_block(&blocks, "pic.token"); let (_, _, on_miss) = tower_cond_br(token); + // #10498: through the class-accessor arm, every guard of which declines + // to the front. + let arm = verify_accessor_arm(&blocks).unwrap_or_else(|e| panic!("{e}: {blocks:?}")); assert!( - on_miss.starts_with("pic.miss.front"), + on_miss == arm[0], "the compare's false edge must reach the front, which recognises a \ - spill entry: {token:?}" + spill entry, through the accessor arm: {token:?}" ); } diff --git a/crates/perry-codegen/src/expr/put_value_store_ic.rs b/crates/perry-codegen/src/expr/put_value_store_ic.rs index 4e768a604b..178338fca8 100644 --- a/crates/perry-codegen/src/expr/put_value_store_ic.rs +++ b/crates/perry-codegen/src/expr/put_value_store_ic.rs @@ -298,7 +298,32 @@ pub(crate) fn emit_static_store_ic( let sid = ctx.block().load(I32, &sid_ptr); let stamp = ctx.block().trunc(I64, &word, I32); let shape_eq = ctx.block().icmp_eq(I32, &sid, &stamp); - ctx.block().cond_br(&shape_eq, &kind_label, &add_label); + // #10498: a store to a key the receiver inherits as a compiled class + // setter calls it inline (`setter_arm`), ahead of the key-add memo and the + // ways. 64-bit targets only: the entry is a record of 8-byte words. + let setter_entry = setter_arm_target(ctx.target_triple) + .then(|| ctx.new_block(&format!("{STORE_IC_STEM}.acc"))); + let shape_miss = setter_entry + .map(|idx| ctx.block_label(idx)) + .unwrap_or_else(|| add_label.clone()); + ctx.block().cond_br(&shape_eq, &kind_label, &shape_miss); + let setter_end = setter_entry.map(|entry_idx| { + let header_bytes = crate::target_layout::object_header_size_bytes(ctx.target_triple) as i64; + setter_arm::emit_setter_arm( + ctx, + entry_idx, + &word, + PACKED_SET_EMPTY, + &cache_slot_ref, + &sid, + obj_box, + value_double, + value_bits, + header_bytes, + &add_label, + &merge_label, + ) + }); let mut word_incoming: Vec<(String, String)> = vec![(word, tok_label.clone())]; // A word miss: the key-add memo's primary pre-shape next, BEFORE the @@ -503,16 +528,26 @@ pub(crate) fn emit_static_store_ic( ctx.block().br(&merge_label); ctx.current_block = merge_idx; - ctx.block().phi( - DOUBLE, - &[ - (value_double, &hit_end_label), - (value_double, &add_end_label), - (&miss_value, &miss_end_label), - ], - ) + let mut incoming: Vec<(&str, &str)> = vec![ + (value_double, &hit_end_label), + (value_double, &add_end_label), + (&miss_value, &miss_end_label), + ]; + if let Some(setter_end) = setter_end.as_ref() { + incoming.push((value_double, setter_end)); + } + ctx.block().phi(DOUBLE, &incoming) } +/// Does `triple` take the compiled-setter arm? 64-bit targets only. +fn setter_arm_target(triple: &str) -> bool { + (triple.starts_with("x86_64") || triple.starts_with("aarch64") || triple.starts_with("arm64")) + && !triple.contains("32") +} + +#[path = "put_value_store_ic/setter_arm.rs"] +mod setter_arm; + /// The key-add hit: `k` is not own on the receiver, and one of the site's /// add memos (`perry_runtime::proxy::put_value::packed_add`) names the /// receiver's PRE-shape. Entered from the pre-shape compare that matched, diff --git a/crates/perry-codegen/src/expr/put_value_store_ic/setter_arm.rs b/crates/perry-codegen/src/expr/put_value_store_ic/setter_arm.rs new file mode 100644 index 0000000000..7c5d395ea7 --- /dev/null +++ b/crates/perry-codegen/src/expr/put_value_store_ic/setter_arm.rs @@ -0,0 +1,173 @@ +//! The store site's compiled-setter arm (#10498): `recv.k = v` where `k` is a +//! compiled class setter the receiver inherits, answered by two ShapeId +//! compares, one lane load and a direct call. +//! +//! The runtime primes the entry (`perry-runtime/src/proxy/put_value/ +//! setter_site.rs`) and publishes it in the ways cache's setter word as +//! `SETTER_SITE_TAG | address` of a `#[repr(C)]` record. Every fact the hit +//! uses is a shape fact or the lane's own value: +//! +//! * the receiver's ShapeId proves `k` is not own and names the receiver's +//! prototype identity, hence the holder (a recorded serial, or a bare class +//! whose registry link retires the holder's ShapeId if it is ever replaced); +//! * the holder's ShapeId proves `k`'s slot is still an accessor lane; +//! * the lane still holds the primed pair, which names the compiled setter. +//! +//! Emitted on the shape compare's false edge, ahead of the key-add memo and +//! the existing-key ways: +//! +//! ```text +//! packed == PACKED_SET_EMPTY else ADD +//! c = @site ; c != null else ADD +//! w = c[SETTER] ; w & ~MASK == SETTER_SITE_TAG else ADD +//! e = w & MASK ; e.recv_shape == [recv+4] else ADD +//! [e.holder+4] == e.holder_shape else ADD +//! [e.holder + HDR + 8*e.slot] == POINTER_TAG | e.pair +//! && no worker && v is a Number else ADD +//! e.code(recv, v) ; the store's value is v +//! ``` +//! +//! A non-Number value takes the miss entry, which roots it across the setter +//! (the assignment's value is the stored value, and a heap value can move). + +use super::super::FnCtx; +use crate::runtime_abi as abi; +use crate::types::{I1, I32, I64, I8, PTR}; + +/// Emit the arm starting at `entry_idx` (a fresh block the shape compare's +/// false edge targets). Every failing test branches to `next_label` (the +/// key-add check); the call reaches `merge_label`. Returns the label of the +/// block that branches to the merge (its value is `value_double`). +#[allow(clippy::too_many_arguments)] +pub(super) fn emit_setter_arm( + ctx: &mut FnCtx<'_>, + entry_idx: usize, + packed_word: &str, + packed_empty: i64, + cache_slot_ref: &str, + sid: &str, + obj_box: &str, + value_double: &str, + value_bits: &str, + header_bytes: i64, + next_label: &str, + merge_label: &str, +) -> String { + let cache_idx = ctx.new_block("put.acc.cache"); + let tag_idx = ctx.new_block("put.acc.entry"); + let recv_idx = ctx.new_block("put.acc.recv"); + let holder_idx = ctx.new_block("put.acc.holder"); + let lane_idx = ctx.new_block("put.acc.lane"); + let call_idx = ctx.new_block("put.acc.call"); + let cache_l = ctx.block_label(cache_idx); + let tag_l = ctx.block_label(tag_idx); + let recv_l = ctx.block_label(recv_idx); + let holder_l = ctx.block_label(holder_idx); + let lane_l = ctx.block_label(lane_idx); + let call_l = ctx.block_label(call_idx); + + // A site with own-data or key-add traffic keeps its misses on the ways + // and the miss entry, which asks the same entry first. + ctx.current_block = entry_idx; + { + let blk = ctx.block(); + let empty = blk.icmp_eq(I64, packed_word, &packed_empty.to_string()); + blk.cond_br(&empty, &cache_l, next_label); + } + + ctx.current_block = cache_idx; + let cache = { + let blk = ctx.block(); + let cache = blk.load_atomic_acquire(PTR, cache_slot_ref, 8); + let present = blk.icmp_ne(PTR, &cache, "null"); + blk.cond_br(&present, &tag_l, next_label); + cache + }; + + ctx.current_block = tag_idx; + let entry = { + let blk = ctx.block(); + let word_ptr = blk.gep( + I64, + &cache, + &[(I64, &abi::PACKED_SET_SETTER_WORD.to_string())], + ); + let word = blk.load_atomic_monotonic(I64, &word_ptr, 8); + let tag = blk.and( + I64, + &word, + &(!abi::SETTER_SITE_ADDRESS_MASK as i64).to_string(), + ); + let tagged = blk.icmp_eq(I64, &tag, &(abi::SETTER_SITE_TAG as i64).to_string()); + let addr = blk.and( + I64, + &word, + &(abi::SETTER_SITE_ADDRESS_MASK as i64).to_string(), + ); + blk.cond_br(&tagged, &recv_l, next_label); + addr + }; + let field = |ctx: &mut FnCtx<'_>, offset: usize, ty| -> String { + let blk = ctx.block(); + let a = blk.add(I64, &entry, &offset.to_string()); + let p = blk.inttoptr(I64, &a); + blk.load(ty, &p) + }; + + ctx.current_block = recv_idx; + let recv_shape = field(ctx, abi::SETTER_SITE_RECV_SHAPE_OFFSET, I32); + { + let blk = ctx.block(); + let same = blk.icmp_eq(I32, &recv_shape, sid); + blk.cond_br(&same, &holder_l, next_label); + } + + ctx.current_block = holder_idx; + let holder = field(ctx, abi::SETTER_SITE_HOLDER_OFFSET, I64); + let holder_shape = field(ctx, abi::SETTER_SITE_HOLDER_SHAPE_OFFSET, I32); + { + let blk = ctx.block(); + let sid_addr = blk.add(I64, &holder, "4"); + let sid_ptr = blk.inttoptr(I64, &sid_addr); + let live = blk.load(I32, &sid_ptr); + let same = blk.icmp_eq(I32, &live, &holder_shape); + blk.cond_br(&same, &lane_l, next_label); + } + + ctx.current_block = lane_idx; + let slot = field(ctx, abi::SETTER_SITE_SLOT_OFFSET, I32); + let pair = field(ctx, abi::SETTER_SITE_PAIR_OFFSET, I64); + let code = field(ctx, abi::SETTER_SITE_CODE_OFFSET, I64); + { + let blk = ctx.block(); + let slot64 = blk.zext(I32, &slot, I64); + let base_addr = blk.add(I64, &holder, &header_bytes.to_string()); + let base = blk.inttoptr(I64, &base_addr); + let lane_ptr = blk.gep(I64, &base, &[(I64, &slot64)]); + let lane = blk.load(I64, &lane_ptr); + let tagged = blk.or(I64, &pair, crate::nanbox::POINTER_TAG_I64); + let same = blk.icmp_eq(I64, &lane, &tagged); + let workers = blk.load_atomic_seq_cst(I8, "@PERRY_METHOD_SITE_WORKERS_PRESENT", 1); + let workers = blk.zext(I8, &workers, I32); + let no_workers = blk.icmp_eq(I32, &workers, "0"); + // A Number: its top sixteen bits are outside Perry's tag band + // 0x7FF9..=0x7FFF (`JSValue::is_number`). + let top = blk.lshr(I64, value_bits, "48"); + let band = blk.sub(I64, &top, "32761"); + let number = blk.icmp_uge(I64, &band, "7"); + let ok = blk.and(I1, &same, &no_workers); + let ok = blk.and(I1, &ok, &number); + blk.cond_br(&ok, &call_l, next_label); + } + + // The setter runs user code: a versioned loop records its bailout here, + // as on the collecting miss entry. + ctx.current_block = call_idx; + crate::expr::emit_versioned_loop_callback_deopt(ctx); + let blk = ctx.block(); + let code_ptr = blk.inttoptr(I64, &code); + crate::expr::body_call::emit_class_setter_call(blk, &code_ptr, obj_box, value_double); + let end = blk.label.clone(); + blk.br(merge_label); + end +} diff --git a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv index 78dbfb296a..d2475415a9 100644 --- a/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/linux-x86_64.tsv @@ -2730,7 +2730,7 @@ js_register_class_constructor_flags Leaf js_register_class_extends_data_view Leaf js_register_class_extends_error Leaf js_register_class_extends_typed_array Leaf -js_register_class_generic_origin Leaf +js_register_class_generic_origin Reenters js_register_class_getter Reenters js_register_class_has_instance Leaf js_register_class_id Leaf diff --git a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv index dbd5c305e9..7119859491 100644 --- a/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv +++ b/crates/perry-codegen/src/gc_effects/windows-x86_64.tsv @@ -2728,7 +2728,7 @@ js_register_class_constructor_flags Leaf js_register_class_extends_data_view Leaf js_register_class_extends_error Leaf js_register_class_extends_typed_array Leaf -js_register_class_generic_origin Leaf +js_register_class_generic_origin Reenters js_register_class_getter Reenters js_register_class_has_instance Leaf js_register_class_id Leaf diff --git a/crates/perry-runtime/src/object/accessor_pair.rs b/crates/perry-runtime/src/object/accessor_pair.rs index 7bc8f92140..b7ffaf4361 100644 --- a/crates/perry-runtime/src/object/accessor_pair.rs +++ b/crates/perry-runtime/src/object/accessor_pair.rs @@ -192,10 +192,12 @@ pub(crate) unsafe fn pair_of_value(value: u64) -> Option { } /// The compiled INSTANCE getter at an already-proved accessor slot, or -/// `Some(0)` for a setter-only pair (whose read is `undefined`). A class -/// accessor site's hit needs neither closure nor static-entry decoding. -/// The pair's tag, GC kind and length are still checked on every hit because -/// the slot's value may be replaced without a holder ShapeId transition. +/// `Some(0)` for a setter-only pair (whose read is `undefined`): what a class +/// accessor read site may call with the receiver as `this`. Asked when the +/// site primes. A pair is immutable once published, so the site keeps the +/// answer with the pair it came from, and each hit compares the slot's value +/// with that pair (the value may be replaced without a holder ShapeId +/// transition). /// /// # Safety /// `value` is the slot value of a key proved to carry `ENTRY_ACCESSOR`. diff --git a/crates/perry-runtime/src/object/class_meta_registry.rs b/crates/perry-runtime/src/object/class_meta_registry.rs index 76798e09f1..4455e677c8 100644 --- a/crates/perry-runtime/src/object/class_meta_registry.rs +++ b/crates/perry-runtime/src/object/class_meta_registry.rs @@ -231,6 +231,9 @@ pub extern "C" fn js_register_class_generic_origin(class_id: u32, generic_id: u3 if class_id == 0 || generic_id == 0 || class_id == generic_id { return; } + // The declared prototype `class_id`'s bare CLASS identity named before + // the redirect: a site may hold it as a shape-pinned holder. + let before = crate::object::class_decl_prototype_object(class_id); GENERIC_ORIGIN_LATCH.arm(); { let mut g = CLASS_GENERIC_ORIGIN.write().unwrap(); @@ -244,6 +247,11 @@ pub extern "C" fn js_register_class_generic_origin(class_id: u32, generic_id: u3 // `lookup_prototype_method`'s chain hop to the generic's id, so a cached // per-class-id chain verdict must retire (#10696). crate::object::class_lookup_surface_gen_bump(); + // A redirect that changes the holder retires the old one's ShapeId, as a + // registry replacement does (`retire_displaced_decl_prototype`). + if !before.is_null() && crate::object::class_decl_prototype_object(class_id) != before { + crate::object::class_registry::retire_displaced_decl_prototype(before); + } } /// Keepalive anchor: emitted only from generated module-init code, so the diff --git a/crates/perry-runtime/src/object/class_registry.rs b/crates/perry-runtime/src/object/class_registry.rs index 4824891ea0..06abd4744e 100644 --- a/crates/perry-runtime/src/object/class_registry.rs +++ b/crates/perry-runtime/src/object/class_registry.rs @@ -80,6 +80,7 @@ pub(crate) use crate::object::class_value::CLASS_ACCESSOR_DEFAULT_ATTRS; pub(crate) use state::async_resource_prototype_value; #[cfg(test)] pub(crate) use state::class_decl_prototype_object_root_store; +pub(crate) use state::retire_displaced_decl_prototype; pub(crate) use state::{ builtin_parent_ctor_in_chain, class_decl_prototype_method_names, class_decl_prototype_object, class_decl_prototype_value, class_decl_prototype_value_for_instance_class, diff --git a/crates/perry-runtime/src/object/class_registry/state.rs b/crates/perry-runtime/src/object/class_registry/state.rs index e65b367613..a48f51a2e7 100644 --- a/crates/perry-runtime/src/object/class_registry/state.rs +++ b/crates/perry-runtime/src/object/class_registry/state.rs @@ -799,11 +799,12 @@ pub(crate) fn class_decl_prototype_object_root_store(class_id: u32, proto_ptr: * if class_id == 0 || proto_ptr.is_null() { return; } - CLASS_DECL_PROTOTYPE_OBJECTS.with(|table| { + let displaced = CLASS_DECL_PROTOTYPE_OBJECTS.with(|table| { let mut guard = table.write().unwrap(); - guard - .get_or_insert_with(DeclPrototypeTable::default) - .insert(class_id, proto_ptr as usize); + let table = guard.get_or_insert_with(DeclPrototypeTable::default); + let previous = table.get(class_id).unwrap_or(0); + table.insert(class_id, proto_ptr as usize); + previous }); crate::gc::runtime_write_barrier_root_raw_ptr(proto_ptr); // Its sole caller, `class_decl_prototype_value`, argues at length against @@ -811,6 +812,33 @@ pub(crate) fn class_decl_prototype_object_root_store(class_id: u32, proto_ptr: * // whole class hierarchy). The lookup-surface generation is the separate // counter that exists for exactly this store (#10696). super::class_lookup_surface_gen_bump(); + if displaced != 0 && displaced != proto_ptr as usize { + retire_displaced_decl_prototype(displaced as *mut ObjectHeader); + } +} + +/// A bare CLASS prototype identity (`shapes::PROTO_ID_CLASS | class`) names +/// its holder through this registry, so the link `class -> C.prototype` is a +/// fact of every receiver ShapeId that carries the identity. The link is +/// written once per class identity; a write that REPLACES it (or a +/// generic-origin redirect that changes what it answers) must leave no site +/// trusting the old holder. The displaced prototype takes a semantic shape +/// transition: a process-unique ShapeId no site was trained on. Every site +/// that names that holder compares its ShapeId on each hit, so the relink is +/// seen through shapes alone, without a global generation word. +pub(crate) fn retire_displaced_decl_prototype(old: *mut ObjectHeader) { + if old.is_null() { + return; + } + // The mint is a no-move window here: `old` is a raw registry address. + let _no_move = crate::gc::GcSuppressScope::new(); + // SAFETY: `old` was a registered (rooted) prototype object until the + // store above, and nothing between that read and here can collect. + unsafe { + if crate::object::shapes::object_shape_stamp(old) != 0 { + crate::object::shapes::transition_object_shape_semantics(old); + } + } } pub(crate) fn class_parent_closure_root_store(class_id: u32, closure_addr: usize) { diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 7330fc6ce0..c2784bfc4b 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -261,6 +261,23 @@ fn ic_slow_body( unsafe { let header = &*((addr - crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader); if header.obj_type == crate::gc::GC_TYPE_OBJECT { + // --- 1. the site's class-accessor entry --------------------- + // An inherited compiled getter: the emitted compare and the + // leaf front both miss on it by construction (the key is not + // own, and a getter can collect). Its hit is two ShapeId + // compares and one lane load (`read_holder`), so it is asked + // before anything else this entry would re-derive. An + // explicit-this native alias (#11725) keeps the miss + // handler's order: its alias read comes first. + if !crate::object::native_this_alias::alias_active() { + if let Some(value) = + crate::object::method_site::read_holder::try_cached_class_accessor( + obj, cache_slot, + ) + { + return f64::from_bits(value.bits()); + } + } let plain = header._reserved & crate::gc::OBJ_FLAG_HAS_DESCRIPTORS == 0; // --- 2. the MRU token hit the emitted hit path declined ----- if plain && !packed.is_null() { diff --git a/crates/perry-runtime/src/object/method_site/read_holder.rs b/crates/perry-runtime/src/object/method_site/read_holder.rs index 2adebfc79e..7cc163658e 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder.rs @@ -5,8 +5,11 @@ //! is an ordinary object, and its [[Prototype]] identity. Only a serial //! identity or `PROTO_ID_DEFAULT` (the realm's `Object.prototype`) pins ONE //! object for the GC-leaf data/absent path. A collecting accessor entry may -//! also use a declared class identity: a registry generation check proves -//! its rooted holder is still the registered prototype on every hit. +//! also use a declared class identity: the registry link from a class to +//! its declared prototype is written once, and a write that replaces or +//! redirects it retires the displaced prototype's ShapeId +//! (`class_registry::retire_displaced_decl_prototype`), so the holder-shape +//! compare below sees the relink. //! * The holder's ShapeId `SH` vouches that `k` is an own inline data slot of //! the holder `H` — or, for an ABSENT entry, that the terminal object lacks //! `k` and has a null [[Prototype]]. @@ -19,8 +22,10 @@ //! holder's slot is seen because the hit LOADS the slot. A delete is a shape //! transition (#10826), so a holder whose ShapeId matches still has the slot: //! the hit needs no `TAG_HOLE` test, as the emitted MRU hit needs none. The -//! collecting class-accessor route additionally checks the class registry's -//! lookup-surface generation for a bare declared-prototype link. +//! collecting class-accessor route is the same two compares: the holder's +//! ShapeId declares the key an accessor lane (`ENTRY_ACCESSOR` in its key +//! list), and the hit loads the lane and compares it with the pair it was +//! primed with, which names the compiled getter it then calls. //! //! The entry lives in the read site's own cache (`PicCache` words //! [`HOLDER_RECV`]..=[`HOLDER_REGISTERED`]). The holder and the hops are @@ -83,8 +88,11 @@ pub const HOLDER_SHAPE: usize = crate::codegen_abi::PIC_HOLDER_SHAPE_WORD; pub const HOLDER_KIND: usize = crate::codegen_abi::PIC_HOLDER_KIND_WORD; /// First of three intermediate hop addresses (depth 2..=4). pub const HOLDER_HOPS: usize = HOLDER_KIND + 1; -/// Data/absence: first and second hop ShapeIds. Class accessor: the class -/// lookup-surface generation at prime time (no hop pointer uses this word). +/// Data/absence: first and second hop ShapeIds. Class accessor: the compiled +/// getter entry the primed pair names (0 for a setter-only pair); a code +/// address, never a GC pointer, and the root scan never visits this word. +/// The accessor's pair itself (its raw address) is in [`HOLDER_HOPS`], a +/// strong root rewritten on move like the hop words it replaces. pub const HOLDER_HOP_SHAPES: usize = HOLDER_HOPS + 3; /// The site's holder state: [`STATE_REGISTERED`], [`STATE_LATCHED`] and the /// count of re-primes for a different receiver shape. @@ -103,8 +111,13 @@ pub const HOLDER_ABSENT_DEPTH1: i64 = crate::codegen_abi::PIC_HOLDER_ABSENT_DEPT pub const HOLDER_STUB: u64 = 1 << 63; const HOLDER_ABSENT_BIT: u64 = 1 << 62; /// A direct class-prototype accessor. It can collect and therefore never -/// answers from the GC-leaf front call. -const HOLDER_ACCESSOR: u64 = 1 << 61; +/// answers from the GC-leaf front call: the emitted arm calls the getter, and +/// the collecting slow call asks [`try_cached_class_accessor`] first. +const HOLDER_ACCESSOR: u64 = crate::codegen_abi::PIC_HOLDER_ACCESSOR_BIT as u64; +// The emitted class-accessor arm (`perry-codegen/src/expr/property_get/ +// accessor_arm.rs`) reads the pair and the getter from these words. +const _: () = assert!(HOLDER_HOPS == crate::codegen_abi::PIC_HOLDER_PAIR_WORD); +const _: () = assert!(HOLDER_HOP_SHAPES == crate::codegen_abi::PIC_HOLDER_GETTER_WORD); /// Depth-1 ABSENT entries can share one terminal holder across several /// receiver shapes. The spare hop words hold ShapeIds, never GC pointers. const HOLDER_MULTI_ABSENT: u64 = 1 << 60; @@ -357,6 +370,9 @@ struct Walk { slot: Option, hops: [(usize, u32); HOLDER_MAX_DEPTH - 1], depth: usize, + /// A class accessor entry's compiled getter (0 for data/absence and for + /// a setter-only pair). Its pair is `hops[0].0`. + raw_get: usize, } /// A hop the entry may name: an ordinary, shaped, non-exotic object whose @@ -401,8 +417,9 @@ pub(super) unsafe fn admitted_proto_id(obj: *const ObjectHeader) -> Option } /// A MIXED identity records an explicit serial link. A bare CLASS identity -/// does not pin its registry-resolved prototype, so priming resolves the live -/// declared-prototype pointer and the hit checks the registry's generation. +/// names its registry-resolved prototype: priming resolves the live +/// declared-prototype pointer, and a later relink retires that pointer's +/// ShapeId (see the module docs), which the hit's holder compare sees. unsafe fn class_link(recv: *const ObjectHeader) -> Option<*const ObjectHeader> { let pid = shape_proto_id(object_shape_stamp(recv))?; if object_proto_id(recv) != pid { @@ -418,31 +435,12 @@ unsafe fn class_link(recv: *const ObjectHeader) -> Option<*const ObjectHeader> { (!holder.is_null() && holder != recv).then_some(holder) } -/// The accessor's prime-time holder is still the direct prototype. Explicit -/// MIXED links are checked by pointer. Every writer that can replace a bare -/// CLASS registry link bumps the lookup-surface generation; GC rewrites both -/// this site's rooted holder and the registry root without changing it. -#[inline] -unsafe fn accessor_link_still_current(recv: *const ObjectHeader, stamp: u32, c: &PicCache) -> bool { - let Some(pid) = shape_proto_id(stamp) else { - return false; - }; - if object_proto_id(recv) != pid || c[HOLDER_OBJ] as usize == recv as usize { - return false; - } - if (PROTO_ID_CLASS..PROTO_ID_MIXED).contains(&pid) { - c[HOLDER_HOP_SHAPES] as u64 == crate::object::class_lookup_surface_generation() - } else if (PROTO_ID_MIXED..PROTO_ID_UNIQUE).contains(&pid) { - next_prototype(recv) as usize == c[HOLDER_OBJ] as usize - } else { - false - } -} - struct ClassAccessor { holder: usize, shape: u32, slot: u32, + /// The pair's raw address: the value the holder's lane holds. + pair: usize, raw_get: usize, } @@ -485,30 +483,28 @@ unsafe fn class_accessor_walk(recv: *const ObjectHeader, name: &[u8]) -> Option< { return None; } - let acc = crate::object::accessor_pair::pair_of_value(slot_bits(holder, slot))?; - if acc.raw_get == 0 && (acc.get != 0 || acc.raw_set == 0) { - return None; - } + let lane = slot_bits(holder, slot); + let raw_get = crate::object::accessor_pair::raw_instance_getter_of_value(lane)?; Some(ClassAccessor { holder, shape: object_shape_stamp(holder as *const ObjectHeader), slot, - raw_get: acc.raw_get, + pair: (lane & crate::value::POINTER_MASK) as usize, + raw_get, }) } +/// Call a compiled class getter with `recv` as `this`. The receiver is the +/// call's argument and nothing reads it afterwards, so nothing is rooted +/// here: the getter's own frame roots its parameter. +#[inline] unsafe fn invoke_class_getter(recv: *const ObjectHeader, raw_get: usize) -> crate::value::JSValue { if raw_get == 0 { return crate::value::JSValue::from_bits(crate::value::TAG_UNDEFINED); } - let scope = crate::gc::RuntimeHandleScope::new(); - let receiver = scope.root_raw_mut_ptr(recv as *mut ObjectHeader); let f = crate::closure::body_call::js_method_body_fn!(raw_get as *const u8;); - let bits = receiver.with_mut_ptr::(|ptr| { - let this = crate::value::js_nanbox_pointer(ptr as i64); - f(f64::from_bits(this.to_bits())).to_bits() - }); - crate::value::JSValue::from_bits(bits) + let this = crate::value::js_nanbox_pointer(recv as i64); + crate::value::JSValue::from_bits(f(this).to_bits()) } /// Collecting-path class data/absence memo; the leaf front never consults it. @@ -520,48 +516,43 @@ pub(crate) unsafe fn try_cached_class_read( } /// Collecting read-miss arm. The GC-leaf front always declines this kind. -/// Every hit confirms the receiver's shape and live link, the rooted holder's -/// shape, and the current accessor pair before invoking with the ORIGINAL receiver. +/// +/// Every fact is a shape fact or the lane's own value: +/// * the receiver's ShapeId (the token) proves the key is not own and names +/// the receiver's prototype identity, hence the holder (a serial, or a +/// bare class whose registry link retires the holder's ShapeId if it ever +/// changes); +/// * the holder's ShapeId proves the slot is still an accessor lane; +/// * the lane's value is the primed pair, so the cached compiled getter is +/// the one `[[Get]]` would call. +/// +/// The getter is called with the ORIGINAL receiver as `this`. +#[inline] pub(crate) unsafe fn try_cached_class_accessor( recv: *const ObjectHeader, cache_slot: *mut PicCacheSlot, ) -> Option { - if cache_slot.is_null() || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { - return None; - } let cache = crate::object::field_get_set::pic_slot_peek::(cache_slot); if cache.is_null() { return None; } let c = &*cache; - if c[HOLDER_KIND] as u64 & HOLDER_ACCESSOR == 0 - || crate::agent::current_agent() != crate::agent::PRIMARY_AGENT - || crate::object::field_get_set::accessor_receiver_override_armed() - || crate::object::prototype_chain::resolution_stack_savepoint() != 0 - { + let kind = c[HOLDER_KIND] as u64; + if kind & HOLDER_ACCESSOR == 0 || WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return None; } let stamp = object_shape_stamp(recv); - if stamp == 0 - || c[HOLDER_RECV] != (u64::from(stamp) | PIC_ID_TOKEN_BIT) as i64 - || !accessor_link_still_current(recv, stamp, c) - { + if stamp == 0 || c[HOLDER_RECV] != (u64::from(stamp) | PIC_ID_TOKEN_BIT) as i64 { return None; } let holder = c[HOLDER_OBJ] as usize; - if shape_word(holder) != c[HOLDER_SHAPE] as u32 { + if shape_word(holder) != c[HOLDER_SHAPE] as u32 + || slot_bits(holder, kind as u32) != crate::value::POINTER_TAG | c[HOLDER_HOPS] as u64 + { return None; } - // The holder's unchanged ShapeId carries the prime-time proof that this - // inline slot exists and is an accessor. Key/attribute changes transition - // the ShapeId; a raw-only pair replacement may not, so reread the pair - // itself on every hit before invoking. - let slot = c[HOLDER_KIND] as u32; - let raw_get = - crate::object::accessor_pair::raw_instance_getter_of_value(slot_bits(holder, slot))?; HITS_ACCESSOR.fetch_add(1, Ordering::Relaxed); - super::stats_report_enabled(); - Some(invoke_class_getter(recv, raw_get)) + Some(invoke_class_getter(recv, c[HOLDER_HOP_SHAPES] as usize)) } unsafe fn walk(recv: *const ObjectHeader, name: &[u8], class_first: bool) -> Option { @@ -571,6 +562,7 @@ unsafe fn walk(recv: *const ObjectHeader, name: &[u8], class_first: bool) -> Opt slot: None, hops: [(0, 0); HOLDER_MAX_DEPTH - 1], depth: 0, + raw_get: 0, }; let object_prototype = crate::array::object_prototype_addr_if_resolved(); let mut current = recv; @@ -693,12 +685,15 @@ pub(crate) unsafe fn prime_read_holder( if let Some(acc) = class_accessor_walk(recv, name) { let cache = crate::object::field_get_set::pic_slot_resolve::(cache_slot); if !cache.is_null() { + let mut hops = [(0, 0); HOLDER_MAX_DEPTH - 1]; + hops[0].0 = acc.pair; let w = Walk { holder: acc.holder, holder_shape: acc.shape, slot: Some(acc.slot), - hops: [(0, 0); HOLDER_MAX_DEPTH - 1], + hops, depth: 1, + raw_get: acc.raw_get, }; publish(cache, recv, &w, true); } @@ -866,7 +861,7 @@ unsafe fn publish(cache: *mut PicCache, recv: *const ObjectHeader, w: &Walk, acc c[HOLDER_HOPS + i] = w.hops[i].0 as i64; } c[HOLDER_HOP_SHAPES] = if accessor { - crate::object::class_lookup_surface_generation() as i64 + w.raw_get as i64 } else { (u64::from(w.hops[0].1) | u64::from(w.hops[1].1) << 32) as i64 }; @@ -937,10 +932,25 @@ mod tests { 8.0 } + /// The entry a class accessor prime publishes for `acc`. + fn accessor_entry(acc: &ClassAccessor) -> Walk { + let mut hops = [(0, 0); HOLDER_MAX_DEPTH - 1]; + hops[0].0 = acc.pair; + Walk { + holder: acc.holder, + holder_shape: acc.shape, + slot: Some(acc.slot), + hops, + depth: 1, + raw_get: acc.raw_get, + } + } + /// Two holders with exactly one ShapeId but different compiled getters. /// Replacing a declared class's registry pointer leaves the receiver's - /// bare CLASS ShapeId unchanged; the collecting hit must compare the live - /// link, rather than trust receiver and holder shapes alone. + /// bare CLASS ShapeId unchanged. The replacement must retire the old + /// holder's ShapeId, so the hit's holder compare refuses the stale entry + /// with no global word to consult. #[test] fn class_accessor_rechecks_same_shape_holder_link() { if !crate::object::method_site::run_with_fresh_worker_gate( @@ -1009,19 +1019,12 @@ mod tests { let cache: &'static mut PicCache = Box::leak(Box::new([0; crate::codegen_abi::PIC_CACHE_WORDS])); let mut slot: PicCacheSlot = cache; - let w = Walk { - holder: first.holder, - holder_shape: first.shape, - slot: Some(first.slot), - hops: [(0, 0); HOLDER_MAX_DEPTH - 1], - depth: 1, - }; - unsafe { publish(cache, receiver, &w, true) }; - let first_generation = cache[HOLDER_HOP_SHAPES]; + unsafe { publish(cache, receiver, &accessor_entry(&first), true) }; assert_eq!( - first_generation as u64, - crate::object::class_lookup_surface_generation() + cache[HOLDER_HOP_SHAPES] as usize, + getter_two as *const () as usize ); + assert_eq!(cache[HOLDER_HOPS] as usize, first.pair); assert_eq!( unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), Some(2.0) @@ -1031,25 +1034,19 @@ mod tests { p2.with_const_ptr::(|ptr| { crate::object::test_seed_class_decl_prototype_object_root(CID, ptr as usize); }); - assert_ne!( - cache[HOLDER_HOP_SHAPES] as u64, - crate::object::class_lookup_surface_generation() - ); assert_eq!(unsafe { object_shape_stamp(receiver) }, recv_shape); + // The relink is seen through the old holder's ShapeId alone. + p1.with_const_ptr::(|old| { + assert_ne!(unsafe { object_shape_stamp(old) }, first.shape); + assert_ne!(unsafe { object_shape_stamp(old) }, 0); + }); assert!( unsafe { try_cached_class_accessor(receiver, &mut slot) }.is_none(), "stale getter was served after registry replacement" ); let second = unsafe { class_accessor_walk(receiver, b"path") }.expect("second accessor"); - let w = Walk { - holder: second.holder, - holder_shape: second.shape, - slot: Some(second.slot), - hops: [(0, 0); HOLDER_MAX_DEPTH - 1], - depth: 1, - }; - unsafe { publish(cache, receiver, &w, true) }; + unsafe { publish(cache, receiver, &accessor_entry(&second), true) }; assert!(read_accessor_same_shape_relinks() > old_relinks); assert_eq!( unsafe { try_cached_class_accessor(receiver, &mut slot) }.map(|v| v.as_number()), @@ -1120,6 +1117,7 @@ mod tests { slot: None, hops: [(0, 0); HOLDER_MAX_DEPTH - 1], depth: 1, + raw_get: 0, }; for i in 0..11 { recv.parent_class_id = base + i; diff --git a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs index 03e02ab227..4f9c68236f 100644 --- a/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs +++ b/crates/perry-runtime/src/object/method_site/read_holder/class_read.rs @@ -381,8 +381,20 @@ mod tests { } crate::object::class_decl_prototype_object_root_store(CID, b); assert_eq!(unsafe { answer(&entry, recv) }, None); + // The displaced holder's ShapeId was retired: an entry naming it can + // never answer again, whatever the registry says later. + assert_ne!(unsafe { object_shape_stamp(a) }, proto_shape); crate::object::class_decl_prototype_object_root_store(CID, a); + assert_eq!(unsafe { answer(&entry, recv) }, None); + let entry = Entry { + holder_shape: unsafe { object_shape_stamp(a) }, + ..entry + }; + assert_eq!( + unsafe { answer(&entry, recv) }, + Some(crate::value::TAG_UNDEFINED) + ); let record = Box::into_raw(Box::new(Site { entries: [entry; WAYS], next: 0, diff --git a/crates/perry-runtime/src/proxy/put_value/packed_set.rs b/crates/perry-runtime/src/proxy/put_value/packed_set.rs index 543d136bf5..377661312e 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_set.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_set.rs @@ -145,6 +145,13 @@ pub extern "C" fn js_put_value_set_packed_miss( packed: *const AtomicU64, ) -> f64 { let site = packed as *const super::packed_add::PackedSetSite; + // The site's compiled-setter entry: a store whose key the receiver + // inherits as a class accessor misses the emitted ways by construction. + // Its hit is two ShapeId compares and one lane load (`setter_site`), so it + // is asked before any other miss work re-derives what it already proves. + if let Some(stored) = unsafe { setter_site::try_hit(cache_slot, target, key, value) } { + return stored; + } // Charter step 5: migrate a receiver whose shape the lineage generalized // before the key-add memo or a way is keyed by it. let target_bits = target.to_bits(); diff --git a/crates/perry-runtime/src/proxy/put_value/setter_site.rs b/crates/perry-runtime/src/proxy/put_value/setter_site.rs index 549b00c0ca..90e5a34b0f 100644 --- a/crates/perry-runtime/src/proxy/put_value/setter_site.rs +++ b/crates/perry-runtime/src/proxy/put_value/setter_site.rs @@ -2,28 +2,51 @@ //! //! The packed store's first eight words are own-data ways and word eight is //! the key-add chain verdict. Word nine names this bounded, collecting-path -//! setter entry. The emitted leaf never reads it. A miss validates the live -//! class-prototype link, receiver and holder shapes, inline accessor slot and -//! raw setter before calling with the original receiver. Any uncertainty -//! falls through to ordinary `[[Set]]`. +//! setter entry. The emitted leaf never reads it. +//! +//! A hit is shape facts and the lane's own value, nothing global: +//! * the receiver's ShapeId proves the key is not own and names the +//! receiver's prototype identity, hence the holder (a recorded serial, or a +//! bare class whose registry link retires the holder's ShapeId if it is ever +//! replaced: `class_registry::retire_displaced_decl_prototype`); +//! * the holder's ShapeId proves the key's slot is still an accessor lane; +//! * the lane still holds the primed pair, which names the compiled setter. +//! +//! Any uncertainty falls through to ordinary `[[Set]]`, which re-primes. use super::*; use std::sync::atomic::{AtomicU64, Ordering}; -const SITE_TAG: u64 = 0xA2C2_0000_0000_0000; +const SITE_TAG: u64 = crate::codegen_abi::SETTER_SITE_TAG; +const _: () = assert!(PACKED_SET_SETTER_WORD == crate::codegen_abi::PACKED_SET_SETTER_WORD); +const _: () = assert!(crate::value::POINTER_MASK == crate::codegen_abi::SETTER_SITE_ADDRESS_MASK); +/// The site's entry. The emitted store tower reads it at the offsets +/// `perry_abi::SETTER_SITE_*_OFFSET` pin (`setter_arm.rs` in codegen). +#[repr(C)] struct Entry { - key: usize, - holder: usize, - class_id: u32, receiver_shape: u32, - bare_class_link: bool, holder_shape: u32, + /// The holder (a strong root, rewritten on move). + holder: usize, slot: u32, + /// The pair the holder's lane held at prime time: its raw address, a + /// strong root rewritten on move, so a hit compares one loaded word. + pair: usize, + /// The compiled setter the pair names. raw_set: usize, - validity: u64, - vtable_gen: u64, + /// The interned key (a strong root). + key: usize, } +const _: () = { + use crate::codegen_abi as abi; + assert!(std::mem::offset_of!(Entry, receiver_shape) == abi::SETTER_SITE_RECV_SHAPE_OFFSET); + assert!(std::mem::offset_of!(Entry, holder_shape) == abi::SETTER_SITE_HOLDER_SHAPE_OFFSET); + assert!(std::mem::offset_of!(Entry, holder) == abi::SETTER_SITE_HOLDER_OFFSET); + assert!(std::mem::offset_of!(Entry, slot) == abi::SETTER_SITE_SLOT_OFFSET); + assert!(std::mem::offset_of!(Entry, pair) == abi::SETTER_SITE_PAIR_OFFSET); + assert!(std::mem::offset_of!(Entry, raw_set) == abi::SETTER_SITE_CODE_OFFSET); +}; crate::perry_thread_local! { static ENTRIES: std::cell::UnsafeCell> = @@ -171,9 +194,6 @@ unsafe fn candidate( return None; } - let bare_class_link = (crate::object::shapes::PROTO_ID_CLASS - ..crate::object::shapes::PROTO_ID_MIXED) - .contains(&recv_shape.proto_id); let holder = class_link(recv)?; let holder_gc = crate::value::addr_class::try_read_gc_header(holder as usize)?; if holder_gc.obj_type != crate::gc::GC_TYPE_OBJECT @@ -199,10 +219,8 @@ unsafe fn candidate( { return None; } - let field = (holder as *const u8) - .add(std::mem::size_of::() + slot as usize * 8) - as *const u64; - let acc = crate::object::accessor_pair::pair_of_value(*field)?; + let lane = lane_bits(holder as usize, slot); + let acc = crate::object::accessor_pair::pair_of_value(lane)?; if acc.raw_set == 0 { return None; } @@ -215,104 +233,105 @@ unsafe fn candidate( Some(Entry { key: key as usize, holder: holder as usize, - class_id, receiver_shape: crate::object::shapes::object_shape_stamp(recv), - bare_class_link, holder_shape: crate::object::shapes::object_shape_stamp(holder), slot, + pair: (lane & crate::value::POINTER_MASK) as usize, raw_set: acc.raw_set, - validity: crate::object::proto_validity::proto_validity(), - vtable_gen: crate::object::vtable_generation(), }) } +/// The value of `holder`'s inline slot `slot`. +#[inline] +unsafe fn lane_bits(holder: usize, slot: u32) -> u64 { + std::ptr::read( + (holder as *const u8).add(std::mem::size_of::() + slot as usize * 8) + as *const u64, + ) +} + +/// The compiled setter `e` names for `recv`, when every fact still holds: the +/// receiver's ShapeId (key not own, prototype identity, hence the holder), +/// the holder's ShapeId (the slot is an accessor lane) and the lane's pair. +/// A ShapeId match also proves a live, non-forwarded ordinary object (#10828 +/// rule 3), so nothing per-object is re-read. +#[inline] unsafe fn validated_raw_set( e: &Entry, recv: *const crate::ObjectHeader, key: *const crate::StringHeader, ) -> Option { - let gc = crate::value::addr_class::try_read_gc_header(recv as usize)?; - if gc.obj_type != crate::gc::GC_TYPE_OBJECT - || gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 - || gc._reserved & crate::gc::OBJ_FLAG_TYPED_ARRAY_PROTO != 0 - || crate::object::dictionary::is_dictionary(recv) - { - return None; - } - let meta = (*recv).meta; - if !meta.is_null() - && ((*meta).elements != 0 - || (*meta).flags & crate::object::OBJECT_META_FLAG_EXOTIC_READ_RECEIVER != 0) - { - return None; - } - if e.key != key as usize - || e.class_id != (*recv).class_id - || e.receiver_shape != crate::object::shapes::object_shape_stamp(recv) - || e.validity != crate::object::proto_validity::proto_validity() - || e.vtable_gen != crate::object::vtable_generation() - || crate::object::shapes::object_shape_stamp(e.holder as *const crate::ObjectHeader) - != e.holder_shape - { - return None; - } - if e.bare_class_link { - // ShapeId proves the class-link mode. Every declared-prototype root - // replacement (including generic-origin redirects) bumps the validity - // word checked above. GC moves both the registry root and this rooted - // holder entry together. A per-instance prototype change must either - // restamp the ShapeId or leave an explicit meta link, rejected here. - if gc._reserved & crate::gc::OBJ_FLAG_NULL_PROTO != 0 - || (!meta.is_null() && (*meta).prototype != 0) - || e.holder == recv as usize - { - return None; - } - } else if class_link(recv)? as usize != e.holder { - // MIXED receivers keep the full live per-object link comparison. - return None; - } - let holder_gc = crate::value::addr_class::try_read_gc_header(e.holder)?; - if holder_gc.obj_type != crate::gc::GC_TYPE_OBJECT - || holder_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 - { - return None; - } - let field = (e.holder as *const u8) - .add(std::mem::size_of::() + e.slot as usize * 8) - as *const u64; - let acc = crate::object::accessor_pair::pair_of_value(*field)?; - (acc.raw_set == e.raw_set && acc.raw_set != 0).then_some(acc.raw_set) + let stamp = crate::object::shapes::object_shape_stamp(recv); + (stamp != 0 + && e.receiver_shape == stamp + && e.key == key as usize + && crate::object::shapes::object_shape_stamp(e.holder as *const crate::ObjectHeader) + == e.holder_shape + && lane_bits(e.holder, e.slot) == crate::value::POINTER_TAG | e.pair as u64) + .then_some(e.raw_set) } +/// Call the compiled setter with `target` as `this`. The store's result is +/// `value`; a Number needs no root across the call, anything else is rooted +/// (a heap value can move while the setter runs). +#[inline] unsafe fn invoke(raw_set: usize, target: f64, value: f64) -> f64 { + let f = crate::closure::body_call::js_method_body_fn!(raw_set as *const u8; value); + if crate::value::JSValue::from_bits(value.to_bits()).is_number() { + let _ = f(target, value); + return value; + } let scope = crate::gc::RuntimeHandleScope::new(); - let recv_h = scope.root_nanbox_f64(target); let value_h = scope.root_nanbox_f64(value); - let f = crate::closure::body_call::js_method_body_fn!(raw_set as *const u8; value); - let _ = f(recv_h.get_nanbox_f64(), value_h.get_nanbox_f64()); + let _ = f(target, value); value_h.get_nanbox_f64() } -/// Collecting miss only; the emitted GC-leaf store never consults this word. -pub(super) unsafe fn try_set( +/// The receiver `target` names, when it is a heap pointer the entry could +/// describe. +#[inline] +fn receiver_of(target: f64) -> Option<*const crate::ObjectHeader> { + let bits = target.to_bits(); + if bits & !crate::value::POINTER_MASK != crate::value::POINTER_TAG { + return None; + } + let addr = (bits & crate::value::POINTER_MASK) as usize; + crate::value::addr_class::is_above_handle_band(addr) + .then_some(addr as *const crate::ObjectHeader) +} + +/// The entry's hit and nothing else: allocation-free until the setter runs, +/// and never a candidate walk, so the store-miss entry asks it before any +/// other miss work. +#[inline] +pub(super) unsafe fn try_hit( slot: *mut PackedSetWaysSlot, target: f64, key: *const crate::StringHeader, value: f64, ) -> Option { - if !primary_only() || slot.is_null() || key.is_null() { + let e = entry(slot)?; + if crate::object::method_site::WORKER_AGENTS_EXIST.load(Ordering::SeqCst) != 0 { return None; } - let bits = target.to_bits(); - if bits & !crate::value::POINTER_MASK != crate::value::POINTER_TAG { - return None; + let raw_set = validated_raw_set(e, receiver_of(target)?, key)?; + if stats_enabled() { + HITS.fetch_add(1, Ordering::Relaxed); } - let addr = (bits & crate::value::POINTER_MASK) as usize; - if !crate::value::addr_class::is_above_handle_band(addr) { + Some(invoke(raw_set, target, value)) +} + +/// Collecting miss only; the emitted GC-leaf store never consults this word. +pub(super) unsafe fn try_set( + slot: *mut PackedSetWaysSlot, + target: f64, + key: *const crate::StringHeader, + value: f64, +) -> Option { + if !primary_only() || slot.is_null() || key.is_null() { return None; } - let recv = addr as *const crate::ObjectHeader; + let recv = receiver_of(target)?; if let Some(e) = entry(slot) { if let Some(raw_set) = validated_raw_set(e, recv, key) { if stats_enabled() { @@ -354,6 +373,9 @@ pub(crate) fn scan_roots(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { if visitor.visit_tagged_usize_slot(&mut e.holder, crate::value::POINTER_TAG) { ROOT_REWRITES.fetch_add(1, Ordering::Relaxed); } + if visitor.visit_tagged_usize_slot(&mut e.pair, crate::value::POINTER_TAG) { + ROOT_REWRITES.fetch_add(1, Ordering::Relaxed); + } } }); } @@ -480,7 +502,10 @@ mod tests { } assert_eq!(call_set!(5.0), Some(5.0)); assert_eq!(call_set!(6.0), Some(6.0)); - assert!(unsafe { entry(&mut slot).unwrap().bare_class_link }); + assert_eq!( + unsafe { entry(&mut slot).unwrap().raw_set }, + first as *const () as usize + ); assert_eq!(FIRST.load(Ordering::Relaxed), 2); p2.with_const_ptr::(|p| { crate::object::test_seed_class_decl_prototype_object_root(CID, p as usize) diff --git a/test-files/test_gap_class_accessor_shape_facts.ts b/test-files/test_gap_class_accessor_shape_facts.ts new file mode 100644 index 0000000000..a2e72d05b9 --- /dev/null +++ b/test-files/test_gap_class_accessor_shape_facts.ts @@ -0,0 +1,213 @@ +// #10498: class getters/setters answered from shape facts at the read and +// store sites (receiver ShapeId -> holder, holder ShapeId -> accessor lane, +// the lane's pair -> the compiled getter/setter). Every scenario runs a site +// hot first, then changes one fact the site relies on, and keeps reading. + +class Point { + _x = 0; + _y = 0; + constructor(x: number, y: number) { + this.x = x; + this.y = y; + } + get x() { return this._x; } + set x(v: number) { this._x = v; } + get y() { return this._y; } + set y(v: number) { this._y = v * 2; } +} + +function sumXY(ps: any[]): number { + let s = 0; + for (let i = 0; i < ps.length; i++) s += ps[i].x + ps[i].y; + return s; +} +function writeX(p: any, n: number): number { + let s = 0; + for (let i = 0; i < n; i++) { p.x = i; s += p._x; } + return s; +} + +const pts: Point[] = []; +for (let i = 0; i < 40; i++) pts.push(new Point(i, i + 1)); +console.log("basic", sumXY(pts), writeX(pts[3], 50), pts[3].x, pts[3].y); + +// Inherited through two levels, and overridden in a subclass. +class A { + _v = 1; + get v() { return this._v * 10; } + set v(n: number) { this._v = n; } +} +class B extends A { tag = "b"; } +class C extends B { more = 3; } +class D extends B { + get v() { return -this._v; } + set v(n: number) { this._v = n + 100; } +} +function readV(o: any): number { return o.v; } +function writeV(o: any, n: number): void { o.v = n; } +const c = new C(); +const d = new D(); +let acc = 0; +for (let i = 0; i < 60; i++) { + writeV(c, i); + acc += readV(c); +} +console.log("two-levels", acc, c.v, (c as any)._v); +acc = 0; +for (let i = 0; i < 60; i++) { + const o: any = i % 3 === 0 ? d : c; + writeV(o, i); + acc += readV(o); +} +console.log("override", acc, d.v, (d as any)._v, c.v); + +// Redefined at runtime on the prototype: hot sites must see the new pair. +class R { + _n = 5; + get n() { return this._n; } + set n(v: number) { this._n = v; } +} +const r = new R(); +function readN(o: any): number { return o.n; } +function writeN(o: any, v: number): void { o.n = v; } +let before = 0; +for (let i = 0; i < 50; i++) { writeN(r, i); before += readN(r); } +Object.defineProperty(R.prototype, "n", { + get() { return this._n + 1000; }, + set(v: number) { this._n = v * 3; }, + configurable: true, +}); +let after = 0; +for (let i = 0; i < 50; i++) { writeN(r, i); after += readN(r); } +console.log("redefine", before, after, r.n, (r as any)._n); +// Getter-only replacement: the store has no setter now (sloppy: ignored). +Object.defineProperty(R.prototype, "n", { get() { return 7; }, configurable: true }); +writeN(r, 99); +console.log("getter-only-redefine", readN(r), (r as any)._n); +// Back to a data property on the prototype. +Object.defineProperty(R.prototype, "n", { value: 42, writable: true, configurable: true }); +console.log("data-redefine", readN(r), Object.prototype.hasOwnProperty.call(r, "n")); +writeN(r, 8); +console.log("own-after-data", readN(r), Object.prototype.hasOwnProperty.call(r, "n")); +// Deleted from the prototype: reads fall to undefined. +class Del { get g() { return 1; } } +const del = new Del(); +function readG(o: any): any { return o.g; } +let gs = 0; +for (let i = 0; i < 30; i++) gs += readG(del); +delete (Del.prototype as any).g; +console.log("delete", gs, readG(del)); + +// An own property shadowing the accessor on one instance. +const sh1 = new Point(1, 2); +const sh2 = new Point(3, 4); +Object.defineProperty(sh2, "x", { value: 77, writable: true }); +let shs = 0; +for (let i = 0; i < 30; i++) shs += (i % 2 ? sh1 : sh2).x; +console.log("shadow", shs); + +// setPrototypeOf on an instance moves it off the class. +const moved = new Point(5, 6); +let ms = 0; +for (let i = 0; i < 20; i++) ms += readX(moved); +Object.setPrototypeOf(moved, { get x() { return -1; } }); +for (let i = 0; i < 20; i++) ms += readX(moved); +function readX(o: any): number { return o.x; } +console.log("setPrototypeOf", ms); + +// Static accessors. +class S { + static _count = 0; + static get count() { return S._count; } + static set count(v: number) { S._count = v + 1; } +} +let ss = 0; +for (let i = 0; i < 30; i++) { S.count = i; ss += S.count; } +console.log("static", ss, S.count); + +// Accessors on object literals, several objects of one shape. +function mk(k: number) { + return { + _k: k, + get k2() { return this._k * 2; }, + set k2(v: number) { this._k = v; }, + }; +} +const lits = [mk(1), mk(2), mk(3)]; +let ls = 0; +for (let i = 0; i < 30; i++) { + const o: any = lits[i % 3]; + ls += o.k2; + o.k2 = i; +} +console.log("literal", ls, lits.map((o) => o.k2).join(",")); + +// Setter-only and getter-only class accessors. +class OnlyOne { + _w = 0; + set w(v: number) { this._w = v; } + get r() { return this._w + 1; } +} +const oo: any = new OnlyOne(); +let os = 0; +let refused = 0; +for (let i = 0; i < 30; i++) { + oo.w = i; + os += oo.r; + os += oo.w === undefined ? 1 : 0; + // Module code is strict: a store to a getter-only accessor throws. + try { oo.r = 5; } catch (e) { refused += (e as Error) instanceof TypeError ? 1 : 0; } +} +console.log("one-sided", os, refused, oo._w, oo.r); + +// Non-Number values through a setter (strings and objects) and back. +class Box { + _v: any = null; + get v() { return this._v; } + set v(x: any) { this._v = x; } +} +const bx: any = new Box(); +const kept: any[] = []; +for (let i = 0; i < 40; i++) { + bx.v = i % 2 ? "s" + i : { i }; + kept.push(bx.v); +} +const lastAssign = (bx.v = { done: true }); +console.log("values", kept.length, JSON.stringify(kept[38]), kept[39], JSON.stringify(lastAssign), JSON.stringify(bx.v)); + +// A throwing getter and setter, from hot sites inside try/catch. +class T { + _t = 0; + get t() { if (this._t > 25) throw new Error("get " + this._t); return this._t; } + set t(v: number) { if (v === 30) throw new Error("set " + v); this._t = v; } +} +const tt: any = new T(); +let caught = ""; +let ts = 0; +for (let i = 0; i < 35; i++) { + try { tt.t = i; ts += tt.t; } catch (e) { caught += (e as Error).message + ";"; } +} +console.log("throws", ts, caught); + +// The getter sees the original receiver as `this`. +class Self { get me() { return this; } } +class Sub2 extends Self { id = 9; } +const s2 = new Sub2(); +let same = 0; +for (let i = 0; i < 20; i++) if ((s2 as any).me === s2) same++; +console.log("this", same, (s2 as any).me.id); + +// Values allocated inside getters, many collections while sites are hot. +class Alloc { + n = 0; + get fresh() { return { n: this.n++, pad: [1, 2, 3, 4, 5, 6, 7, 8] }; } + set slot(v: any) { this.n += v.pad.length; } +} +const al: any = new Alloc(); +let as = 0; +for (let i = 0; i < 20000; i++) { + const f = al.fresh; + as += f.pad[i & 7]; + al.slot = f; +} +console.log("alloc", as, al.n);