perf: class accessors validated by shape facts, called directly (#10498)

changelog.d/11784-accessor-shape-facts.md

@@ -0,0 +1,13 @@
+Class getters and setters are answered from shape facts at the read and
+store sites (#10498). A site that inherits a compiled class accessor checks
+the receiver's ShapeId (the key is not own, and the prototype identity names
+the holder), the holder's ShapeId (the key is still an accessor lane) and the
+lane's value against the accessor pair it primed, then calls the compiled
+getter or setter directly: inline in the emitted read and store towers, and
+first thing in the collecting miss entries. The class-registry link from a
+class to its declared prototype is written once; a replacement or a
+generic-origin redirect retires the displaced prototype's ShapeId, so the
+accessor read and setter sites no longer compare the global
+`class_lookup_surface_generation`, `proto_validity` or `vtable_generation`
+words. getter_read2 1,540 -> 223 instructions per iteration, setter_write2
+1,207 -> 313, setter_ctor 3.6x -> 1.5x field_ctor.

crates/perry-abi/src/lib.rs

@@ -538,6 +538,36 @@ pub const PIC_HOLDER_RECV_WORD: usize = 12;
 pub const PIC_HOLDER_OBJ_WORD: usize = 13;
 pub const PIC_HOLDER_SHAPE_WORD: usize = 14;
 pub const PIC_HOLDER_KIND_WORD: usize = 15;
+/// A class-accessor entry (#10498): its kind word carries
+/// [`PIC_HOLDER_ACCESSOR_BIT`] over the holder's inline slot (low 32 bits);
+/// [`PIC_HOLDER_PAIR_WORD`] holds the raw address of the accessor pair that
+/// slot held when the site primed (a strong root the collector rewrites), and
+/// [`PIC_HOLDER_GETTER_WORD`] the compiled getter that pair names
+/// (`double get(double this)`; 0 for a setter-only pair). A hit is the
+/// receiver token, the holder's ShapeId and the slot's value equal to the
+/// pair: then the getter is called with the receiver as `this`.
+pub const PIC_HOLDER_ACCESSOR_BIT: i64 = 1 << 61;
+pub const PIC_HOLDER_PAIR_WORD: usize = 16;
+pub const PIC_HOLDER_GETTER_WORD: usize = 19;
+
+/// `proxy::put_value::setter_site` (#10498): the word of a static-key store
+/// site's ways cache that names the site's compiled-setter entry, as
+/// [`SETTER_SITE_TAG`] over the entry's address ([`SETTER_SITE_ADDRESS_MASK`]).
+/// The entry is a `#[repr(C)]` record the emitted store tower reads
+/// (`perry-codegen/src/expr/put_value_store_ic/setter_arm.rs`): the receiver
+/// ShapeId and the holder ShapeId (u32 each), the holder's raw address, the
+/// holder's inline slot (u32), the raw address of the accessor pair that slot
+/// held at prime time, and the compiled setter it names
+/// (`double set(double this, double v)`).
+pub const PACKED_SET_SETTER_WORD: usize = 9;
+pub const SETTER_SITE_TAG: u64 = 0xA2C2_0000_0000_0000;
+pub const SETTER_SITE_ADDRESS_MASK: u64 = 0x0000_FFFF_FFFF_FFFF;
+pub const SETTER_SITE_RECV_SHAPE_OFFSET: usize = 0;
+pub const SETTER_SITE_HOLDER_SHAPE_OFFSET: usize = 4;
+pub const SETTER_SITE_HOLDER_OFFSET: usize = 8;
+pub const SETTER_SITE_SLOT_OFFSET: usize = 16;
+pub const SETTER_SITE_PAIR_OFFSET: usize = 24;
+pub const SETTER_SITE_CODE_OFFSET: usize = 32;
 /// The site's holder state word, and its bit for a LATCHED site: one that
 /// refused, or whose non-own receivers took several shapes. Its misses ask the
 /// inherited-read hook, as a never-primed site's do.

crates/perry-codegen/src/expr/body_call.rs

@@ -107,6 +107,30 @@ pub(crate) fn emit_js_body_call_gc_leaf(
     blk.call_indirect_gc_leaf(DOUBLE, code_ptr, &native)
 }
 
+/// Call a compiled class INSTANCE getter through its code address (a site's
+/// class-accessor entry, `perry_abi::PIC_HOLDER_GETTER_WORD`): a method body
+/// `double get(double this)`, the convention the runtime calls the same entry
+/// with (`perry-runtime/src/closure/body_call.rs`, `js_method_body_fn!`).
+/// `code_ptr` is a `ptr`, `this_box` the NaN-boxed receiver as a `double`.
+/// The getter can run any JS: this is a collecting, possibly throwing call.
+pub(crate) fn emit_class_getter_call(blk: &mut LlBlock, code_ptr: &str, this_box: &str) -> String {
+    blk.call_indirect(DOUBLE, code_ptr, &[(DOUBLE, this_box)])
+}
+
+/// Call a compiled class INSTANCE setter through its code address (a store
+/// site's compiled-setter entry, `perry_abi::SETTER_SITE_CODE_OFFSET`): a
+/// method body `double set(double this, double v)`, the convention the runtime
+/// calls the same entry with. Its result is ignored: the assignment's value
+/// is `v`. A collecting, possibly throwing call.
+pub(crate) fn emit_class_setter_call(
+    blk: &mut LlBlock,
+    code_ptr: &str,
+    this_box: &str,
+    value: &str,
+) -> String {
+    blk.call_indirect(DOUBLE, code_ptr, &[(DOUBLE, this_box), (DOUBLE, value)])
+}
+
 /// The receiver bits for a call whose callee binds `this` to `undefined`
 /// (or whose callee is an arrow and never reads it).
 pub(crate) const JS_THIS_UNDEFINED: &str = crate::nanbox::TAG_UNDEFINED_I64;

crates/perry-codegen/src/expr/property_get.rs

@@ -44,6 +44,7 @@ use super::property_get_names::{
     is_net_native_method_value, is_url_pattern_data_property,
 };
 
+mod accessor_arm;
 pub(crate) mod generic_dispatch;
 pub(crate) mod globalget;
 mod helpers;

crates/perry-codegen/src/expr/property_get/accessor_arm.rs

@@ -0,0 +1,165 @@
+//! The read site's class-accessor arm (#10498): `recv.k` where `k` is a
+//! compiled class getter the receiver inherits, answered by two ShapeId
+//! compares, one lane load and a direct call.
+//!
+//! The runtime primes the entry in the site's own cache
+//! (`perry-runtime/src/object/method_site/read_holder.rs`, kind
+//! `PIC_HOLDER_ACCESSOR_BIT`). Every fact the hit uses is a shape fact or the
+//! lane's own value:
+//!
+//! * the receiver's ShapeId (the token) proves `k` is not own and names the
+//!   receiver's prototype identity, hence the holder: a recorded serial, or a
+//!   bare class whose registry link retires the holder's ShapeId if it is
+//!   ever replaced (`class_registry::retire_displaced_decl_prototype`);
+//! * the holder's ShapeId proves `k`'s slot is still an accessor lane;
+//! * the lane still holds the primed pair, which names the compiled getter.
+//!
+//! Emitted on the MRU compare's false edge, ahead of the GC-leaf front:
+//!
+//! ```text
+//!   packed == PACKED_GET_EMPTY                     else FRONT
+//!   c = @site ; c != null                          else FRONT
+//!   (u32)c[RECV] == [recv+4]                       else FRONT
+//!   c[KIND] & ACCESSOR                             else FRONT
+//!   [c[OBJ]+4] == (u32)c[SHAPE]                    else FRONT
+//!   [c[OBJ] + HDR + 8*(u32)c[KIND]] == POINTER_TAG | c[PAIR]
+//!   && no worker && c[GETTER] != 0                 else FRONT
+//!   r = c[GETTER](recv)
+//! ```
+//!
+//! Only a site whose MRU word was never primed takes the arm: a site that
+//! also reads own data keeps its misses on the front, which declines the
+//! accessor kind, and the collecting slow call asks the same entry first. A
+//! worker's start (`PERRY_METHOD_SITE_WORKERS_PRESENT`) sends every read to
+//! the front: holder entries belong to the primary heap.
+
+use super::super::FnCtx;
+use crate::runtime_abi as abi;
+use crate::types::{I1, I32, I64, I8, PTR};
+
+/// Emit the arm starting at `entry_idx` (a fresh block the MRU compare's false
+/// edge targets). Every failing test branches to `miss_label` (the front); the
+/// call's result reaches `merge_label`. Returns `(value, end label)` for the
+/// tower's merge phi.
+#[allow(clippy::too_many_arguments)]
+pub(super) fn emit_class_accessor_arm(
+    ctx: &mut FnCtx<'_>,
+    entry_idx: usize,
+    packed_word: &str,
+    packed_empty: i64,
+    cache_slot_ref: &str,
+    recv_biased: &str,
+    recv_box: &str,
+    header_bytes: i64,
+    miss_label: &str,
+    merge_label: &str,
+) -> (String, String) {
+    let cache_idx = ctx.new_block("pic.acc.cache");
+    let recv_idx = ctx.new_block("pic.acc.recv");
+    let kind_idx = ctx.new_block("pic.acc.kind");
+    let holder_idx = ctx.new_block("pic.acc.holder");
+    let lane_idx = ctx.new_block("pic.acc.lane");
+    let call_idx = ctx.new_block("pic.acc.call");
+    let cache_l = ctx.block_label(cache_idx);
+    let recv_l = ctx.block_label(recv_idx);
+    let kind_l = ctx.block_label(kind_idx);
+    let holder_l = ctx.block_label(holder_idx);
+    let lane_l = ctx.block_label(lane_idx);
+    let call_l = ctx.block_label(call_idx);
+
+    // A site that reads own data has a primed MRU word; its accessor reads
+    // stay on the front and the slow call.
+    ctx.current_block = entry_idx;
+    {
+        let blk = ctx.block();
+        let empty = blk.icmp_eq(I64, packed_word, &packed_empty.to_string());
+        blk.cond_br(&empty, &cache_l, miss_label);
+    }
+
+    // The full cache is allocated lazily; it is published with release order.
+    ctx.current_block = cache_idx;
+    let cache = {
+        let blk = ctx.block();
+        let cache = blk.load_atomic_acquire(PTR, cache_slot_ref, 8);
+        let present = blk.icmp_ne(PTR, &cache, "null");
+        blk.cond_br(&present, &recv_l, miss_label);
+        cache
+    };
+    let word = |ctx: &mut FnCtx<'_>, index: usize| -> String {
+        let blk = ctx.block();
+        let p = blk.gep(I64, &cache, &[(I64, &index.to_string())]);
+        blk.load(I64, &p)
+    };
+
+    // The receiver token against the receiver's ShapeId, re-read here so the
+    // hot compare's load keeps its single use. A token is
+    // `PIC_ID_TOKEN_BIT | ShapeId` and an empty entry is 0, so its low half
+    // is a ShapeId or 0; a receiver word equal to a ShapeId proves a live
+    // ordinary object of that shape (#10828 rule 3), and 0 is never one.
+    ctx.current_block = recv_idx;
+    let recv_word = word(ctx, abi::PIC_HOLDER_RECV_WORD);
+    {
+        let blk = ctx.block();
+        let sid_ptr = crate::expr::receiver_range::emit_field_ptr(blk, recv_biased, 4);
+        let sid = blk.load(I32, &sid_ptr);
+        let primed = blk.trunc(I64, &recv_word, I32);
+        let same = blk.icmp_eq(I32, &sid, &primed);
+        blk.cond_br(&same, &kind_l, miss_label);
+    }
+
+    ctx.current_block = kind_idx;
+    let kind = word(ctx, abi::PIC_HOLDER_KIND_WORD);
+    {
+        let blk = ctx.block();
+        let bit = blk.and(I64, &kind, &abi::PIC_HOLDER_ACCESSOR_BIT.to_string());
+        let accessor = blk.icmp_ne(I64, &bit, "0");
+        blk.cond_br(&accessor, &holder_l, miss_label);
+    }
+
+    // The holder's ShapeId against the primed one.
+    ctx.current_block = holder_idx;
+    let holder = word(ctx, abi::PIC_HOLDER_OBJ_WORD);
+    let holder_shape = word(ctx, abi::PIC_HOLDER_SHAPE_WORD);
+    {
+        let blk = ctx.block();
+        let sid_addr = blk.add(I64, &holder, "4");
+        let sid_ptr = blk.inttoptr(I64, &sid_addr);
+        let sid = blk.load(I32, &sid_ptr);
+        let primed = blk.trunc(I64, &holder_shape, I32);
+        let same = blk.icmp_eq(I32, &sid, &primed);
+        blk.cond_br(&same, &lane_l, miss_label);
+    }
+
+    // The lane still holds the primed pair; no worker; a getter to call.
+    ctx.current_block = lane_idx;
+    let pair = word(ctx, abi::PIC_HOLDER_PAIR_WORD);
+    let getter = word(ctx, abi::PIC_HOLDER_GETTER_WORD);
+    {
+        let blk = ctx.block();
+        let slot = blk.and(I64, &kind, "4294967295");
+        let base_addr = blk.add(I64, &holder, &header_bytes.to_string());
+        let base = blk.inttoptr(I64, &base_addr);
+        let lane_ptr = blk.gep(I64, &base, &[(I64, &slot)]);
+        let lane = blk.load(I64, &lane_ptr);
+        let tagged = blk.or(I64, &pair, crate::nanbox::POINTER_TAG_I64);
+        let same = blk.icmp_eq(I64, &lane, &tagged);
+        let workers = blk.load_atomic_seq_cst(I8, "@PERRY_METHOD_SITE_WORKERS_PRESENT", 1);
+        let workers = blk.zext(I8, &workers, I32);
+        let no_workers = blk.icmp_eq(I32, &workers, "0");
+        let has_getter = blk.icmp_ne(I64, &getter, "0");
+        let ok = blk.and(I1, &same, &no_workers);
+        let ok = blk.and(I1, &ok, &has_getter);
+        blk.cond_br(&ok, &call_l, miss_label);
+    }
+
+    // The getter runs user code: a versioned loop records its bailout here,
+    // as on the collecting slow call.
+    ctx.current_block = call_idx;
+    crate::expr::emit_versioned_loop_callback_deopt(ctx);
+    let blk = ctx.block();
+    let code = blk.inttoptr(I64, &getter);
+    let value = crate::expr::body_call::emit_class_getter_call(blk, &code, recv_box);
+    let end = blk.label.clone();
+    blk.br(merge_label);
+    (value, end)
+}

crates/perry-codegen/src/expr/property_get/front_contract_tests.rs

@@ -34,7 +34,18 @@ fn verify_front_flow(blocks: &Blocks) -> Result<usize, String> {
     };
     let (entry, _) = tower_block(blocks, "pic.miss.front");
     let (token, token_body) = tower_block(blocks, "pic.token");
-    if predecessors(blocks, entry) != [token] || tower_cond_br(token_body).2 != entry {
+    // The front is reached from the token compare's false edge, directly or
+    // (#10498) through the class-accessor arm, whose every guard declines to
+    // it and each of which the token compare dominates.
+    let front_preds: Vec<&str> = if blocks.iter().any(|(l, _)| l.starts_with("pic.acc.")) {
+        verify_accessor_arm(blocks)?
+    } else {
+        if tower_cond_br(token_body).2 != entry {
+            return Err("the front entry must be dominated by the token compare".into());
+        }
+        vec![token]
+    };
+    if predecessors(blocks, entry) != front_preds {
         return Err("the front entry must be dominated by the token compare".into());
     }
     // Every branch between the token miss and the front call resolves the
@@ -51,7 +62,7 @@ fn verify_front_flow(blocks: &Blocks) -> Result<usize, String> {
             ));
         }
         for (label, expected_targets, expected_preds) in [
-            (entry, vec![tsd, slow], vec![token]),
+            (entry, vec![tsd, slow], front_preds.clone()),
             (tsd, vec![fast, slow], vec![entry]),
             (fast, vec![join], vec![tsd]),
             (slow, vec![join], vec![entry, tsd]),
@@ -135,6 +146,93 @@ fn verify_front_flow(blocks: &Blocks) -> Result<usize, String> {
     Ok(*index)
 }
 
+/// The class-accessor arm's guards, in chain order (#10498, `accessor_arm.rs`).
+pub(super) const ACCESSOR_ARM_GUARDS: [&str; 6] = [
+    "pic.acc.empty",
+    "pic.acc.cache",
+    "pic.acc.recv",
+    "pic.acc.kind",
+    "pic.acc.holder",
+    "pic.acc.lane",
+];
+
+/// #10498: the class-accessor arm on `pic.token`'s false edge, ahead of the
+/// front. Returns its guard blocks in chain order, after proving that
+///
+/// * the token compare's false edge is the first guard;
+/// * every guard ends in a live conditional branch whose true edge is the
+///   next guard (the last guard's is the call block) and whose false edge is
+///   the front's entry;
+/// * every guard and the call block has exactly one predecessor (the token
+///   compare for the first, the previous guard otherwise), so every guard
+///   dominates the call;
+/// * the call block makes exactly one call, an indirect
+///   `call double %code(double %recv)` (the compiled getter), and continues
+///   only to the tower's merge.
+pub(super) fn verify_accessor_arm(blocks: &Blocks) -> Result<Vec<&str>, String> {
+    let find = |prefix: &str| -> Result<(&str, &[String]), String> {
+        let found: Vec<_> = blocks
+            .iter()
+            .filter(|(l, _)| l.starts_with(prefix))
+            .collect();
+        match found.as_slice() {
+            [(l, b)] => Ok((l.as_str(), b.as_slice())),
+            _ => Err(format!("expected one `{prefix}` block: {found:?}")),
+        }
+    };
+    let (token, token_body) = find("pic.token")?;
+    let (front, _) = find("pic.miss.front")?;
+    let (call, call_body) = find("pic.acc.call")?;
+    let mut guards = Vec::new();
+    for prefix in ACCESSOR_ARM_GUARDS {
+        guards.push(find(prefix)?);
+    }
+    if tower_cond_br(token_body).2 != guards[0].0 {
+        return Err("the token compare's false edge must enter the accessor arm".into());
+    }
+    for (i, (label, body)) in guards.iter().enumerate() {
+        if !body.last().is_some_and(|l| l.starts_with("br i1 %")) {
+            return Err(format!(
+                "accessor guard {label} must branch on a live predicate"
+            ));
+        }
+        let (_, on_true, on_false) = tower_cond_br(body);
+        let next = guards.get(i + 1).map(|(l, _)| *l).unwrap_or(call);
+        if on_true != next || on_false != front {
+            return Err(format!(
+                "accessor guard {label} must continue to {next} and decline to the front: {body:?}"
+            ));
+        }
+        let pred = if i == 0 { token } else { guards[i - 1].0 };
+        if predecessors(blocks, label) != [pred] {
+            return Err(format!(
+                "accessor guard {label} must be reached only from {pred}"
+            ));
+        }
+    }
+    if predecessors(blocks, call) != [guards[guards.len() - 1].0] {
+        return Err("the getter call must be reached only through every guard".into());
+    }
+    let calls: Vec<&String> = call_body
+        .iter()
+        .filter(|l| l.contains(" call ") || l.contains(" invoke ") || l.starts_with("call "))
+        .collect();
+    if calls.len() != 1 || !calls[0].contains(" = call double %") {
+        return Err(format!(
+            "the accessor arm makes exactly one indirect getter call: {call_body:?}"
+        ));
+    }
+    if !call_body
+        .last()
+        .is_some_and(|l| l.starts_with("br label %pget.recv_merge."))
+    {
+        return Err(format!(
+            "the getter's answer must reach the merge: {call_body:?}"
+        ));
+    }
+    Ok(guards.iter().map(|(l, _)| *l).collect())
+}
+
 pub(super) fn front_call_block(blocks: &Blocks) -> (&str, &[String]) {
     let index = verify_front_flow(blocks).unwrap_or_else(|e| panic!("{e}: {blocks:?}"));
     (&blocks[index].0, &blocks[index].1)
@@ -361,5 +459,20 @@ fn front_contract_rejects_lookup_bypasses_wrong_slots_and_wrong_declines() {
         let body = &mut wrong.iter_mut().find(|(l, _)| l == entry).unwrap().1;
         *body.last_mut().unwrap() = format!("br label %{slow}");
         assert!(verify_front_flow(&wrong).is_err(), "{target}: front bypass");
+        // #10498: an accessor guard that stops declining, or the receiver
+        // compare jumped over, must be caught.
+        if blocks.iter().any(|(l, _)| l.starts_with("pic.acc.")) {
+            for guard in ACCESSOR_ARM_GUARDS {
+                let mut wrong = blocks.clone();
+                let (label, body) = tower_block(&blocks, guard);
+                let (_, on_true, _) = tower_cond_br(body);
+                let body = &mut wrong.iter_mut().find(|(l, _)| l == label).unwrap().1;
+                *body.last_mut().unwrap() = format!("br label %{on_true}");
+                assert!(
+                    verify_front_flow(&wrong).is_err(),
+                    "{target}: accessor guard {guard} skipped"
+                );
+            }
+        }
     }
 }