perf(codegen): inline untyped Array element stores and heal forwarded heads on reads (#10513, #10514)

changelog.d/11588-untyped-array-element-access.md

@@ -0,0 +1 @@
+perf(codegen): an untyped `obj[i] = v` onto a live ordinary Array now stores inline behind the guarded in-bounds tier instead of calling `js_dyn_index_set_strict` on every element (#10513). Untyped reads heal one growth-forwarding hop inline, and the packed-f64 loop tier repairs a forwarded receiver slot before its guard (#10514). On qb2: node-forge/rsa_sign −50.4%, big.js/arith_chain −42.6% instructions per iteration; the #10514 grown-array read goes from 564 to 73 instructions per element.

crates/perry-codegen/src/expr/barrier_stem_census_tests.rs

@@ -91,6 +91,7 @@ pub(super) const VERIFIED_BARRIER_STEMS: &[(&str, StemKind)] = &[
     ("apush", StemKind::GenerationTested),
     ("class_field_set", StemKind::PointerTestedStore),
     ("ctor_prologue", StemKind::ValueAndGenerationTested),
+    ("dynarr.set", StemKind::ValueAndGenerationTested),
     ("idxset.inbounds", StemKind::ValueAndGenerationTested),
     ("idxset.recv_captured", StemKind::ValueAndGenerationTested),
     ("idxset.recv_global", StemKind::ValueAndGenerationTested),
@@ -798,6 +799,49 @@ fn idxset_runtime_key_ir() -> String {
         .expect("LLVM IR should be UTF-8")
 }
 
+/// `probe(a: any, k: any, v: any) { a[k] = v }` — the untyped store's
+/// ordinary-Array arm (#10513), whose inline in-bounds store must keep the
+/// value-and-generation-tested barrier for an arbitrary value.
+fn dynarr_set_ir() -> String {
+    const ARR_ID: u32 = 31;
+    let mut m = Module::new("dynarr_set_census.ts");
+    let param = |id: u32, name: &str| Param {
+        id,
+        name: name.to_string(),
+        ty: Type::Any,
+        default: None,
+        decorators: Vec::new(),
+        is_rest: false,
+        arguments_object: None,
+    };
+    m.functions = vec![Function {
+        id: 1,
+        name: "probe".to_string(),
+        type_params: Vec::new(),
+        params: vec![param(ARR_ID, "a"), param(IDX_ID, "k"), param(VAL_ID, "v")],
+        return_type: Type::Any,
+        body: vec![
+            Stmt::Expr(Expr::IndexSet {
+                object: Box::new(Expr::LocalGet(ARR_ID)),
+                index: Box::new(Expr::LocalGet(IDX_ID)),
+                value: Box::new(Expr::LocalGet(VAL_ID)),
+            }),
+            Stmt::Return(Some(Expr::LocalGet(ARR_ID))),
+        ],
+        is_async: false,
+        is_generator: false,
+        is_strict: true,
+        is_exported: false,
+        captures: Vec::new(),
+        decorators: Vec::new(),
+        was_plain_async: false,
+        was_unrolled: false,
+    }];
+    m.init_kind = ModuleInitKind::Eager;
+    String::from_utf8(compile_module(&m, ir_opts()).expect("module compiles"))
+        .expect("LLVM IR should be UTF-8")
+}
+
 /// `class Boxed { v: any; constructor(v) { this.v = v } }` plus an escaping
 /// `new Boxed(1)` — the complete parameter-to-field constructor is what selects
 /// constructor-free prologue stores, and the boxed field requires their
@@ -813,6 +857,7 @@ fn probe_ir(stem: &str) -> String {
         "apush" => apush_ir(),
         "class_field_set" => super::class_field_barrier_tests::ir(),
         "ctor_prologue" => ctor_prologue_ir(),
+        "dynarr.set" => dynarr_set_ir(),
         "idxset.inbounds" => idxset_inbounds_ir(),
         "idxset.recv_captured" => idxset_recv_captured_ir(),
         "idxset.recv_global" => idxset_recv_global_ir(),

crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs

@@ -119,6 +119,12 @@ pub(super) fn lower_inline_dyn_typed_array_get(
     let elem_bounds_idx = ctx.new_block("arrlike.elem.bounds");
     let elem_load_idx = ctx.new_block("arrlike.elem.load");
     let elem_value_idx = ctx.new_block("arrlike.elem.value");
+    let fwd_check_idx = ctx.new_block("arrlike.ic.fwd_check");
+    let fwd_follow_idx = ctx.new_block("arrlike.ic.fwd_follow");
+    let fwd_header_idx = ctx.new_block("arrlike.ic.fwd_header");
+    let fwd_check_label = ctx.block_label(fwd_check_idx);
+    let fwd_follow_label = ctx.block_label(fwd_follow_idx);
+    let fwd_header_label = ctx.block_label(fwd_header_idx);
     let object_miss_idx = ctx.new_block("arrlike.ic.miss");
     let merge_idx = ctx.new_block("arrlike.ic.merge");
     let object_header_label = ctx.block_label(object_header_idx);
@@ -185,16 +191,87 @@ pub(super) fn lower_inline_dyn_typed_array_get(
     let forwarded = ctx.block().and(I8, &gc_flags, "128");
     let not_forwarded = ctx.block().icmp_eq(I8, &forwarded, "0");
     let header_ok = ctx.block().and(I1, &object_idx_is_int, &not_forwarded);
+    let header_end_label = ctx.block().label.clone();
     ctx.block()
-        .cond_br(&header_ok, &object_brand_label, &object_miss_label);
+        .cond_br(&header_ok, &object_brand_label, &fwd_check_label);
+
+    // #10514: heal ONE growth-forwarding hop inline. An Array that outgrew its
+    // storage leaves a forwarding stub at the old head, and every binding that
+    // is not the grown local itself — an object field (`this.data`, jsbn's
+    // BigInteger digits), a module global, a closure capture, a parameter —
+    // keeps that stub forever. Rejecting the stub sent every later read of such
+    // an array out of line through `js_packed_arraylike_index_get` →
+    // `js_array_get_f64`, which classifies the address again and follows the
+    // chain on every read (~470 instructions per element vs ~70 presized).
+    // The guarded store tier and `guarded_array.rs` already follow this edge:
+    // the stub's first payload word is the live user address. It is trusted
+    // only once it is in the heap band and its own header re-brands as a
+    // non-forwarded `GC_TYPE_ARRAY`; a longer or corrupt chain, and every
+    // forwarded non-Array, keeps the unchanged slow exit.
+    ctx.current_block = fwd_check_idx;
+    let follow = {
+        let blk = ctx.block();
+        let is_forwarded = blk.icmp_ne(I8, &forwarded, "0");
+        let forwarded_array = blk.and(I1, &is_array, &is_forwarded);
+        blk.and(I1, &forwarded_array, &object_idx_is_int)
+    };
+    ctx.block()
+        .cond_br(&follow, &fwd_follow_label, &object_miss_label);
+
+    ctx.current_block = fwd_follow_idx;
+    let fwd_target = {
+        let blk = ctx.block();
+        let stub_ptr = blk.inttoptr(I64, &object_raw);
+        let target = blk.load(I64, &stub_ptr);
+        let above_floor = blk.icmp_uge(I64, &target, &heap_floor);
+        let below_ceiling = blk.icmp_ult(I64, &target, &heap_ceiling);
+        let in_band = blk.and(I1, &above_floor, &below_ceiling);
+        blk.cond_br(&in_band, &fwd_header_label, &object_miss_label);
+        target
+    };
+
+    ctx.current_block = fwd_header_idx;
+    {
+        let blk = ctx.block();
+        let type_addr = blk.sub(I64, &fwd_target, "8");
+        let type_ptr = blk.inttoptr(I64, &type_addr);
+        let live_type = blk.load(I8, &type_ptr);
+        let live_is_array = blk.icmp_eq(I8, &live_type, "1");
+        let flags_addr = blk.sub(I64, &fwd_target, "7");
+        let flags_ptr = blk.inttoptr(I64, &flags_addr);
+        let live_flags = blk.load(I8, &flags_ptr);
+        let live_forwarded = blk.and(I8, &live_flags, "128");
+        let live_not_forwarded = blk.icmp_eq(I8, &live_forwarded, "0");
+        let live_ok = blk.and(I1, &live_is_array, &live_not_forwarded);
+        blk.cond_br(&live_ok, &object_brand_label, &object_miss_label);
+    }
 
     // `GC_TYPE_ARRAY` takes the direct guarded load. Everything else is offered
     // to the typed-array arm, then to the elements-backed Array-subclass
     // probe; `GC_TYPE_LAZY_ARRAY`, native Buffers and every exotic cell fail
     // both brand tests and are classified by the slow exit. Both `tav.brand`
     // and `arrlike.elem.kind` re-test the brand they need before they read a
     // header word, so nothing else can reach those loads.
+    //
+    // From here on the receiver is `object_raw`: the original head, or the
+    // live head one forwarding hop away (which re-branded as an Array).
     ctx.current_block = object_brand_idx;
+    let fwd_header_label_s = ctx.block_label(fwd_header_idx);
+    let object_raw = ctx.block().phi(
+        I64,
+        &[
+            (object_raw.as_str(), header_end_label.as_str()),
+            (fwd_target.as_str(), fwd_header_label_s.as_str()),
+        ],
+    );
+    let gc_type = ctx.block().phi(
+        I8,
+        &[
+            (gc_type.as_str(), header_end_label.as_str()),
+            ("1", fwd_header_label_s.as_str()),
+        ],
+    );
+    let is_array = ctx.block().icmp_eq(I8, &gc_type, "1");
     ctx.block()
         .cond_br(&is_array, &object_array_guard_label, &ta_brand_label);
 

crates/perry-codegen/src/expr/index_get_claim_tests.rs

@@ -215,6 +215,10 @@ fn unknown_numeric_read_is_one_inline_hit_and_one_out_of_line_exit() {
             "arrlike.elem.bounds",
             "arrlike.elem.load",
             "arrlike.elem.value",
+            // #10514: one growth-forwarding hop healed inline.
+            "arrlike.ic.fwd_check",
+            "arrlike.ic.fwd_follow",
+            "arrlike.ic.fwd_header",
             "arrlike.ic.miss",
             "arrlike.ic.merge",
         ],
@@ -370,7 +374,7 @@ fn the_number_context_coercion_is_coupled_across_every_arm() {
         );
         assert_eq!(
             dynamic_index_site_blocks(&ir).len(),
-            21,
+            24,
             "{name}: a number context must not change the emitted block shape:\n{ir}"
         );
         let miss = super::class_field_barrier_tests::block_body(&ir, "arrlike.ic.miss.")

crates/perry-codegen/src/expr/index_set.rs

@@ -37,7 +37,7 @@ use crate::native_value::{
 };
 use crate::rooting;
 use crate::type_analysis::{is_array_expr, is_numeric_expr, is_string_expr, receiver_class_name};
-use crate::types::{DOUBLE, I1, I32, I64};
+use crate::types::{DOUBLE, I32, I64};
 
 use super::index_set_packed_loop::lower_packed_numeric_loop_index_set;
 use super::index_set_typed_array::lower_inline_dyn_typed_array_set;
@@ -319,28 +319,8 @@ fn lower_array_index_set_via_runtime_key(
             // side has done this since #7286 (`aidx.canonical`). A rejected key
             // becomes index -1, which the guarded in-bounds store declines
             // onto the same helper arm, so the helper is emitted once.
-            let guard_idx_i32 = {
-                let blk = ctx.block();
-                let raw_ge_zero = blk.fcmp("oge", &idx_double, "0.0");
-                let raw_le_i32_max = blk.fcmp("ole", &idx_double, "2147483647.0");
-                let raw_in_range = blk.and(I1, &raw_ge_zero, &raw_le_i32_max);
-                // `fptosi` is poison for NaN/out-of-range input: convert the
-                // range-sanitized value.
-                let safe_raw = blk.select(I1, &raw_in_range, DOUBLE, &idx_double, "0.0");
-                let raw_i32 = blk.fptosi(DOUBLE, &safe_raw, I32);
-                let raw_round_trip = blk.sitofp(I32, &raw_i32, DOUBLE);
-                let raw_is_integral = blk.fcmp("oeq", &raw_round_trip, &idx_double);
-                let raw_is_canonical = blk.and(I1, &raw_in_range, &raw_is_integral);
-                let bits = blk.bitcast_double_to_i64(&idx_double);
-                let top16 = blk.lshr(I64, &bits, "48");
-                let is_boxed_i32 = blk.icmp_eq(I64, &top16, crate::nanbox::INT32_TAG_TOP16_I64);
-                let boxed_i32 = blk.trunc(I64, &bits, I32);
-                let boxed_nonnegative = blk.icmp_sge(I32, &boxed_i32, "0");
-                let boxed_is_canonical = blk.and(I1, &is_boxed_i32, &boxed_nonnegative);
-                let canonical = blk.or(I1, &raw_is_canonical, &boxed_is_canonical);
-                let idx_i32 = blk.select(I1, &is_boxed_i32, I32, &boxed_i32, &raw_i32);
-                blk.select(I1, &canonical, I32, &idx_i32, "-1")
-            };
+            let guard_idx_i32 =
+                super::index_set_guarded::emit_canonical_element_index_i32(ctx, &idx_double);
             super::index_set_guarded::emit_guarded_inbounds_array_store(
                 ctx,
                 &arr_box,
@@ -670,7 +650,8 @@ pub(crate) fn lower(
                                 &vals[1],
                                 &vals[2],
                                 assignment_strict,
-                            );
+                                None,
+                            )?;
                             let slow = LoweredValue::js_value(result.clone());
                             ctx.record_lowered_value_with_access_mode(
                                 "TypedArraySet",
@@ -704,7 +685,8 @@ pub(crate) fn lower(
                         &vals[1],
                         &vals[2],
                         assignment_strict,
-                    );
+                        None,
+                    )?;
                     let slow = LoweredValue::js_value(vals[2].clone());
                     ctx.record_lowered_value_with_access_mode(
                         "TypedArraySet",
@@ -794,6 +776,13 @@ pub(crate) fn lower(
             ) || is_string_expr(ctx, index);
             if recv_unknown && !index_is_static_string_or_symbol {
                 let strict = assignment_strict;
+                // #10513: the receiver's layout is unknown, so the layout note
+                // stays on; the barrier and numeric note follow the VALUE.
+                let array_facts = super::index_set_typed_array::DynArrayStoreFacts {
+                    layout_note_needed: true,
+                    write_barrier_needed: array_store_needs_write_barrier(ctx, value),
+                    value_is_numeric: is_numeric_expr(ctx, value),
+                };
                 return rooting::with_operands_rooted_across(
                     ctx,
                     &[object, index],
@@ -814,13 +803,14 @@ pub(crate) fn lower(
                         // at the access site, falling back to `js_dyn_index_set` on
                         // any guard miss. #7640: both the receiver and key are
                         // re-read after the allocating RHS.
-                        Ok(lower_inline_dyn_typed_array_set(
+                        lower_inline_dyn_typed_array_set(
                             ctx,
                             &vals[0],
                             &vals[1],
                             &val_double,
                             strict,
-                        ))
+                            Some(array_facts),
+                        )
                     },
                 );
             }

crates/perry-codegen/src/expr/index_set_barrier_tests.rs

@@ -428,32 +428,46 @@ fn the_guarded_property_receiver_store_follows_one_forwarding_edge_inline() {
     let ir = ir();
     let deref =
         block_body(&ir, "idxset.recv_prop.deref.").expect("guarded store emits its `deref` block");
+    let follow = block_body(&ir, "idxset.recv_prop.deref.follow.")
+        .expect("guarded store emits its `deref.follow` block");
     let live = block_body(&ir, "idxset.recv_prop.deref.live.")
         .expect("guarded store emits its `deref.live` block");
     let fast =
         block_body(&ir, "idxset.recv_prop.fast.").expect("guarded store emits its `fast` block");
 
-    // (1) `deref` reads the stub's first payload word and selects it as the
-    // live handle when the header says ARRAY + FORWARDED.
-    let select_line = deref
+    // (0) #10513: `deref` decides on the ARRAY brand byte alone, so a receiver
+    // that is not an Array leaves before any forwarding or integrity work.
+    assert!(
+        deref.contains("sub i64") && deref.contains("load i8"),
+        "`deref` reads the brand byte:\n{deref}"
+    );
+    assert!(
+        deref.contains("br i1") && deref.contains("idxset.recv_prop.deref.follow."),
+        "`deref` must branch into `deref.follow` on the ARRAY brand:\n{deref}"
+    );
+
+    // (1) `deref.follow` reads the stub's first payload word and selects it as
+    // the live handle when the header says FORWARDED.
+    let select_line = follow
         .lines()
         .map(str::trim)
         .find(|line| line.contains("select i1") && line.contains("i64"))
-        .expect("`deref` selects between the forwarding target and the receiver");
+        .expect("`deref.follow` selects between the forwarding target and the receiver");
     let live_handle = select_line
         .split(" = ")
         .next()
         .expect("select defines a register")
         .to_string();
     let target = operand(select_line, 2).expect("select's taken operand");
-    let target_def = def_of(&deref, &target).expect("forwarding target is defined in `deref`");
+    let target_def =
+        def_of(&follow, &target).expect("forwarding target is defined in `deref.follow`");
     assert!(
         target_def.contains("load i64"),
         "the forwarding target must be the stub's first payload word, got `{target_def}`"
     );
     assert!(
-        deref.contains("br i1") && deref.contains("idxset.recv_prop.deref.live."),
-        "`deref` must branch into `deref.live` after the heap-band test of the live handle"
+        follow.contains("br i1") && follow.contains("idxset.recv_prop.deref.live."),
+        "`deref.follow` must branch into `deref.live` after the heap-band test of the live handle"
     );
 
     // (2) `deref.live` re-reads the ARRAY brand and the FORWARDED bit from the