diff --git a/changelog.d/11554-s2-ic-fast-slow-split.md b/changelog.d/11554-s2-ic-fast-slow-split.md new file mode 100644 index 0000000000..81038340e1 --- /dev/null +++ b/changelog.d/11554-s2-ic-fast-slow-split.md @@ -0,0 +1,9 @@ +- **perf(gc): S2 of the deferred-collection RFC — GC-leaf fast paths for the full-outline ICs, the dynamic-callee unbox and template coercion (#11554).** A call that can collect is an RS4GC statepoint: every GC value live across it is spilled before the call and reloaded after it. Six helpers collect or re-enter JS only on a rare arm, yet every call paid that. Each now has its common case on a path that is no call at all, or a call to an audited `CannotCollect` leaf (`"gc-leaf-function"`, nounwind so it stays a plain `call` inside a `try`), and keeps the original collecting call on a cold arm (`expr/ic_fast_split.rs`): + - full-outline generic read: `js_object_get_field_ic_fast` (MRU-word hit) / `js_object_get_field_ic_fast_miss` (the unchanged ladder minus the MRU probe); + - full-outline static-key store: `js_put_value_set_packed_fast` (existing-key inline-way store) / `js_put_value_set_packed_miss`; + - full-outline class-field get/set: `js_class_field_{get,set}_ic_fast` (the side-effect-free fast contract plus the slot access) / `_fast_miss` (exactly the full helper's remainder, guard evaluated once); + - `js_closure_unbox_callee_checked` and the string arm of `js_template_string_coerce_box`: inline tag tests, helper only on the cold arm. + + The fast entries decline (TAG_HOLE or a status) while typed feedback or property descriptors are in use: on today's runtime those arms take the feedback registry lock (a `GcRootRegistryGuard` whose drop can flush a deferred collection, #11523) or walk descriptors through an indirect call. Deferred, because their hit path allocates or needs more proof: class allocation (`js_object_alloc_class_inline_keys{,_stamped}`, `js_build_class_keys_array`, `js_gc_typed_shape_id_for_keys`), `js_array_push_f64` growth, the box reads (`js_box_get_bits*`, superseded by #11179's scope objects), `js_ctor_return_override` (already behind an inline `undefined` test), `js_packed_arraylike_index_get` and `js_typed_feedback_array_index_get_fallback_boxed`. + + Static effect is nil by design (the slow call keeps the same live set: `__25747` 3,312,145 relocations and 7,127 statepoints before and after, +2.4 % pre-RS4GC IR); the win is at run time on full-outline code. Tests: `native_root_coverage::ic_fast_split` (post-RS4GC: every fast call is a plain gc-leaf call and not a statepoint, every slow call is a statepoint with a non-empty live set, with a strip-the-attribute control), `tests/ic_fast_split_slow_path_evacuation.rs` + `test_gap_ic_fast_split_slow_path.ts` (getter / Proxy trap / setter / `toString` / not-callable throw that allocate on the slow arm, seeded evacuating schedule with from-space protection and the verifier, asserting copying minors moved objects). Sabotage: classifying the `_fast_miss` continuations `CannotCollect` faults in retired from-space. diff --git a/crates/perry-codegen/src/expr/ic_fast_split.rs b/crates/perry-codegen/src/expr/ic_fast_split.rs new file mode 100644 index 0000000000..3a70e32e12 --- /dev/null +++ b/crates/perry-codegen/src/expr/ic_fast_split.rs @@ -0,0 +1,164 @@ +//! Fast/slow splits of re-entering runtime helpers (deferred-collection RFC, +//! step S2). +//! +//! A call that can collect is an RS4GC statepoint: every GC value live across +//! it is spilled before the call and reloaded after it. Several helpers can +//! collect only on a rare arm — a getter, a setter, a throw, an allocating +//! miss — yet every call paid that cost. Each lowering here puts the common +//! case on a path that is either no call at all or a call to an audited +//! `CannotCollect` leaf (`gc_call_effects.rs`, which is what makes it +//! `"gc-leaf-function"`), and keeps the original collecting call on a cold +//! arm. RS4GC then places `gc.relocate`s only at that cold statepoint. +//! +//! Every value the cold arm passes is computed before the fast test, so it +//! dominates the arm; the arm rejoins through a phi. + +use super::FnCtx; +use crate::types::{LlvmType, DOUBLE, I1, I32, I64}; + +/// `fast(fast_args)`; when it answers `TAG_HOLE`, a cold arm calls +/// `slow(slow_args)`. Returns the joined DOUBLE. +/// +/// `fast` must be a `CannotCollect` symbol whose contract is "serve the hit, +/// `TAG_HOLE` for anything else", and `slow` must reproduce the full helper +/// for every case `fast` declines. +pub(crate) fn emit_hole_declining_split( + ctx: &mut FnCtx<'_>, + tag: &str, + fast: &str, + fast_args: &[(LlvmType, &str)], + slow: &str, + slow_args: &[(LlvmType, &str)], +) -> String { + let slow_idx = ctx.new_block(&format!("{tag}.split_slow")); + let merge_idx = ctx.new_block(&format!("{tag}.split_merge")); + let slow_label = ctx.block_label(slow_idx); + let merge_label = ctx.block_label(merge_idx); + + let fast_val = ctx.block().call(DOUBLE, fast, fast_args); + let fast_bits = ctx.block().bitcast_double_to_i64(&fast_val); + let served = ctx + .block() + .icmp_ne(I64, &fast_bits, crate::nanbox::TAG_HOLE_I64); + let fast_end = ctx.block().label.clone(); + // The SERVED edge is the true edge, like every guard-passing edge in the + // property towers (#7883). + ctx.block().cond_br(&served, &merge_label, &slow_label); + + ctx.current_block = slow_idx; + let slow_val = ctx.block().call(DOUBLE, slow, slow_args); + let slow_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + ctx.block().phi( + DOUBLE, + &[ + (fast_val.as_str(), fast_end.as_str()), + (slow_val.as_str(), slow_end.as_str()), + ], + ) +} + +/// The full-outline class-field SET: `js_class_field_set_ic_fast` answers a +/// status; anything but DONE (1) calls `js_class_field_set_ic_fast_miss` with +/// the status prepended to the same operands. +pub(crate) fn emit_class_field_set_split(ctx: &mut FnCtx<'_>, args: &[(LlvmType, &str)]) { + let slow_idx = ctx.new_block("class_field_set.split_slow"); + let merge_idx = ctx.new_block("class_field_set.split_merge"); + let slow_label = ctx.block_label(slow_idx); + let merge_label = ctx.block_label(merge_idx); + + let status = ctx.block().call(I32, "js_class_field_set_ic_fast", args); + let done = ctx.block().icmp_eq(I32, &status, "1"); + ctx.block().cond_br(&done, &merge_label, &slow_label); + + ctx.current_block = slow_idx; + let mut slow_args: Vec<(LlvmType, &str)> = Vec::with_capacity(args.len() + 1); + slow_args.push((I32, &status)); + slow_args.extend_from_slice(args); + ctx.block() + .call_void("js_class_field_set_ic_fast_miss", &slow_args); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; +} + +/// `` `${v}` `` for an operand that is already a string (heap or SSO) is the +/// operand itself — `js_template_string_coerce_box`'s second short-circuit. +/// That arm is now inline and calls nothing; every other operand (numbers, +/// which allocate their text, and objects, whose `toString` is user code) +/// keeps the call on a cold arm. +pub(crate) fn emit_template_string_coerce(ctx: &mut FnCtx<'_>, v: &str) -> String { + let slow_idx = ctx.new_block("tmpl_coerce.slow"); + let merge_idx = ctx.new_block("tmpl_coerce.merge"); + let slow_label = ctx.block_label(slow_idx); + let merge_label = ctx.block_label(merge_idx); + + let entry_end = { + let blk = ctx.block(); + let bits = blk.bitcast_double_to_i64(v); + let top16 = blk.lshr(I64, &bits, "48"); + let is_heap = blk.icmp_eq(I64, &top16, crate::nanbox::STRING_TAG_TOP16_I64); + let is_sso = blk.icmp_eq(I64, &top16, crate::nanbox::SHORT_STRING_TAG_TOP16_I64); + let is_str = blk.or(I1, &is_heap, &is_sso); + let end = blk.label.clone(); + blk.cond_br(&is_str, &merge_label, &slow_label); + end + }; + + ctx.current_block = slow_idx; + let coerced = ctx + .block() + .call(DOUBLE, "js_template_string_coerce_box", &[(DOUBLE, v)]); + let slow_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + ctx.block().phi( + DOUBLE, + &[ + (v, entry_end.as_str()), + (coerced.as_str(), slow_end.as_str()), + ], + ) +} + +/// The checked dynamic-callee unbox (#5504) with its hit inline: a +/// `POINTER_TAG` value unboxes to its low 48 bits, which is everything +/// `js_closure_unbox_callee_checked` does on that arm. Every other value +/// takes the call on a cold arm, where it throws `TypeError: value is not a +/// function` exactly as before. +pub(crate) fn emit_checked_callee_unbox(ctx: &mut FnCtx<'_>, callee: &str) -> String { + let slow_idx = ctx.new_block("callee_unbox.slow"); + let merge_idx = ctx.new_block("callee_unbox.merge"); + let slow_label = ctx.block_label(slow_idx); + let merge_label = ctx.block_label(merge_idx); + + let (handle, entry_end) = { + let blk = ctx.block(); + let bits = blk.bitcast_double_to_i64(callee); + let top16 = blk.lshr(I64, &bits, "48"); + let is_ptr = blk.icmp_eq(I64, &top16, crate::nanbox::POINTER_TAG_TOP16_I64); + let handle = blk.and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + let end = blk.label.clone(); + blk.cond_br(&is_ptr, &merge_label, &slow_label); + (handle, end) + }; + + ctx.current_block = slow_idx; + let checked = ctx + .block() + .call(I64, "js_closure_unbox_callee_checked", &[(DOUBLE, callee)]); + let slow_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + ctx.block().phi( + I64, + &[ + (handle.as_str(), entry_end.as_str()), + (checked.as_str(), slow_end.as_str()), + ], + ) +} diff --git a/crates/perry-codegen/src/expr/logical_collections.rs b/crates/perry-codegen/src/expr/logical_collections.rs index 9aacf847bf..e14f20e1ab 100644 --- a/crates/perry-codegen/src/expr/logical_collections.rs +++ b/crates/perry-codegen/src/expr/logical_collections.rs @@ -1177,9 +1177,10 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { } Expr::TemplateStringCoerce(operand) => { let v = lower_expr(ctx, operand)?; - Ok(ctx - .block() - .call(DOUBLE, "js_template_string_coerce_box", &[(DOUBLE, &v)])) + // S2: a string operand is answered inline; see `ic_fast_split.rs`. + Ok(crate::expr::ic_fast_split::emit_template_string_coerce( + ctx, &v, + )) } // -------- Object(value) coercion (#3149) -------- diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index a048761ede..a6f29829c9 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -3072,6 +3072,7 @@ pub(crate) use instance_misc1::builtin_parent_reserved_class_id; pub(crate) mod class_field_inline_guard; pub(crate) mod element_shape_guard; pub(crate) mod element_shape_reads; +pub(crate) mod ic_fast_split; mod js_runtime; mod literals_vars; mod logical_collections; diff --git a/crates/perry-codegen/src/expr/property_get.rs b/crates/perry-codegen/src/expr/property_get.rs index 3c7da5bdb8..f906cd9769 100644 --- a/crates/perry-codegen/src/expr/property_get.rs +++ b/crates/perry-codegen/src/expr/property_get.rs @@ -1703,18 +1703,24 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { let key_bits = blk.bitcast_double_to_i64(&key_box); blk.and(I64, &key_bits, POINTER_MASK_I64) }; - let val = ctx.block().call( - DOUBLE, - "js_class_field_get_ic", - &[ - (I64, &site_id), - (DOUBLE, &recv_box), - (I32, &expected_class_id_str), - (I32, &expected_shape_id), - (I64, &key_raw), - (I32, &field_idx_str), - (I32, requires_raw_f64_str), - ], + // S2: guard + load is a GC-leaf call; the by-name + // fallback is the cold collecting arm. + let ic_args = [ + (I64, site_id.as_str()), + (DOUBLE, recv_box.as_str()), + (I32, expected_class_id_str.as_str()), + (I32, expected_shape_id.as_str()), + (I64, key_raw.as_str()), + (I32, field_idx_str.as_str()), + (I32, requires_raw_f64_str), + ]; + let val = crate::expr::ic_fast_split::emit_hole_declining_split( + ctx, + "class_field_get", + "js_class_field_get_ic_fast", + &ic_args, + "js_class_field_get_ic_fast_miss", + &ic_args, ); return Ok(val); } diff --git a/crates/perry-codegen/src/expr/property_get/array_length_tests.rs b/crates/perry-codegen/src/expr/property_get/array_length_tests.rs index b2391b0651..d664efe8d6 100644 --- a/crates/perry-codegen/src/expr/property_get/array_length_tests.rs +++ b/crates/perry-codegen/src/expr/property_get/array_length_tests.rs @@ -507,8 +507,12 @@ fn a_full_outline_length_read_serves_a_live_plain_array_before_the_helper() { #[test] fn a_full_outline_non_length_read_is_the_bare_helper_call() { let ir = emit_outlined_read("foo"); + // S2: a non-`.length` full-outline read is the GC-leaf hit plus the cold + // collecting miss (`ic_fast_split.rs`); `.length` keeps the single call. assert!( - ir.contains("@js_object_get_field_ic("), + ir.contains("@js_object_get_field_ic_fast(") + && ir.contains("@js_object_get_field_ic_fast_miss(") + && !ir.contains("call double @js_object_get_field_ic("), "test premise: the read is full-outlined:\n{ir}" ); for gone in [ diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 4366061ed4..37f93626cf 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -407,16 +407,31 @@ pub(crate) fn lower_generic_property_get( (arm, merge_idx, merge_label) }); let key_handle = emit_key_handle(ctx, &key_handle_global); - let val = ctx.block().call( - DOUBLE, - "js_object_get_field_ic", - &[ - (I64, &obj_bits), - (I64, &key_handle), - (I64, &feedback_site_id), - (PTR, &cache_slot_ref), - ], - ); + let ic_args = [ + (I64, obj_bits.as_str()), + (I64, key_handle.as_str()), + (I64, feedback_site_id.as_str()), + (PTR, cache_slot_ref.as_str()), + ]; + // S2: the MRU hit is a GC-leaf call; only its decline arm is the + // collecting (statepoint) call. See `ic_fast_split.rs`. NOT for + // `.length`: what reaches this call there is mostly a string or + // another non-Array receiver no MRU word can serve, so the leaf call + // would be a pure extra call in front of the helper (+0.65 % + // instructions on a string-`.length` loop, measured). It keeps the + // single call. + let val = if property == "length" { + ctx.block().call(DOUBLE, "js_object_get_field_ic", &ic_args) + } else { + crate::expr::ic_fast_split::emit_hole_declining_split( + ctx, + "pget.outline", + "js_object_get_field_ic_fast", + &ic_args, + "js_object_get_field_ic_fast_miss", + &ic_args, + ) + }; let Some(((len, len_end_label), merge_idx, merge_label)) = array_arm else { return Ok(val); }; diff --git a/crates/perry-codegen/src/expr/property_set.rs b/crates/perry-codegen/src/expr/property_set.rs index 3151c0349f..55a6006918 100644 --- a/crates/perry-codegen/src/expr/property_set.rs +++ b/crates/perry-codegen/src/expr/property_set.rs @@ -1324,8 +1324,10 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr, assignment_strict: bool) - ctx, super::store_census::CFIELD_IC_CALL, ); - ctx.block().call_void( - "js_class_field_set_ic", + // S2: guard + store is a GC-leaf call; only + // a decline takes the collecting call. + crate::expr::ic_fast_split::emit_class_field_set_split( + ctx, &[ (I64, &site_id), (DOUBLE, &recv_box), diff --git a/crates/perry-codegen/src/expr/put_value_store_ic.rs b/crates/perry-codegen/src/expr/put_value_store_ic.rs index 07bb5294f2..cea0ced3bf 100644 --- a/crates/perry-codegen/src/expr/put_value_store_ic.rs +++ b/crates/perry-codegen/src/expr/put_value_store_ic.rs @@ -241,8 +241,17 @@ pub(crate) fn emit_static_store_ic( if crate::codegen::full_outline_ic_enabled() || straight_line_site_outlined(ctx) { super::store_census::bump(ctx, super::store_census::PIC_MISS); let key_handle = emit_key_handle(ctx, &key_handle_global); - return ctx.block().call( - DOUBLE, + // S2: an existing-key store from an inline way is a GC-leaf call; + // everything else takes the collecting miss entry on a cold arm. + return super::ic_fast_split::emit_hole_declining_split( + ctx, + "pset.outline", + "js_put_value_set_packed_fast", + &[ + (DOUBLE, obj_box), + (DOUBLE, value_double), + (PTR, &cache_slot_ref), + ], "js_put_value_set_packed_miss", &[ (DOUBLE, obj_box), diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index 36fe7963f4..99df4be74c 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -109,6 +109,45 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { // slow call. Listed so nothing is spilled or reloaded around it on // the declined-guard edge of every generic property read. | "js_inherited_read_cache_hit_f64" + // S2 of the deferred-collection RFC (`expr/ic_fast_split.rs`): the + // GC-leaf hits of the four full-outline inline caches. Each answers a + // decline (TAG_HOLE, or a status) for every case it cannot serve, and + // the emitted code takes the collecting `_fast_miss` call instead. + // Audited 2026-09-27 against the runtime bodies, the checked items + // being: no Perry-heap allocation, no `GcRootRegistryGuard`, no + // throw, no call into generated code, no poll, no indirect call. + // `js_object_get_field_ic_fast` (`object/field_get_set/ic_miss/ + // outline_split.rs`): a static read, a tag compare, + // `pic_outlined_mru_hit` (a OnceLock env read, + // `pic_slot_peek` — never the allocating `pic_slot_resolve` —, a + // ShapeId compare, one slot load). + // `js_class_field_{get,set}_ic_fast` (`typed_feedback/guards.rs`): + // two static reads, `class_field_{,set_}fast_contract` (header, + // shape-descriptor and layout side-table reads), then one slot + // load, or one store through `runtime_store_jsvalue_slot` (addref, + // layout note, slot barrier — the bodies of `js_string_addref`, + // `js_gc_note_slot_layout`, `js_write_barrier_slot`). + // Deliberately NOT reached, because the census call graph shows each + // reaching the collector or an indirect call on today's runtime: the + // typed-feedback observe/record calls (the registry lock is a + // `GcRootRegistryGuard` whose drop can flush a deferred collection, + // #11523) and the descriptor walk (`get_accessor_descriptor`). The + // fast entries decline outright while feedback or descriptors are in + // use. Nor `js_object_set_field`'s diagnostics (formatting is an + // indirect call) or a live-bound widening (mints a descriptor). + // The S1 generated table and its call-graph checker must pick these + // up and are the authority over this comment (see the S2 PR for the + // checker run over the built archives). + | "js_object_get_field_ic_fast" + | "js_class_field_get_ic_fast" + | "js_class_field_set_ic_fast" + // `js_put_value_set_packed_fast` (`proxy/put_value/packed_set.rs`): + // `pic_slot_peek`, the receiver test, a ShapeId compare over the + // site's ways, `packed_hit_receiver_ok` (header reads) and + // `store_object_field_slot` (the same `runtime_store_jsvalue_slot` + // as above). A spill way (`dyn_ic_try_store`) and the key-add memo + // (which can allocate) are declined, not served. + | "js_put_value_set_packed_fast" | "js_transition_ic_spill_append" | "js_write_barrier_slot" | "js_write_barrier_slot_validated_parent" diff --git a/crates/perry-codegen/src/lower_call/console_promise.rs b/crates/perry-codegen/src/lower_call/console_promise.rs index 513251e4d5..d9f420959c 100644 --- a/crates/perry-codegen/src/lower_call/console_promise.rs +++ b/crates/perry-codegen/src/lower_call/console_promise.rs @@ -1615,20 +1615,15 @@ fn lower_closure_call_rooted<'a>( // below it — hoisting the unbox above the argument list instead is not an // option, because its throw is observable and the spec evaluates arguments // before it. - let closure_handle = { - let blk = ctx.block(); - match method_recv { - Some(ref this_val) => blk.call( - I64, - "js_closure_unbox_callee_checked_rebind", - &[(DOUBLE, &recv_box), (DOUBLE, this_val)], - ), - None => blk.call( - I64, - "js_closure_unbox_callee_checked", - &[(DOUBLE, &recv_box)], - ), - } + let closure_handle = match method_recv { + Some(ref this_val) => ctx.block().call( + I64, + "js_closure_unbox_callee_checked_rebind", + &[(DOUBLE, &recv_box), (DOUBLE, this_val)], + ), + // S2: the POINTER_TAG hit is inline; only a non-callable value calls + // (and throws from) the checked unbox. See `ic_fast_split.rs`. + None => crate::expr::ic_fast_split::emit_checked_callee_unbox(ctx, &recv_box), }; // Re-read the arguments BELOW the unbox. `closure_handle` itself is a raw diff --git a/crates/perry-codegen/src/module/linkage.rs b/crates/perry-codegen/src/module/linkage.rs index 7fa87607c8..118c04f4ed 100644 --- a/crates/perry-codegen/src/module/linkage.rs +++ b/crates/perry-codegen/src/module/linkage.rs @@ -291,7 +291,12 @@ pub(crate) fn helper_decl_attrs(name: &str) -> &'static str { "js_object_get_field_ic_miss_packed" | "js_object_get_field_ic_slow" | "js_object_get_field_ic_nonptr" - | "js_write_barrier_root_nanbox" => " cold", + | "js_write_barrier_root_nanbox" + // S2 (`expr/ic_fast_split.rs`): the collecting continuations behind + // a declined GC-leaf hit. Still throwing, still GC-capable. + | "js_object_get_field_ic_fast_miss" + | "js_class_field_get_ic_fast_miss" + | "js_class_field_set_ic_fast_miss" => " cold", // PURE — each verified: pure bit tests/masking on the f64/i64 args, // total over arbitrary bits, no memory access anywhere in the body. // js_nanbox_pointer value/nanbox.rs — tag ladder, 0 → TAG_NULL @@ -338,7 +343,17 @@ pub(crate) fn helper_decl_attrs(name: &str) -> &'static str { | "js_typed_feedback_plain_array_index_set_guard" | "js_typed_feedback_numeric_array_index_set_guard" | "js_typed_feedback_numeric_array_push_guard" - | "js_array_numeric_value_to_raw_f64" => " #4", + | "js_array_numeric_value_to_raw_f64" + // S2 GC-leaf hits (`expr/ic_fast_split.rs`), each `extern "C"` with + // no throw anywhere in its call graph — every case that would throw + // (nullish receiver, setter/getter, TDZ) is declined to the `_fast_miss` + // continuation instead — and no loop outside the audited guards' + // bounded ones. Being nounwind is also what keeps them a plain `call` + // inside a `try`, so the leaf marking never depends on the invoke arm. + | "js_object_get_field_ic_fast" + | "js_class_field_get_ic_fast" + | "js_class_field_set_ic_fast" + | "js_put_value_set_packed_fast" => " #4", _ => "", } } diff --git a/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs b/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs new file mode 100644 index 0000000000..a89e6e04fc --- /dev/null +++ b/crates/perry-codegen/src/native_root_coverage/ic_fast_split.rs @@ -0,0 +1,276 @@ +//! S2 of the deferred-collection RFC (`expr/ic_fast_split.rs`): each split +//! site's fast call is a GC-leaf call RS4GC leaves alone, and its slow call is +//! the ONLY statepoint, still carrying the caller's live GC values. +//! +//! Asserted on the post-RS4GC IR of the production pass string, on both +//! native-roots targets. Both halves are non-vacuous: +//! +//! * the slow-call claims go through [`Statepoints::at`], which panics when +//! the callee produced no safepoint, and require a NON-EMPTY live set (the +//! fixture keeps a fresh object live across every access); +//! * the fast-call claim ("no statepoint at `_fast`") is paired with a +//! differential control: the same IR with the attribute stripped from the +//! fast call DOES produce a statepoint there, so the absence is the +//! attribute's doing and not a parse miss. +//! +//! Sabotage run by hand for the PR (2026-09-27): classifying the four slow +//! continuations `CannotCollect` too makes +//! `split_sites_keep_their_slow_call_as_the_only_statepoint` fail at +//! `Statepoints::at("js_put_value_set_packed_miss")` — no subject. + +use super::*; +use perry_hir::{Class, ClassField}; + +const FAST_SLOW: [(&str, &str); 4] = [ + ( + "js_put_value_set_packed_fast", + "js_put_value_set_packed_miss", + ), + ( + "js_object_get_field_ic_fast", + "js_object_get_field_ic_fast_miss", + ), + ( + "js_class_field_get_ic_fast", + "js_class_field_get_ic_fast_miss", + ), + ( + "js_class_field_set_ic_fast", + "js_class_field_set_ic_fast_miss", + ), +]; + +fn point_class() -> Class { + Class { + id: 101, + name: "Point".to_string(), + type_params: Vec::new(), + extends: None, + extends_name: None, + native_extends: None, + extends_expr: None, + heritage_lexically_shadowed: false, + fields: vec![ClassField { + name: "x".to_string(), + key_expr: None, + ty: Type::Number, + init: None, + is_private: false, + is_readonly: false, + decorators: Vec::new(), + }], + constructor: None, + methods: Vec::new(), + getters: Vec::new(), + setters: Vec::new(), + static_accessor_names: Vec::new(), + static_accessor_fn_ids: Vec::new(), + computed_members: Vec::new(), + static_fields: Vec::new(), + static_methods: Vec::new(), + decorators: Vec::new(), + is_exported: false, + aliases: Vec::new(), + is_nested: false, + alloc_width_hint: 0, + specialized_from: None, + } +} + +fn param(id: u32, name: &str, ty: Type) -> Param { + Param { + id, + name: name.to_string(), + ty, + default: None, + decorators: Vec::new(), + is_rest: false, + arguments_object: None, + } +} + +/// `probe(o: any, p: Point, f: any, q: Point)`: a fresh object stays live across a +/// generic read, a class-field read, a class-field write, a template coercion +/// and a dynamic call. 4,000 empty padding functions push the module past the +/// full-outline threshold, which is where the IC splits apply. +fn split_module() -> Module { + let mut module = bare_module("ic_fast_split.ts"); + module.classes = vec![point_class()]; + module.functions = (0..4000u32) + .map(|i| Function { + id: 10_000 + i, + name: format!("pad{i}"), + type_params: Vec::new(), + params: Vec::new(), + return_type: Type::Void, + body: Vec::new(), + is_async: false, + is_generator: false, + is_strict: true, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }) + .collect(); + module.functions.push(Function { + id: 1, + name: "probe".to_string(), + type_params: Vec::new(), + params: vec![ + param(100, "o", Type::Any), + param(101, "p", Type::Named("Point".to_string())), + param(102, "f", Type::Any), + param(103, "q", Type::Named("Point".to_string())), + ], + return_type: Type::Any, + body: vec![ + let_stmt(20, "keep", heap_value()), + let_stmt(21, "a", field_get(100, "foo")), + let_stmt(22, "b", field_get(101, "x")), + Stmt::Expr(Expr::PropertySet { + object: Box::new(Expr::LocalGet(100)), + property: "bar".to_string(), + value: Box::new(Expr::LocalGet(22)), + }), + Stmt::Expr(Expr::PropertySet { + object: Box::new(Expr::LocalGet(103)), + property: "x".to_string(), + value: Box::new(Expr::Number(7.0)), + }), + let_stmt( + 23, + "s", + Expr::TemplateStringCoerce(Box::new(Expr::LocalGet(21))), + ), + let_stmt( + 24, + "r", + Expr::Call { + callee: Box::new(Expr::LocalGet(102)), + args: vec![Expr::LocalGet(22)], + type_args: Vec::new(), + byte_offset: 0, + }, + ), + Stmt::Return(Some(Expr::Array(vec![ + Expr::LocalGet(20), + Expr::LocalGet(21), + Expr::LocalGet(22), + Expr::LocalGet(23), + Expr::LocalGet(24), + ]))), + ], + is_async: false, + is_generator: false, + is_strict: true, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }); + module +} + +/// Remove `"gc-leaf-function"` from every call to `callee` — the control arm. +fn strip_leaf(ir: &str, callee: &str) -> String { + let marker = format!("@{callee}("); + ir.lines() + .map(|line| { + if line.contains(&marker) && line.contains("call ") { + line.replace(" \"gc-leaf-function\"", "") + } else { + line.to_string() + } + }) + .collect::>() + .join("\n") +} + +#[test] +fn split_sites_keep_their_slow_call_as_the_only_statepoint() { + let module = split_module(); + assert!( + crate::codegen::decide_full_outline_ic(crate::codegen::module_callable_count(&module)), + "test premise: the fixture must cross the full-outline threshold" + ); + for target in NATIVE_TARGETS { + let ir = native_ir(&module, target, false); + let probe = probe_body_symbol(&ir, "ic_fast_split.ts"); + let body = function_slice(&ir, &probe); + let points = statepoints_of(&ir, target, &probe); + + for (fast, slow) in FAST_SLOW { + let fast_calls: Vec<&str> = body + .lines() + .filter(|l| l.contains(&format!("@{fast}(")) && l.contains("call ")) + .collect(); + assert!( + !fast_calls.is_empty(), + "[{target}] test premise: `{fast}` is emitted in @{probe}" + ); + for line in &fast_calls { + assert!( + line.contains("\"gc-leaf-function\"") && !line.contains("invoke "), + "[{target}] the fast call must be a plain gc-leaf call: {line}" + ); + } + assert!( + points.iter().all(|sp| sp.callee != fast), + "[{target}] `{fast}` must not be a statepoint" + ); + for sp in points.at(slow) { + assert!( + !sp.live.is_empty(), + "[{target}] the slow call must still relocate the live object: {sp:?}" + ); + } + + // Control: without the attribute the same fast call IS a + // statepoint, so the absence above is the attribute's doing. + let control = statepoints_of(&strip_leaf(&ir, fast), target, &probe); + assert!( + !control.at(fast).is_empty(), + "[{target}] control must make `{fast}` a statepoint" + ); + } + } +} + +/// The two inline splits: the hit is no call at all, and the original helper +/// is called only from the cold block. +#[test] +fn inline_splits_call_their_helper_only_from_the_cold_block() { + let module = split_module(); + for target in NATIVE_TARGETS { + let ir = native_ir(&module, target, false); + let probe = probe_body_symbol(&ir, "ic_fast_split.ts"); + let body = function_slice(&ir, &probe); + for (helper, cold_prefix) in [ + ("js_template_string_coerce_box", "tmpl_coerce.slow"), + ("js_closure_unbox_callee_checked", "callee_unbox.slow"), + ] { + let mut current = String::new(); + let mut calls = 0; + for line in body.lines() { + if !line.starts_with(char::is_whitespace) && line.ends_with(':') { + current = line.trim_end_matches(':').to_string(); + } else if line.contains(&format!("@{helper}(")) { + calls += 1; + assert!( + current.starts_with(cold_prefix), + "[{target}] `{helper}` called outside `{cold_prefix}`: block {current}" + ); + } + } + assert_eq!( + calls, 1, + "[{target}] `{helper}` must be called exactly once" + ); + let points = statepoints_of(&ir, target, &probe); + assert!(!points.at(helper).is_empty()); + } + } +} diff --git a/crates/perry-codegen/src/native_root_coverage/mod.rs b/crates/perry-codegen/src/native_root_coverage/mod.rs index d0d0d178a2..9d87dd0b2d 100644 --- a/crates/perry-codegen/src/native_root_coverage/mod.rs +++ b/crates/perry-codegen/src/native_root_coverage/mod.rs @@ -97,6 +97,7 @@ use perry_hir::types::Type; use perry_hir::{Expr, Function, Module, ModuleInitKind, Param, Stmt}; mod harness_self_tests; +mod ic_fast_split; mod mechanics; mod specialized_calls; diff --git a/crates/perry-codegen/src/root_reload.rs b/crates/perry-codegen/src/root_reload.rs index b08af9bd57..9fdc234299 100644 --- a/crates/perry-codegen/src/root_reload.rs +++ b/crates/perry-codegen/src/root_reload.rs @@ -203,6 +203,11 @@ const NON_COLLECTING: &[&str] = &[ "perry_transition_cache_base", "js_transition_ic_note_hit", "js_inherited_read_cache_hit_f64", + // S2 GC-leaf IC hits; audited in `gc_call_effects.rs`. + "js_object_get_field_ic_fast", + "js_class_field_get_ic_fast", + "js_class_field_set_ic_fast", + "js_put_value_set_packed_fast", "js_transition_ic_spill_append", "js_write_barrier_slot", "js_write_barrier_slot_validated_parent", diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index 567f47ef4b..8a2968126d 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -235,6 +235,30 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { DOUBLE, &[I64, DOUBLE, I32, I32, I64, I32, I32], ); + // S2 (deferred-collection RFC): the two full-outline class-field ICs as a + // GC-leaf hit (`_fast`, same operands as the full helper) plus a + // collecting miss continuation (`_fast_miss`, the same operands; the SET + // miss takes the fast entry's status first). + module.declare_function( + "js_class_field_get_ic_fast", + DOUBLE, + &[I64, DOUBLE, I32, I32, I64, I32, I32], + ); + module.declare_function( + "js_class_field_get_ic_fast_miss", + DOUBLE, + &[I64, DOUBLE, I32, I32, I64, I32, I32], + ); + module.declare_function( + "js_class_field_set_ic_fast", + I32, + &[I64, DOUBLE, I32, I32, I64, I32, DOUBLE, I32], + ); + module.declare_function( + "js_class_field_set_ic_fast_miss", + VOID, + &[I32, I64, DOUBLE, I32, I32, I64, I32, DOUBLE, I32], + ); module.declare_function( "js_typed_feedback_native_call_method", DOUBLE, @@ -390,6 +414,14 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // single call for oversized modules. Args: (obj_bits, key_handle, site_id, // per-site IC cache global) -> field value. module.declare_function("js_object_get_field_ic", DOUBLE, &[I64, I64, I64, PTR]); + // S2: its MRU hit as a GC-leaf call answering TAG_HOLE on a decline, and + // the collecting rest of the ladder. Same operands as the full helper. + module.declare_function("js_object_get_field_ic_fast", DOUBLE, &[I64, I64, I64, PTR]); + module.declare_function( + "js_object_get_field_ic_fast_miss", + DOUBLE, + &[I64, I64, I64, PTR], + ); // T1: the two exits of the inline generic-get tower. Every guard failure — // SSO / INT32 class ref / nullish / non-object receiver / overflow slot / // deleted slot / named prefix / miss+prime — branches to one of these @@ -601,6 +633,12 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { DOUBLE, &[DOUBLE, I64, DOUBLE, I32, PTR, PTR], ); + // S2: the GC-leaf existing-key way store of a full-outline store site. + module.declare_function( + "js_put_value_set_packed_fast", + DOUBLE, + &[DOUBLE, DOUBLE, PTR], + ); // #9708: takes the site's cache SLOT plus the way index to prime. module.declare_function( "js_put_value_set_ic_miss", diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index e566d4edea..6d5c710ad0 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -589,8 +589,12 @@ js_class_env_set void f64,i32u,i32u,f64 js_class_env_stamp f64 f64,i32u,f64 js_class_field_add f64 f64,f64,f64 js_class_field_get_ic f64 i64,f64,i32u,i32u,ptr,i32u,i32s +js_class_field_get_ic_fast f64 i64,f64,i32u,i32u,ptr,i32u,i32s +js_class_field_get_ic_fast_miss f64 i64,f64,i32u,i32u,ptr,i32u,i32s js_class_field_set_fallback void i64,i64,i64,f64 js_class_field_set_ic void i64,f64,i32u,i32u,ptr,i32u,f64,i32s +js_class_field_set_ic_fast i32s i64,f64,i32u,i32u,ptr,i32u,f64,i32s +js_class_field_set_ic_fast_miss void i32s,i64,f64,i32u,i32u,ptr,i32u,f64,i32s js_class_lexical_binding_get f64 f64 js_class_lexical_binding_set f64 f64,f64 js_class_method_bind f64 f64,ptr,usize @@ -2545,6 +2549,8 @@ js_object_get_field_by_name_f64 f64 ptr,ptr js_object_get_field_by_property_id_f64 f64 ptr,i64 js_object_get_field_f64 f64 ptr,i32u js_object_get_field_ic f64 i64,ptr,i64,ptr +js_object_get_field_ic_fast f64 i64,ptr,i64,ptr +js_object_get_field_ic_fast_miss f64 i64,ptr,i64,ptr js_object_get_field_ic_miss f64 ptr,ptr,ptr js_object_get_field_ic_miss_packed f64 ptr,ptr,ptr,ptr js_object_get_field_ic_nonptr f64 i64,ptr,i64 @@ -3009,6 +3015,7 @@ js_put_value_set f64 f64,f64,f64,f64,i32s js_put_value_set_dyn_ic f64 ptr,f64,f64,f64,i32s js_put_value_set_dyn_ic_miss f64 ptr,f64,f64,f64,i32s js_put_value_set_ic_miss f64 f64,ptr,f64,i32s,ptr,i32s +js_put_value_set_packed_fast f64 f64,f64,ptr js_put_value_set_packed_miss f64 f64,ptr,f64,i32s,ptr,ptr js_querystring_escape f64 f64 js_querystring_native_dispatch f64 ptr,usize,ptr,usize diff --git a/crates/perry-codegen/tests/typed_feedback.rs b/crates/perry-codegen/tests/typed_feedback.rs index 93f8f54573..1e4e2b7758 100644 --- a/crates/perry-codegen/tests/typed_feedback.rs +++ b/crates/perry-codegen/tests/typed_feedback.rs @@ -766,7 +766,10 @@ fn full_outline_ic_collapses_class_field_set_to_single_call() { { let _g = EnvVarGuard::set("PERRY_FULL_OUTLINE_IC", Some("1")); let ir = ir_for(build()); - assert!(ir.contains("call void @js_class_field_set_ic")); + // S2: the outlined call is a GC-leaf hit plus a cold collecting miss. + assert!(ir.contains("call i32 @js_class_field_set_ic_fast(")); + assert!(ir.contains("call void @js_class_field_set_ic_fast_miss(")); + assert!(!ir.contains("call void @js_class_field_set_ic(")); assert!(!ir.contains("class_field_set.fast")); assert!(!ir.contains("class_field_set.fallback")); assert!(!ir.contains("call i32 @js_typed_feedback_class_field_set_guard")); @@ -809,7 +812,10 @@ fn full_outline_ic_collapses_class_field_get_to_single_call() { { let _g = EnvVarGuard::set("PERRY_FULL_OUTLINE_IC", Some("1")); let ir = ir_for(build()); - assert!(ir.contains("call double @js_class_field_get_ic")); + // S2: the outlined call is a GC-leaf hit plus a cold collecting miss. + assert!(ir.contains("call double @js_class_field_get_ic_fast(")); + assert!(ir.contains("call double @js_class_field_get_ic_fast_miss(")); + assert!(!ir.contains("call double @js_class_field_get_ic(")); assert!(!ir.contains("class_field_get.fast")); assert!(!ir.contains("class_field_inline.deref")); assert!(!ir.contains("call i32 @js_typed_feedback_class_field_get_guard(")); diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index 43d45ad1f5..262e196a2d 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -337,6 +337,11 @@ pub use ic_slot::{ pic_arena_bytes, pic_slot_peek, pic_slot_resolve, pic_slot_resolve_init, pic_slots_resolved, }; pub use ic_slow::{js_object_get_field_ic_nonptr, js_object_get_field_ic_slow}; +/// S2 of the deferred-collection RFC: the full-outline read as a GC-leaf hit +/// plus a collecting miss continuation. +#[path = "field_get_set/ic_miss/outline_split.rs"] +mod outline_split; +pub use outline_split::{js_object_get_field_ic_fast, js_object_get_field_ic_fast_miss}; #[cfg(test)] mod buffer_ic_miss_tests { diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index 50752c4a5b..c71aeb2367 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -1375,7 +1375,7 @@ fn outlined_mru_hit_enabled() -> bool { /// the caller has established that the tag was `POINTER`. `cache_slot` is the /// codegen-emitted per-site slot or null. #[inline] -unsafe fn pic_outlined_mru_hit( +pub(super) unsafe fn pic_outlined_mru_hit( obj_handle: *const ObjectHeader, cache_slot: *mut PicCacheSlot, ) -> Option { @@ -1422,6 +1422,19 @@ pub extern "C" fn js_object_get_field_ic( key: *const crate::StringHeader, site_id: u64, cache_slot: *mut PicCacheSlot, +) -> f64 { + get_field_ic_dispatch(obj_bits, key, site_id, cache_slot, true) +} + +/// The whole full-outline read ladder; `probe_mru` is false only on the cold +/// arm of the S2 split, whose leaf entry has already asked the MRU word. +#[inline(always)] +pub(super) fn get_field_ic_dispatch( + obj_bits: i64, + key: *const crate::StringHeader, + site_id: u64, + cache_slot: *mut PicCacheSlot, + probe_mru: bool, ) -> f64 { // POINTER_MASK: lower 48 bits — strips the NaN-box tag to a raw heap pointer. const POINTER_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; @@ -1476,7 +1489,7 @@ pub extern "C" fn js_object_get_field_ic( // values — a heap STRING's `+4` is a `StringHeader` field that rule 3 // deliberately does not bound. The kind test used to be what turned a // string away here; the tag does it now, one compare earlier. - if tag == 0x7FFD { + if probe_mru && tag == 0x7FFD { if let Some(value) = unsafe { pic_outlined_mru_hit(obj_handle, cache_slot) } { crate::typed_feedback::js_typed_feedback_record_guard_pass(site_id); return value; diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/outline_split.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/outline_split.rs new file mode 100644 index 0000000000..59c4872bec --- /dev/null +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/outline_split.rs @@ -0,0 +1,178 @@ +//! The full-outline generic property read, split into a GC-leaf hit and a +//! collecting miss (deferred-collection RFC, step S2). +//! +//! # Why this exists +//! +//! In a module over the full-outline threshold (#5391 path 3) every generic +//! `obj.prop` read is ONE call to [`js_object_get_field_ic`]. That helper can +//! run a getter, a Proxy trap or an allocating miss, so the call is an RS4GC +//! statepoint: every GC value live across it is spilled before the call and +//! reloaded after, on every read, including the ~95 % that are monomorphic +//! hits answered from the site's MRU word (`pic_outlined_mru_hit`). On the +//! claude-code bundle that is 191,964 call sites. +//! +//! The split moves the hit into [`js_object_get_field_ic_fast`], a call that +//! reaches no collector entry, so codegen marks it `"gc-leaf-function"` and +//! RS4GC leaves it alone. Only its decline arm calls +//! [`js_object_get_field_ic_fast_miss`], which is still a statepoint. +//! +//! # Why the fast entry is a Perry-GC leaf on today's runtime +//! +//! Its whole call graph, read at the source (the S1 call-graph checker is the +//! authority; this is what it has to agree with): +//! +//! * a tag compare and a mask on the receiver bits; +//! * `pic_outlined_mru_hit`: `outlined_mru_hit_enabled` (a `OnceLock` +//! whose first call reads one environment variable into Rust-heap memory, +//! which cannot arm a Perry trigger), the handle-band compare, +//! `pic_slot_peek` (one acquire load, never `pic_slot_resolve`, which is the +//! allocating variant), `object_shape_stamp` (one header load), two cache +//! word compares and one slot load; +//! * `typed_feedback_active` (one static read). +//! +//! It makes NO typed-feedback call. With feedback on, the ladder's observe and +//! guard-pass record take the feedback registry lock, a `GcRootRegistryGuard` +//! whose drop can flush a deferred collection request (#11523) — the census +//! call graph reaches the collector exactly there. So under feedback the fast +//! entry declines every read and the continuation records it; with feedback +//! off those two calls are early returns, and skipping them changes nothing. +//! +//! No Perry allocation, no `GcRootRegistryGuard`, no throw, no call into +//! generated code, no poll. Everything it cannot serve — a non-POINTER +//! receiver (SSO, class ref, nullish, primitive, heap string), an unprimed or +//! mismatched MRU word, an overflow slot, the Array-subclass word, a +//! polymorphic way — answers `TAG_HOLE`. +//! +//! # Why the pair is behaviourally identical to the one call +//! +//! `TAG_HOLE` is unambiguous: #10826 makes every delete a ShapeId transition, +//! so a stamp hit never reads a hole. A hit is served only with feedback off, +//! where the ladder's two feedback calls are no-ops; the miss continuation is +//! the unchanged ladder with only the MRU probe skipped when the fast entry +//! already asked it (feedback off). A receiver the fast entry declined for its +//! tag never reached the MRU probe in the old ladder either. + +use super::ic_miss::{get_field_ic_dispatch, pic_outlined_mru_hit}; +use crate::object::{ObjectHeader, PicCacheSlot}; + +/// The GC-leaf hit of the full-outline generic read. Answers the slot value on +/// an MRU hit and `TAG_HOLE` for everything else; the caller then calls +/// [`js_object_get_field_ic_fast_miss`] with the same operands. +/// +/// Same operands as [`super::js_object_get_field_ic`], so the emitted miss +/// arm passes them through unchanged. +#[no_mangle] +pub extern "C" fn js_object_get_field_ic_fast( + obj_bits: i64, + key: *const crate::StringHeader, + site_id: u64, + cache_slot: *mut PicCacheSlot, +) -> f64 { + let _ = (key, site_id); + if crate::typed_feedback::typed_feedback_active() { + return f64::from_bits(crate::value::TAG_HOLE); + } + mru_hit_or_hole(obj_bits, cache_slot) +} + +/// The fast entry's hit, without the feedback gate (unit tests run with +/// feedback forced on). +#[inline(always)] +fn mru_hit_or_hole(obj_bits: i64, cache_slot: *mut PicCacheSlot) -> f64 { + const POINTER_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; + let bits = obj_bits as u64; + // POINTER tag only, exactly as the full ladder admits the MRU probe + // (#10833): a heap STRING's `+4` is not a ShapeId word. + if bits >> 48 != 0x7FFD { + return f64::from_bits(crate::value::TAG_HOLE); + } + let obj_handle = (bits & POINTER_MASK) as usize as *const ObjectHeader; + match unsafe { pic_outlined_mru_hit(obj_handle, cache_slot) } { + Some(value) => value, + None => f64::from_bits(crate::value::TAG_HOLE), + } +} + +/// The collecting miss continuation of [`js_object_get_field_ic_fast`]: the +/// complete full-outline ladder minus the MRU probe the fast entry already +/// made. A statepoint, like the single call it replaces. +#[no_mangle] +pub extern "C" fn js_object_get_field_ic_fast_miss( + obj_bits: i64, + key: *const crate::StringHeader, + site_id: u64, + cache_slot: *mut PicCacheSlot, +) -> f64 { + // Under feedback the fast entry asked nothing, so the probe runs here + // (and records its observe and guard pass, as the single helper did). + let probe_mru = crate::typed_feedback::typed_feedback_active(); + get_field_ic_dispatch(obj_bits, key, site_id, cache_slot, probe_mru) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::object::{PicCache, PIC_CACHE_WORDS}; + + fn key_of(bytes: &[u8]) -> *const crate::StringHeader { + crate::string::js_string_from_bytes(bytes.as_ptr(), bytes.len() as u32) + } + + fn boxed(obj: *mut ObjectHeader) -> i64 { + (0x7FFD_u64 << 48 | obj as u64) as i64 + } + + /// The pair answers what the single helper answers, and the fast entry + /// declines exactly the reads it cannot serve from the MRU word. + #[test] + fn fast_hit_and_miss_continuation_match_the_single_helper() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 4)); + let k = scope.root_string_ptr(key_of(b"split_present")); + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, 7.0)) + }); + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let slot_ptr: *mut PicCacheSlot = &mut slot; + + obj.with_mut_ptr(|o| { + k.with_const_ptr(|kp| { + // Unit tests run with typed feedback forced ON, where the + // fast entry must decline every read (its hit would skip the + // observe the ladder records) and the continuation answers. + assert!(crate::typed_feedback::typed_feedback_active()); + let declined = js_object_get_field_ic_fast(boxed(o), kp, 0, slot_ptr); + assert_eq!(declined.to_bits(), crate::value::TAG_HOLE); + assert_eq!( + js_object_get_field_ic_fast_miss(boxed(o), kp, 0, slot_ptr), + 7.0 + ); + // The hit itself (the feedback-off path): primed by the + // continuation above, served from the MRU word. + assert_eq!( + mru_hit_or_hole(boxed(o), slot_ptr), + 7.0, + "a primed monomorphic read must be served by the leaf hit" + ); + assert_eq!( + super::super::js_object_get_field_ic(boxed(o), kp, 0, slot_ptr), + 7.0 + ); + // Non-POINTER receivers never reach the probe. + for recv in [ + crate::value::TAG_UNDEFINED as i64, + crate::value::TAG_NULL as i64, + (0x7FFE_u64 << 48 | 3) as i64, + 1.5f64.to_bits() as i64, + ] { + assert_eq!( + mru_hit_or_hole(recv, slot_ptr).to_bits(), + crate::value::TAG_HOLE + ); + } + }) + }); + } +} diff --git a/crates/perry-runtime/src/proxy/put_value/packed_set.rs b/crates/perry-runtime/src/proxy/put_value/packed_set.rs index 93a271080b..ba452eef8d 100644 --- a/crates/perry-runtime/src/proxy/put_value/packed_set.rs +++ b/crates/perry-runtime/src/proxy/put_value/packed_set.rs @@ -256,6 +256,40 @@ pub extern "C" fn js_put_value_set_packed_miss( result } +/// S2 of the deferred-collection RFC: the GC-leaf hit of a full-outline +/// static-key store. Serves an existing key from an INLINE way — the +/// receiver test, a ShapeId compare, `packed_hit_receiver_ok` (header reads) +/// and `store_object_field_slot` (`runtime_store_jsvalue_slot`: addref, +/// layout note, slot barrier) — and answers `TAG_HOLE` for everything else: +/// an unprimed site, a spill way (`dyn_ic_try_store` is not audited leaf), a +/// key add (the add memo can allocate), a refused receiver. The emitted code +/// then calls [`js_put_value_set_packed_miss`] with its usual operands, +/// which re-asks the same ways (a declined way declines again) and runs the +/// full `[[Set]]`. Nothing here allocates, throws or runs user code, and it +/// declines everything while typed feedback is on (see the body). +#[no_mangle] +pub extern "C" fn js_put_value_set_packed_fast( + target: f64, + value: f64, + cache_slot: *mut PackedSetWaysSlot, +) -> f64 { + // With typed feedback on, the store's layout note can retire feedback + // through the registry lock — a `GcRootRegistryGuard` whose release can + // flush a deferred collection (#11523). Decline; the miss entry stores. + if crate::typed_feedback::typed_feedback_active() { + return f64::from_bits(crate::value::TAG_HOLE); + } + unsafe { + let cache = crate::object::pic_slot_peek(cache_slot); + if cache.is_null() { + return f64::from_bits(crate::value::TAG_HOLE); + } + let ways = &*(cache as *const [AtomicU64; PACKED_SET_WAYS]); + packed_ways_store_impl(ways, 0, target, value, false) + .unwrap_or(f64::from_bits(crate::value::TAG_HOLE)) + } +} + /// Serve `target` from the runtime-compared ways (and any spill way), with /// the same per-object tests and barriers as the emitted hit. /// @@ -266,6 +300,19 @@ unsafe fn packed_ways_store( first_way: usize, target: f64, value: f64, +) -> Option { + packed_ways_store_impl(ways, first_way, target, value, true) +} + +/// `packed_ways_store`, optionally declining a spill way instead of serving it +/// through `dyn_ic_try_store` (the S2 leaf entry below serves inline ways only). +#[inline(always)] +unsafe fn packed_ways_store_impl( + ways: &[AtomicU64; PACKED_SET_WAYS], + first_way: usize, + target: f64, + value: f64, + serve_spill: bool, ) -> Option { let bits = target.to_bits(); // The emitted receiver test: POINTER tag and a payload above the handle @@ -293,6 +340,9 @@ unsafe fn packed_ways_store( return Some(value); } if stamp ^ SPILL_FLIP == sid { + if !serve_spill { + return None; + } let token = crate::object::shapes::PIC_ID_TOKEN_BIT | sid as u64; return dyn_ic_try_store(target, token, index | IC_SLOT_OVERFLOW_BIT, value); } diff --git a/crates/perry-runtime/src/typed_feedback/guards.rs b/crates/perry-runtime/src/typed_feedback/guards.rs index 616955ab75..280a9e2fbe 100644 --- a/crates/perry-runtime/src/typed_feedback/guards.rs +++ b/crates/perry-runtime/src/typed_feedback/guards.rs @@ -827,6 +827,16 @@ pub extern "C" fn js_class_field_get_ic( } } + class_field_get_after_guard_fail(site_id, receiver, key) +} + +/// `js_class_field_get_ic`'s guard-FAIL arm, shared with the S2 miss +/// continuation [`js_class_field_get_ic_fast_miss`]. +fn class_field_get_after_guard_fail( + site_id: u64, + receiver: f64, + key: *const crate::StringHeader, +) -> f64 { crate::typed_feedback::js_typed_feedback_record_fallback_call(site_id); let obj_bits = receiver.to_bits(); // #7153: this function is the full-outline of the codegen class-field-get @@ -853,6 +863,225 @@ pub extern "C" fn js_class_field_get_ic( ) } +// --------------------------------------------------------------------------- +// S2 of the deferred-collection RFC: the full-outline class-field IC split +// into a GC-leaf hit and a collecting miss. +// +// `js_class_field_get_ic` / `js_class_field_set_ic` can run a getter or setter +// (the by-name fallback), so each full-outline site is a statepoint that +// spills and reloads every live GC value on every access. The `_fast` entries +// below serve the guard-PASS slot access; everything else is declined and the +// emitted cold arm calls the `_fast_miss` continuation, which does exactly +// what the full helper does from that point, so the guard's effects happen +// exactly once. +// +// They serve ONLY while typed feedback is off and no property descriptor is in +// use (`descriptors_in_use`). That is precisely when the class-field guards +// take `class_field_fast_contract` / `class_field_set_fast_contract`, which +// neither observe nor walk descriptors. The other arm is not a Perry-GC leaf +// on today's runtime, per the census call graph: the observe takes the +// feedback registry lock, a `GcRootRegistryGuard` whose drop can flush a +// deferred collection request (#11523), and the descriptor walk +// (`get_accessor_descriptor` / `get_property_attrs`) contains an indirect call. +// Under either condition the fast entry declines without evaluating anything +// and the continuation runs the whole helper. +// +// What the fast entries do reach (the S1 checker is the authority; this is +// what it must agree with): two static reads, the fast contract (GC-header and +// shape-descriptor reads, the typed-layout side table, and a verify-mode +// `abort`), then one slot load, a raw-f64 `ptr::write`, or +// `runtime_store_jsvalue_slot` (typed-slot canonicalization, `js_string_addref`, +// `layout_note_slot`, the slot write barrier). Excluded as well, because each +// can collect or re-enter: `js_object_set_field`'s two diagnostics (formatting +// is an indirect call; the census seeds `js_object_set_field` there) — a +// null-POINTER value is declined before anything runs — and +// `set_object_live_slot_count`, which mints a shape descriptor: a store that +// would widen the live bound is declined after the guard (status 3) and stored +// through `js_object_set_field` on the cold arm. The contract's +// `expected_field_index < live_inline_slot_count` makes that unreachable today; +// it is declined rather than assumed. +// --------------------------------------------------------------------------- + +/// Status of [`js_class_field_set_ic_fast`]: the store is done. +pub const CLASS_FIELD_SET_FAST_DONE: i32 = 1; +/// The guard ran and FAILED: continue with the by-name fallback. +pub const CLASS_FIELD_SET_FAST_GUARD_FAILED: i32 = 0; +/// Nothing ran (feedback or descriptors in use, or a null-POINTER value): +/// replay the whole full-outline helper. +pub const CLASS_FIELD_SET_FAST_NOT_ATTEMPTED: i32 = 2; +/// The guard PASSED but the store would widen the live bound: store through +/// `js_object_set_field`. +pub const CLASS_FIELD_SET_FAST_STORE_SLOW: i32 = 3; + +/// Whether the class-field guards run their side-effect-free, descriptor-free +/// contract — the only case the `_fast` entries serve. Both inputs are +/// set-only latches, so a decline observed by the fast entry is still a +/// decline when the continuation re-reads them. +#[inline(always)] +fn class_field_fast_entries_serve() -> bool { + !typed_feedback_enabled() && !crate::object::descriptors_in_use() +} + +/// GC-leaf hit of the full-outline class-field GET: the guard-PASS slot load, +/// or `TAG_HOLE` (the caller then calls [`js_class_field_get_ic_fast_miss`]). +/// A hole never sits in a slot the contract passes on (#10826: delete +/// transitions the ShapeId); if one did, the continuation's by-name read would +/// answer it correctly. +#[no_mangle] +pub extern "C" fn js_class_field_get_ic_fast( + site_id: u64, + receiver: f64, + expected_class_id: u32, + expected_shape_id: u32, + key: *const crate::StringHeader, + expected_field_index: u32, + require_raw_f64: i32, +) -> f64 { + let _ = (site_id, key); + if !class_field_fast_entries_serve() + || !class_field_fast_contract( + receiver, + expected_class_id, + expected_shape_id, + expected_field_index, + require_raw_f64 != 0, + ) + { + return f64::from_bits(crate::value::TAG_HOLE); + } + let object_addr = normalize_raw_object_addr(receiver.to_bits()); + unsafe { + let fields_ptr = + (object_addr as *const u8).add(std::mem::size_of::()) as *const f64; + std::ptr::read(fields_ptr.add(expected_field_index as usize)) + } +} + +/// Collecting continuation of [`js_class_field_get_ic_fast`]. When the fast +/// entry served nothing because feedback or descriptors are in use, this is +/// the whole `js_class_field_get_ic`; otherwise its (side-effect-free) guard +/// already failed and this is exactly that helper's guard-FAIL arm. +#[no_mangle] +pub extern "C" fn js_class_field_get_ic_fast_miss( + site_id: u64, + receiver: f64, + expected_class_id: u32, + expected_shape_id: u32, + key: *const crate::StringHeader, + expected_field_index: u32, + require_raw_f64: i32, +) -> f64 { + if !class_field_fast_entries_serve() { + return js_class_field_get_ic( + site_id, + receiver, + expected_class_id, + expected_shape_id, + key, + expected_field_index, + require_raw_f64, + ); + } + class_field_get_after_guard_fail(site_id, receiver, key) +} + +/// GC-leaf hit of the full-outline class-field SET; returns one of the +/// `CLASS_FIELD_SET_FAST_*` statuses. On anything but `DONE` the caller calls +/// [`js_class_field_set_ic_fast_miss`] with the status and the same operands. +#[no_mangle] +pub extern "C" fn js_class_field_set_ic_fast( + site_id: u64, + receiver: f64, + expected_class_id: u32, + expected_shape_id: u32, + key: *const crate::StringHeader, + expected_field_index: u32, + value: f64, + require_raw_f64: i32, +) -> i32 { + let _ = (site_id, key); + let vbits = value.to_bits(); + if !class_field_fast_entries_serve() + || ((vbits >> 48) == 0x7FFD && (vbits & crate::value::POINTER_MASK) == 0) + { + return CLASS_FIELD_SET_FAST_NOT_ATTEMPTED; + } + if !class_field_set_fast_contract( + receiver, + expected_class_id, + expected_shape_id, + expected_field_index, + require_raw_f64 != 0, + vbits, + ) { + return CLASS_FIELD_SET_FAST_GUARD_FAILED; + } + let object_addr = normalize_raw_object_addr(receiver.to_bits()); + unsafe { + let fields_ptr = + (object_addr as *mut u8).add(std::mem::size_of::()) as *mut f64; + let slot = fields_ptr.add(expected_field_index as usize); + if require_raw_f64 != 0 { + // GC_STORE_AUDIT(POINTER_FREE): identical to `js_class_field_set_ic`'s + // raw-f64 arm — a passing guard proved the slot pointer-free. + std::ptr::write(slot, value); + return CLASS_FIELD_SET_FAST_DONE; + } + let obj = object_addr as *mut ObjectHeader; + if expected_field_index >= crate::object::object_live_slot_count(obj) { + return CLASS_FIELD_SET_FAST_STORE_SLOW; + } + // `js_object_set_field`'s store for an in-bound index and a value that + // is not a null POINTER (both established above). + crate::gc::runtime_store_jsvalue_slot( + object_addr, + slot as usize, + expected_field_index as usize, + vbits, + ); + } + CLASS_FIELD_SET_FAST_DONE +} + +/// Collecting continuation of [`js_class_field_set_ic_fast`], dispatched on +/// its status so that every path is what `js_class_field_set_ic` would have +/// done, with the guard evaluated once overall. +#[no_mangle] +pub extern "C" fn js_class_field_set_ic_fast_miss( + status: i32, + site_id: u64, + receiver: f64, + expected_class_id: u32, + expected_shape_id: u32, + key: *const crate::StringHeader, + expected_field_index: u32, + value: f64, + require_raw_f64: i32, +) { + match status { + CLASS_FIELD_SET_FAST_GUARD_FAILED => { + let key_raw = key as u64 & crate::value::POINTER_MASK; + js_class_field_set_fallback(site_id, receiver.to_bits(), key_raw, value); + } + CLASS_FIELD_SET_FAST_STORE_SLOW => crate::object::js_object_set_field( + normalize_raw_object_addr(receiver.to_bits()) as *mut ObjectHeader, + expected_field_index, + crate::value::JSValue::from_bits(value.to_bits()), + ), + CLASS_FIELD_SET_FAST_DONE => {} + _ => js_class_field_set_ic( + site_id, + receiver, + expected_class_id, + expected_shape_id, + key, + expected_field_index, + value, + require_raw_f64, + ), + } +} + #[no_mangle] pub unsafe extern "C-unwind" fn js_typed_feedback_native_call_method( site_id: u64, diff --git a/crates/perry/tests/ic_fast_split_slow_path_evacuation.rs b/crates/perry/tests/ic_fast_split_slow_path_evacuation.rs new file mode 100644 index 0000000000..53974496fa --- /dev/null +++ b/crates/perry/tests/ic_fast_split_slow_path_evacuation.rs @@ -0,0 +1,169 @@ +//! S2 of the deferred-collection RFC: a fast/slow split may leave only the +//! FAST call without a statepoint. Its slow arm runs user code — a getter, a +//! Proxy trap, a setter, a `toString`, a not-callable throw — and that code +//! allocates enough to move the caller's live heap values. If the slow call +//! were a GC leaf as well, RS4GC would record no stack map at it, the walker +//! would skip the caller's frame, and the caller's values would still name +//! from-space when it reads them afterwards. +//! +//! The fixture is `test-files/test_gap_ic_fast_split_slow_path.ts` (also a gap +//! test, against node). Here it is compiled with `PERRY_FULL_OUTLINE_IC=1`, +//! where the four IC splits apply, and run under a seeded evacuating schedule +//! with the from-space quarantine and the evacuation verifier on. The run must +//! print node's output AND report copying minors that moved objects — a green +//! run with zero moves would say nothing. +//! +//! Sabotage (run for the PR, 2026-09-27): classifying the four slow +//! continuations `CannotCollect` makes the evacuating run fault in retired +//! from-space (`[gc-fromspace-protect] FAULT`, obj_type=1). + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +fn perry_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_perry")) +} + +const SOURCE: &str = include_str!("../../../test-files/test_gap_ic_fast_split_slow_path.ts"); + +/// node 26.5.1 (`.node-version`) on `SOURCE`. +const NODE_ORACLE: &str = "keep-g:123:1\nkeep-g:123:1\nkeep-g:123:2\nkeep-g:123:3\nkeep-g:123:1\n\ +keep-w23 30\nkeep-w23 31\nkeep-w23 32\nkeep-w23 33\nkeep-w23 34\n\ +keep-cg7:10\nkeep-cg7:10\nkeep-cg7:20\nkeep-cg7:10\n\ +keep-cs45 11\nkeep-cs45 21\nkeep-cs45 12\n\ +keep-t6:s\nkeep-t6:42\nkeep-t6:T\nkeep-t6:s2\n\ +no-throw:keep-c\ncaught:true:keep-c89\nno-throw:keep-c\n"; + +const GC_ENV: &[&str] = &[ + "PERRY_GEN_GC", + "PERRY_GC_MOVING_LOOP_POLLS", + "PERRY_GC_SCHEDULE_SEED", + "PERRY_GC_SCHEDULE_RATE", + "PERRY_GC_SCHEDULE_ALLOC_KB", + "PERRY_GC_FORCE_EVACUATE", + "PERRY_GC_VERIFY_EVACUATION", + "PERRY_GC_PROTECT_FROMSPACE", + "PERRY_GC_PROTECT_FROMSPACE_DEPTH", + "PERRY_CONSERVATIVE_STACK_SCAN", +]; + +fn compile(dir: &Path) -> (PathBuf, String) { + let entry = dir.join("main.ts"); + let output = dir.join("main_bin"); + std::fs::write(&entry, SOURCE).expect("write entry"); + let mut cmd = Command::new(perry_bin()); + cmd.current_dir(dir) + .arg("compile") + .arg(&entry) + .arg("-o") + .arg(&output) + .env("PERRY_NO_CACHE", "1") + .env("PERRY_NO_AUTO_OPTIMIZE", "1") + .env("PERRY_GC_INSTRUMENTS", "1") + .env("PERRY_FULL_OUTLINE_IC", "1") + .env("PERRY_LLVM_KEEP_IR", "1"); + for key in GC_ENV { + cmd.env_remove(key); + } + let compile = cmd.output().expect("run perry compile"); + assert!( + compile.status.success(), + "perry compile failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&compile.stdout), + String::from_utf8_lossy(&compile.stderr) + ); + ( + output, + String::from_utf8_lossy(&compile.stderr).into_owned(), + ) +} + +fn kept_ir(stderr: &str) -> String { + stderr + .lines() + .filter_map(|line| line.split("kept LLVM IR: ").nth(1)) + .map(|p| std::fs::read_to_string(p.trim()).expect("read kept LLVM IR")) + .collect::>() + .join("\n") +} + +fn run(bin: &Path, dir: &Path, scheduled: bool) -> Output { + let mut cmd = Command::new(bin); + cmd.current_dir(dir); + for key in GC_ENV { + cmd.env_remove(key); + } + if scheduled { + cmd.env("PERRY_GC_SCHEDULE_SEED", "11528") + .env("PERRY_GC_SCHEDULE_RATE", "1") + .env("PERRY_GC_FORCE_EVACUATE", "1") + .env("PERRY_GC_VERIFY_EVACUATION", "1") + .env("PERRY_GC_PROTECT_FROMSPACE", "1") + .env("PERRY_GC_PROTECT_FROMSPACE_DEPTH", "64"); + } + cmd.output().expect("run compiled binary") +} + +fn verdict_field(stderr: &str, field: &str) -> u64 { + stderr + .lines() + .rev() + .filter(|line| line.contains("[gc-schedule]")) + .find_map(|line| { + line.split_ascii_whitespace() + .find_map(|part| part.strip_prefix(&format!("{field}="))) + .and_then(|value| value.parse().ok()) + }) + .unwrap_or_else(|| panic!("scheduled run reported no numeric {field}\n{stderr}")) +} + +#[test] +fn slow_arms_that_run_allocating_user_code_keep_the_callers_values_relocated() { + let dir = tempfile::tempdir().expect("tempdir"); + let (bin, compile_stderr) = compile(dir.path()); + + // The subject must be in the artifact: every split, both halves. + let ir = kept_ir(&compile_stderr); + for symbol in [ + "@js_object_get_field_ic_fast(", + "@js_object_get_field_ic_fast_miss(", + "@js_put_value_set_packed_fast(", + "@js_put_value_set_packed_miss(", + "@js_class_field_get_ic_fast(", + "@js_class_field_get_ic_fast_miss(", + "@js_class_field_set_ic_fast(", + "@js_class_field_set_ic_fast_miss(", + "@js_template_string_coerce_box(", + "@js_closure_unbox_callee_checked(", + ] { + assert!( + ir.lines() + .any(|l| l.contains(symbol) && l.contains("call ") && !l.contains("declare ")), + "the fixture no longer exercises `{symbol}`" + ); + } + + let plain = run(&bin, dir.path(), false); + assert!( + plain.status.success(), + "plain run failed\nstderr:\n{}", + String::from_utf8_lossy(&plain.stderr) + ); + assert_eq!(String::from_utf8_lossy(&plain.stdout), NODE_ORACLE); + + let scheduled = run(&bin, dir.path(), true); + let stderr = String::from_utf8_lossy(&scheduled.stderr); + assert!( + scheduled.status.success(), + "evacuating run failed (exit {:?})\nstdout:\n{}\nstderr:\n{stderr}", + scheduled.status.code(), + String::from_utf8_lossy(&scheduled.stdout) + ); + assert_eq!(String::from_utf8_lossy(&scheduled.stdout), NODE_ORACLE); + for field in ["copying_minors", "moved_objects"] { + assert!( + verdict_field(&stderr, field) > 0, + "the evacuating run never exercised {field}: a green run proves nothing\n{stderr}" + ); + } +} diff --git a/scripts/gc_root_dominance_check.py b/scripts/gc_root_dominance_check.py index d101ab8822..cb6185b3a8 100755 --- a/scripts/gc_root_dominance_check.py +++ b/scripts/gc_root_dominance_check.py @@ -485,6 +485,11 @@ def build_cfg(f): # per-thread table probe plus one load through the holder; no allocation, # no user code, no chain walk (declines answer TAG_HOLE). "js_inherited_read_cache_hit_f64", + # S2 GC-leaf IC hits (`expr/ic_fast_split.rs`); audit in gc_call_effects.rs. + "js_object_get_field_ic_fast", + "js_class_field_get_ic_fast", + "js_class_field_set_ic_fast", + "js_put_value_set_packed_fast", "js_transition_ic_spill_append", "js_write_barrier_slot", "js_gc_register_global_root", diff --git a/test-files/test_gap_ic_fast_split_slow_path.ts b/test-files/test_gap_ic_fast_split_slow_path.ts new file mode 100644 index 0000000000..d962915f7b --- /dev/null +++ b/test-files/test_gap_ic_fast_split_slow_path.ts @@ -0,0 +1,143 @@ +// S2 (deferred-collection RFC): the slow arm of every fast/slow split runs +// user code that allocates heavily -- a getter, a Proxy trap, a setter, a +// `toString`, a not-callable throw -- while the caller holds live heap values +// that it reads afterwards. Under a moving collection those values must come +// back relocated. The Rust test `ic_fast_split_slow_path_evacuation.rs` +// compiles this file with PERRY_FULL_OUTLINE_IC=1 (where the IC splits apply) +// and runs it with a seeded evacuating schedule. +export {}; + +function churn(n: number): number { + let sink = 0; + for (let i = 0; i < n; i++) { + const t = { i, s: "c" + i, a: [i, i + 1] }; + sink += t.a[1] - t.i; + } + return sink; +} + +const N = 3000; + +class Point { + x: any; + constructor(v: any) { + this.x = v; + } +} + +function readGeneric(o: any): string { + const keep = { tag: "keep-g", arr: [1, 2, 3] }; + const v = o.foo; + return keep.tag + ":" + keep.arr.join("") + ":" + v.val; +} + +function writeGeneric(o: any, v: any): string { + const keep = { tag: "keep-w", arr: [2, 3] }; + o.bar = v; + return keep.tag + keep.arr.join(""); +} + +function readField(p: Point): string { + const keep = { tag: "keep-cg", n: 7 }; + const v = p.x; + return keep.tag + keep.n + ":" + v.val; +} + +function writeField(p: Point, v: any): string { + const keep = { tag: "keep-cs", arr: [4, 5] }; + p.x = v; + return keep.tag + keep.arr.join(""); +} + +function tmpl(x: any): string { + const keep = { q: "keep-t", arr: [6] }; + const s = `${x}`; + return keep.q + keep.arr[0] + ":" + s; +} + +function callIt(f: any): string { + const keep = { z: "keep-c", arr: [8, 9] }; + try { + f(1); + } catch (e) { + churn(N); + return "caught:" + (e instanceof TypeError) + ":" + keep.z + keep.arr.join(""); + } + return "no-throw:" + keep.z; +} + +// Generic read: a plain data property (the fast hit, twice so the site is +// primed), then a getter and a Proxy trap on the slow arm. +const data = { foo: { val: 1 } }; +const getter = { + get foo() { + churn(N); + return { val: 2 }; + }, +}; +const proxy = new Proxy({} as any, { + get(_t: any, k: any) { + churn(N); + return { val: k === "foo" ? 3 : -1 }; + }, +}); +for (const o of [data, data, getter, proxy, data]) console.log(readGeneric(o)); + +// Generic write: an existing key (primed, then the fast way store), a setter +// and a Proxy `set` trap on the slow arm. +const wdata: any = { bar: 0 }; +let wstored: any = null; +const setter = { + set bar(v: any) { + churn(N); + wstored = v; + }, +}; +const wproxy = new Proxy({} as any, { + set(_t: any, _k: any, v: any) { + churn(N); + wstored = v; + return true; + }, +}); +console.log(writeGeneric(wdata, { val: 30 }), wdata.bar.val); +console.log(writeGeneric(wdata, { val: 31 }), wdata.bar.val); +console.log(writeGeneric(setter, { val: 32 }), wstored.val); +console.log(writeGeneric(wproxy, { val: 33 }), wstored.val); +console.log(writeGeneric(wdata, { val: 34 }), wdata.bar.val); + +// Class field: a plain instance (the guard passes) and one whose `x` is an +// own accessor (the guard fails; the by-name fallback runs the accessor). +const plain = new Point({ val: 10 }); +const accessor = new Point({ val: 0 }); +let stored: any = null; +Object.defineProperty(accessor, "x", { + get() { + churn(N); + return { val: 20 }; + }, + set(v: any) { + churn(N); + stored = v; + }, + configurable: true, +}); +for (const p of [plain, plain, accessor, plain]) console.log(readField(p)); +console.log(writeField(plain, { val: 11 }), plain.x.val); +console.log(writeField(accessor, { val: 21 }), stored.val); +console.log(writeField(plain, { val: 12 }), plain.x.val); + +// Template coercion: a string (inline), a number and an object whose +// `toString` allocates (cold arm). +const heavy = { + toString() { + churn(N); + return "T"; + }, +}; +for (const x of ["s", 42, heavy, "s2"]) console.log(tmpl(x)); + +// Dynamic callee: a function (inline unbox), then a number (cold arm throws). +console.log(callIt((n: number) => n)); +console.log(callIt(42)); +console.log(callIt((n: number) => n + 1));