diff --git a/changelog.d/11177-closure-box-layout.md b/changelog.d/11177-closure-box-layout.md new file mode 100644 index 0000000000..73ce83f275 --- /dev/null +++ b/changelog.d/11177-closure-box-layout.md @@ -0,0 +1,3 @@ +Share compiler-declared boxed-capture layouts by function pointer. Generated closures now register a constant bitmap in one batch and retain only a weak owner address, replacing the per-instance capture-index/cell list. Fresh boxed closures also use bulk capture initialization. Per-box lifetime counts remain in place so frame and async release still wait for the final escaped capture. + +GC tracing, relocation, death pruning, singleton reuse, and runtime clones use the shared layout. The existing dynamic setter retains its exact-edge representation for compatibility. Regression coverage includes sparse layouts beyond 64 slots, duplicate edges, dynamic replacement, clone rebinding, released payload tracing, and wide/async compiled closures. diff --git a/crates/perry-codegen/src/expr/closure.rs b/crates/perry-codegen/src/expr/closure.rs index 7abb8bb0a8..27727122cc 100644 --- a/crates/perry-codegen/src/expr/closure.rs +++ b/crates/perry-codegen/src/expr/closure.rs @@ -270,8 +270,8 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // is created per-call but always with the same `this` (the // World) and same captures (`this._changeset`). // Boxed captures may use the bulk cache helper, but codegen still - // follows it with `js_closure_set_box_capture_ptr` for only those - // slots. The idempotent write declares exact lifetime edges + // follows it with `js_closure_register_box_layout`. The idempotent + // registration declares exact lifetime edges // without guessing from arbitrary pointer-shaped values. // // IDENTITY CAVEAT (#4831 follow-up — Stripe `protoExtend`): @@ -370,16 +370,12 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { None }; - // Bulk-init admission: a fresh closure whose captures are all plain - // bits. Box-cell captures keep the per-slot setter path — their - // `set_closure_box_capture` bookkeeping has no bulk twin. + // Fresh closures can initialize all captures in bulk. Box lifetime + // edges are registered afterward, independently of the stores. let bulk_fresh_init = !no_capture_singleton && !captured_singleton && total_caps > 0 - && !captured_value_bits.is_empty() - && auto_captures.iter().all(|cap_id| { - !ctx.boxed_vars.contains(cap_id) || uncounted_box_capture(cap_id) - }); + && !captured_value_bits.is_empty(); let closure_handle = if no_capture_singleton { let blk = ctx.block(); blk.call(I64, "js_closure_alloc_singleton", &[(PTR, &func_ref)]) @@ -460,41 +456,52 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { .call_void("js_register_closure_async_function", &[(PTR, &func_ref)]); } - // The captured-singleton helper writes captures internally. Boxed - // slots still take the dedicated, idempotent setter afterward so - // their lifetime edges are declared; fresh closures need every - // slot initialized here. The compiler-private plain-async step - // closure is different: its activation refcount already covers - // every queued/running instance of its OWN cells, so declaring its - // whole boxed frame as escaped would delay every terminal cell - // until a full GC. User closures nested inside it still take the - // dedicated setter and therefore preserve #8213's escaped-cell - // lifetime, and so does a step closure's capture of an enclosing - // scope's cell (#10464). - let tracked_box_capture_slots = auto_captures - .iter() - .map(|cap_id| ctx.boxed_vars.contains(cap_id) && !uncounted_box_capture(cap_id)) - .collect::>(); - let blk = ctx.block(); - for (idx, val_bits) in captured_value_bits.iter().enumerate() { - let track_box_capture = tracked_box_capture_slots[idx]; - if bulk_fresh_init { - // Every slot was written by `js_closure_alloc_init`. - continue; + // Lifetime edges are declared once after initialization, using a + // function-wide immutable bitmap. The async step's own cells are + // retained by its activation; only enclosing cells escape here. + let mut box_mask = vec![0u64; auto_captures.len().div_ceil(64)]; + for (index, cap_id) in auto_captures.iter().enumerate() { + if ctx.boxed_vars.contains(cap_id) && !uncounted_box_capture(cap_id) { + box_mask[index / 64] |= 1 << (index % 64); } - if !captured_singleton || track_box_capture { - let idx_str = idx.to_string(); - let setter = if track_box_capture { - "js_closure_set_box_capture_ptr" - } else { - "js_closure_set_capture_bits" - }; - blk.call_void( - setter, - &[(I64, &closure_handle), (I32, &idx_str), (I64, val_bits)], + } + if !bulk_fresh_init && !captured_singleton { + for (idx, val_bits) in captured_value_bits.iter().enumerate() { + ctx.block().call_void( + "js_closure_set_capture_bits", + &[ + (I64, &closure_handle), + (I32, &idx.to_string()), + (I64, val_bits), + ], ); } } + if box_mask.iter().any(|&word| word != 0) { + let name = format!( + "perry_box_layout_{}_{}", + ctx.strings.module_prefix(), + ctx.ic_site_counter + ); + ctx.ic_site_counter += 1; + let words = box_mask + .iter() + .map(|word| format!("i64 {word}")) + .collect::>() + .join(", "); + ctx.typed_parse_rodata.push(format!( + "@{name} = private unnamed_addr constant [{} x i64] [{words}]", + box_mask.len(), + )); + ctx.block().call_void( + "js_closure_register_box_layout", + &[ + (I64, &closure_handle), + (PTR, &format!("@{name}")), + (I32, &box_mask.len().to_string()), + ], + ); + } // Issue #291: when the closure is built inside a method // body (or constructor), the enclosing frame's `this` is the // topmost entry on `this_stack`; load and write that into diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 14fd13db58..461a9791bc 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -674,7 +674,7 @@ pub(crate) struct FnCtx<'a> { pub resolved_versioned_loop_callback_targets: std::collections::HashMap<(u32, usize), String>, /// This is an internal clone of a compiler-proven direct arrow body. Its /// boxed capture slots were installed through - /// `js_closure_set_box_capture_ptr`, so captured-box accesses may use the + /// `js_closure_register_box_layout`, so captured-box accesses may use the /// raw helpers. Public and dynamically dispatched closure bodies keep the /// defensive runtime registry validation. pub trusted_box_captures: bool, diff --git a/crates/perry-codegen/src/gc_call_effects.rs b/crates/perry-codegen/src/gc_call_effects.rs index 49e6c8ef0c..25ea77b53a 100644 --- a/crates/perry-codegen/src/gc_call_effects.rs +++ b/crates/perry-codegen/src/gc_call_effects.rs @@ -231,6 +231,7 @@ pub(crate) fn classify_direct_callee(name: &str) -> GcCallEffect { | "js_closure_get_capture_bits" | "js_closure_set_capture_bits" | "js_closure_set_box_capture_ptr" + | "js_closure_register_box_layout" | "js_closure_get_capture_ptr" | "js_closure_set_capture_ptr" // Variable-box accessors and allocators (#8132), `box.rs`. Boxes are @@ -836,6 +837,7 @@ mod tests { "js_closure_get_capture_bits", "js_closure_set_capture_bits", "js_closure_set_box_capture_ptr", + "js_closure_register_box_layout", "js_closure_get_capture_ptr", "js_closure_set_capture_ptr", "js_box_alloc_bits", diff --git a/crates/perry-codegen/src/root_reload.rs b/crates/perry-codegen/src/root_reload.rs index 83c06c223a..21f0636d89 100644 --- a/crates/perry-codegen/src/root_reload.rs +++ b/crates/perry-codegen/src/root_reload.rs @@ -228,6 +228,7 @@ const NON_COLLECTING: &[&str] = &[ // verified non-allocating bookkeeping stores/reads "js_closure_set_capture_bits", "js_closure_set_box_capture_ptr", + "js_closure_register_box_layout", "js_closure_get_capture_bits", "js_closure_set_capture_ptr", "js_closure_get_capture_ptr", diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index 12d0a42999..87be827d66 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -225,6 +225,7 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { ); module.declare_function("js_closure_set_capture_bits", VOID, &[I64, I32, I64]); module.declare_function("js_closure_set_box_capture_ptr", VOID, &[I64, I32, I64]); + module.declare_function("js_closure_register_box_layout", VOID, &[I64, PTR, I32]); module.declare_function("js_closure_get_capture_bits", I64, &[I64, I32]); module.declare_function("js_closure_set_capture_f64", VOID, &[I64, I32, DOUBLE]); module.declare_function("js_closure_get_capture_f64", DOUBLE, &[I64, I32]); diff --git a/crates/perry-codegen/src/stmt/boxed_frame_release_tests.rs b/crates/perry-codegen/src/stmt/boxed_frame_release_tests.rs index 08f10d66ba..94dfac114f 100644 --- a/crates/perry-codegen/src/stmt/boxed_frame_release_tests.rs +++ b/crates/perry-codegen/src/stmt/boxed_frame_release_tests.rs @@ -135,7 +135,7 @@ fn captured_reassigned_let_is_released_at_every_return() { "premise: n is boxed\n{ir}" ); assert!( - ir.contains("js_closure_set_box_capture_ptr"), + ir.contains("js_closure_register_box_layout"), "premise: counted edge\n{ir}" ); let per_ret = releases_before_each_ret(&ir); @@ -280,7 +280,7 @@ fn plain_async_step_counts_only_enclosing_cells_and_frame_keeps_its_own() { "premise: both cells are minted by this frame:\n{ir}" ); assert_eq!( - ir.matches("call void @js_closure_set_box_capture_ptr(") + ir.matches("call void @js_closure_register_box_layout(") .count(), 1, "the enclosing cell is a counted edge, the activation's own is not:\n{ir}" @@ -291,3 +291,39 @@ fn plain_async_step_counts_only_enclosing_cells_and_frame_keeps_its_own() { "the frame releases OUTER only; OWN belongs to the activation ({per_ret:?}):\n{ir}" ); } + +#[test] +fn wide_box_captures_emit_one_constant_bitmap_and_one_registration() { + let ids: Vec = (100..166).collect(); + let mut body: Vec = ids + .iter() + .map(|&id| let_stmt(id, Expr::Integer(0))) + .collect(); + let nested_body = ids + .iter() + .map(|&id| Stmt::Expr(Expr::LocalSet(id, Box::new(Expr::Integer(1))))) + .collect(); + body.push(Stmt::Return(Some(closure(2, nested_body, ids)))); + let mut module = Module::new("wide_box_layout.ts"); + module + .functions + .push(function("wide_boxes", Vec::new(), body)); + let ir = String::from_utf8( + crate::compile_module(&module, super::prealloc_module_global_tests::ir_opts()).unwrap(), + ) + .unwrap(); + assert!( + ir.contains("constant [2 x i64] [i64 18446744073709551615, i64 3]"), + "{ir}" + ); + assert_eq!( + ir.matches("call void @js_closure_register_box_layout(") + .count(), + 1, + "{ir}" + ); + assert!( + !ir.contains("call void @js_closure_set_box_capture_ptr("), + "{ir}" + ); +} diff --git a/crates/perry-codegen/tests/native_proof_regressions.rs b/crates/perry-codegen/tests/native_proof_regressions.rs index 7264d380d1..20708856cb 100644 --- a/crates/perry-codegen/tests/native_proof_regressions.rs +++ b/crates/perry-codegen/tests/native_proof_regressions.rs @@ -7856,7 +7856,7 @@ fn boxed_local_slot_uses_i64_js_value_bits_until_helper_edges() { ); assert!( ir.contains("call i64 @js_closure_get_capture_bits") - && ir.contains("call void @js_closure_set_box_capture_ptr"), + && ir.contains("call void @js_closure_register_box_layout"), "generated boxed capture traffic should declare exact i64 box capture slots:\n{ir}" ); for old_helper in [ diff --git a/crates/perry-codegen/tests/release_boxes_lowering.rs b/crates/perry-codegen/tests/release_boxes_lowering.rs index 5aa95afc80..eb83c8f8f3 100644 --- a/crates/perry-codegen/tests/release_boxes_lowering.rs +++ b/crates/perry-codegen/tests/release_boxes_lowering.rs @@ -226,7 +226,7 @@ fn release_boxes_lowers_through_closure_captures() { ); } assert!( - !ir.contains("call void @js_closure_set_box_capture_ptr("), + !ir.contains("call void @js_closure_register_box_layout("), "the compiler-private step closure is covered by the activation refcount, \ so its complete frame must not become escaped GC-closure edges:\n{ir}" ); @@ -276,7 +276,7 @@ fn escaped_user_closure_inside_step_keeps_its_box_capture_edge() { let ir = ir_for_fn_body("release_nested_user_capture", body); let tracked_edges = ir .lines() - .filter(|line| line.contains("call void @js_closure_set_box_capture_ptr(")) + .filter(|line| line.contains("call void @js_closure_register_box_layout(")) .count(); assert_eq!( tracked_edges, 1, diff --git a/crates/perry-runtime/src/box.rs b/crates/perry-runtime/src/box.rs index 77e7ebf71b..45fec47a04 100644 --- a/crates/perry-runtime/src/box.rs +++ b/crates/perry-runtime/src/box.rs @@ -1182,7 +1182,7 @@ fn box_get_bits_named(ptr: *mut Box, name: f64) -> i64 { /// Raw read for an internal closure body selected from an exact arrow target. /// /// Codegen emits this only for capture slots installed by -/// `js_closure_set_box_capture_ptr`. The public closure body retains +/// `js_closure_register_box_layout`. The public closure body retains /// `js_box_get_bits` and its authoritative registry check. A live closure's /// exact capture edge keeps the non-moving box cell from being published for /// reuse, so the compiler-installed pointer stays valid for this call. TDZ diff --git a/crates/perry-runtime/src/closure/alloc.rs b/crates/perry-runtime/src/closure/alloc.rs index 78456744db..45f09ca231 100644 --- a/crates/perry-runtime/src/closure/alloc.rs +++ b/crates/perry-runtime/src/closure/alloc.rs @@ -789,6 +789,7 @@ pub extern "C" fn js_closure_set_box_capture_ptr( index: u32, value: i64, ) { + super::box_captures::prepare_dynamic_box_capture(closure); js_closure_set_capture_bits(closure, index, value as u64); let cell = crate::r#box::registered_box_capture_addr(value as usize); super::box_captures::set_closure_box_capture(closure, index, cell); diff --git a/crates/perry-runtime/src/closure/box_captures.rs b/crates/perry-runtime/src/closure/box_captures.rs index bdc7b3fda4..be363e757b 100644 --- a/crates/perry-runtime/src/closure/box_captures.rs +++ b/crates/perry-runtime/src/closure/box_captures.rs @@ -1,6 +1,7 @@ //! Lifetime bridge between GC closures and malloc-side async box cells. //! -//! Codegen identifies the capture slots which contain raw box addresses. We +//! Codegen identifies raw box slots through the shared `box_layout` bitmap. +//! This module keeps per-cell counts and legacy dynamically declared edges. We //! count only those declared edges before a box reaches terminal //! `ReleaseBoxes`; arbitrary JS values must never be guessed to be boxes from //! pointer-shaped bits. Once its async activation drains, the box runtime @@ -55,7 +56,8 @@ impl BoxCaptureSlots { } crate::perry_thread_local! { - /// Closure address -> compiler-declared `(capture index, box address)` edges. + /// Legacy/dynamically patched closures only. Generated closures use the + /// shared function bitmap and weak owner set in `box_layout`. static CLOSURE_BOX_CELLS: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); /// Box address -> packed edge record: the number of capture slots naming @@ -73,7 +75,7 @@ const FRAME_RELEASE_TAG_SHIFT: u32 = 60; const FRAME_RELEASE_TAG_MASK: usize = 0b11 << FRAME_RELEASE_TAG_SHIFT; const EDGE_COUNT_MASK: usize = (1 << FRAME_RELEASE_TAG_SHIFT) - 1; -fn increment_cell_capture_count(cell: usize, amount: usize) { +pub(super) fn increment_cell_capture_count(cell: usize, amount: usize) { BOX_CAPTURE_COUNTS.with(|counts| { let mut counts = counts.borrow_mut(); let record = counts.entry(cell).or_default(); @@ -85,7 +87,7 @@ fn increment_cell_capture_count(cell: usize, amount: usize) { }); } -fn decrement_cell_capture_count(cell: usize, amount: usize) { +pub(super) fn decrement_cell_capture_count(cell: usize, amount: usize) { // `Some(record)` when the final edge disappeared. let reached_zero = BOX_CAPTURE_COUNTS.with(|counts| { let mut counts = counts.borrow_mut(); @@ -173,6 +175,7 @@ pub(crate) fn visit_closure_box_payload_slots_mut(closure: usize, mut visit: imp if closure == 0 || !crate::gc::full_trace_active() { return; } + super::box_layout::visit_payloads(closure, &mut visit); CLOSURE_BOX_CELLS.with(|captures| { let captures = captures.borrow(); let Some(cells) = captures.get(&closure) else { @@ -184,6 +187,27 @@ pub(crate) fn visit_closure_box_payload_slots_mut(closure: usize, mut visit: imp }); } +pub(super) fn has_dynamic_box_captures(closure: *mut ClosureHeader) -> bool { + CLOSURE_BOX_CELLS.with(|all| { + let all = all.borrow(); + !all.is_empty() && all.contains_key(&(closure as usize)) + }) +} + +/// Preserve counted edges before a legacy caller mutates a shared-layout closure. +pub(super) fn prepare_dynamic_box_capture(closure: *mut ClosureHeader) { + let edges = super::box_layout::take_dynamic_edges(closure); + if !edges.is_empty() { + CLOSURE_BOX_CELLS.with(|all| { + let mut slots = BoxCaptureSlots::default(); + for (index, cell) in edges { + slots.push(index, cell); + } + all.borrow_mut().insert(closure as usize, slots); + }); + } +} + /// Record a compiler-declared boxed capture slot. pub(super) fn set_closure_box_capture( closure: *mut ClosureHeader, @@ -225,6 +249,19 @@ pub(crate) fn clone_closure_box_captures( if source.is_null() || destination.is_null() || source.cast_mut() == destination { return; } + if super::box_layout::clone_owner(source, destination) { + return; + } + let shared_edges = super::box_layout::copy_edges(source); + if !shared_edges.is_empty() { + for (index, _) in shared_edges { + let cell = crate::r#box::registered_box_capture_addr( + super::js_closure_get_capture_bits(destination, index) as usize, + ); + set_closure_box_capture(destination, index, cell); + } + return; + } let source = source as usize; let destination = destination as usize; let copied = CLOSURE_BOX_CELLS.with(|all| { @@ -245,6 +282,7 @@ pub(crate) fn closure_box_captures_owner_moved(old_owner: usize, new_owner: usiz if old_owner == 0 || new_owner == 0 || old_owner == new_owner { return; } + super::box_layout::owner_moved(old_owner, new_owner); CLOSURE_BOX_CELLS.with(|all| { let mut all = all.borrow_mut(); if let Some(cells) = all.remove(&old_owner) { @@ -255,6 +293,7 @@ pub(crate) fn closure_box_captures_owner_moved(old_owner: usize, new_owner: usiz } pub(crate) fn prune_dead_closure_box_capture_owners(is_dead_closure: &dyn Fn(usize) -> bool) { + super::box_layout::prune(is_dead_closure); // One pass: dropping a dead owner and collecting its edges together avoids // a second probe per dead closure. Counts (and any publication they // trigger) are settled after the table borrow ends. @@ -276,6 +315,7 @@ pub(crate) fn prune_dead_closure_box_capture_owners(is_dead_closure: &dyn Fn(usi #[cfg(test)] pub(crate) fn test_clear_closure_box_capture_indexes() { + super::box_layout::clear(); CLOSURE_BOX_CELLS.with(|all| all.borrow_mut().clear()); BOX_CAPTURE_COUNTS.with(|all| all.borrow_mut().clear()); } diff --git a/crates/perry-runtime/src/closure/box_layout.rs b/crates/perry-runtime/src/closure/box_layout.rs new file mode 100644 index 0000000000..ddbeac23d9 --- /dev/null +++ b/crates/perry-runtime/src/closure/box_layout.rs @@ -0,0 +1,282 @@ +//! Compiler-declared, immutable box-slot layouts shared by function body. +//! +//! Only the owner address is retained per closure. The authoritative cells +//! stay in its capture payload; GC death pruning reads them before reclaiming +//! the payload, and relocation rekeys the owner before retiring from-space. + +use super::{closure_capture_slots_mut, real_capture_count, ClosureHeader}; +use std::cell::RefCell; + +crate::perry_thread_local! { + // Code addresses and Rust-owned bitmaps; no GC-managed pointers. + static FUNCTION_BOX_LAYOUTS: RefCell>> = + RefCell::new(crate::fast_hash::new_ptr_hash_map()); + // Weak owners, deliberately NOT roots. Rekeyed by owner_moved and + // pruned before either sweep or the copied-minor from-space reset. + static BOX_LAYOUT_OWNERS: RefCell> = + RefCell::new(crate::fast_hash::new_ptr_hash_set()); +} + +unsafe fn visit_mask(closure: *mut ClosureHeader, mask: &[u64], mut visit: impl FnMut(u32, usize)) { + let count = real_capture_count((*closure).capture_count) as usize; + let slots = closure_capture_slots_mut(closure); + for (word_index, &word) in mask.iter().enumerate() { + let mut bits = word; + while bits != 0 { + let index = word_index * 64 + bits.trailing_zeros() as usize; + assert!(index < count, "box capture layout exceeds closure payload"); + visit(index as u32, *slots.add(index) as usize); + bits &= bits - 1; + } + } +} + +fn visit_cells(closure: *mut ClosureHeader, visit: impl FnMut(u32, usize)) { + FUNCTION_BOX_LAYOUTS.with(|layouts| { + let layouts = layouts.borrow(); + // SAFETY: callers hold a live closure or run in pre-reclamation GC pruning. + unsafe { + if let Some(mask) = layouts.get(&((*closure).func_ptr as usize)) { + visit_mask(closure, mask, visit); + } + } + }); +} + +/// Register initialized, compiler-proven box captures in one batch. The mask +/// is constant for a function body; every named slot must contain a live box. +/// No collection occurs here. Repeated registration of a cached singleton is +/// idempotent and needs only the owner-set probe. +/// +/// # Safety +/// `closure` must be live, with initialized captures. `mask` must name `words` +/// readable words; set bits must name live box pointers within the payload. +/// All registrations for the same function must supply the same mask. +#[no_mangle] +pub unsafe extern "C" fn js_closure_register_box_layout( + closure: *mut ClosureHeader, + mask: *const u64, + words: u32, +) { + if closure.is_null() || words == 0 { + return; + } + if super::box_captures::has_dynamic_box_captures(closure) { + return; + } + if !BOX_LAYOUT_OWNERS.with(|owners| owners.borrow_mut().insert(closure as usize)) { + return; + } + FUNCTION_BOX_LAYOUTS.with(|layouts| { + let mut layouts = layouts.borrow_mut(); + let supplied = std::slice::from_raw_parts(mask, words as usize); + let layout = layouts + .entry((*closure).func_ptr as usize) + .or_insert_with(|| supplied.into()); + debug_assert_eq!( + layout.as_ref(), + supplied, + "box layout changed for a function body" + ); + visit_mask(closure, layout, |_, cell| { + super::box_captures::increment_cell_capture_count(cell, 1) + }); + }); +} + +pub(super) fn visit_payloads(closure: usize, visit: &mut impl FnMut(*mut u64)) { + if BOX_LAYOUT_OWNERS.with(|owners| owners.borrow().contains(&closure)) { + visit_cells(closure as *mut ClosureHeader, |_, cell| { + crate::r#box::visit_pending_captured_js_box_payload_slot(cell, visit); + }); + } +} + +/// The compatibility setter permits dynamic slot replacement. Move that rare +/// owner to the old exact-edge representation without changing any counts. +pub(super) fn take_dynamic_edges(closure: *mut ClosureHeader) -> Vec<(u32, usize)> { + let mut edges = Vec::new(); + if BOX_LAYOUT_OWNERS.with(|owners| owners.borrow_mut().remove(&(closure as usize))) { + visit_cells(closure, |index, cell| edges.push((index, cell))); + } + edges +} + +pub(super) fn clone_owner(source: *const ClosureHeader, destination: *mut ClosureHeader) -> bool { + if !BOX_LAYOUT_OWNERS.with(|owners| owners.borrow().contains(&(source as usize))) { + return false; + } + // Runtime rebinding can patch a capture. Only unchanged layouts can stay + // in the shared representation; the caller handles the exceptional case. + let mut unchanged = true; + visit_cells(source.cast_mut(), |index, cell| { + unchanged &= super::js_closure_get_capture_bits(destination, index) as usize == cell; + }); + if !unchanged { + return false; + } + if BOX_LAYOUT_OWNERS.with(|owners| owners.borrow_mut().insert(destination as usize)) { + visit_cells(destination, |_, cell| { + super::box_captures::increment_cell_capture_count(cell, 1) + }); + } + true +} + +pub(super) fn copy_edges(source: *const ClosureHeader) -> Vec<(u32, usize)> { + let mut edges = Vec::new(); + if BOX_LAYOUT_OWNERS.with(|owners| owners.borrow().contains(&(source as usize))) { + visit_cells(source.cast_mut(), |index, cell| edges.push((index, cell))); + } + edges +} + +pub(super) fn owner_moved(old: usize, new: usize) { + BOX_LAYOUT_OWNERS.with(|owners| { + let mut owners = owners.borrow_mut(); + if owners.remove(&old) { + owners.insert(new); + } + }); +} + +pub(super) fn prune(is_dead: &dyn Fn(usize) -> bool) { + let mut cells = Vec::new(); + BOX_LAYOUT_OWNERS.with(|owners| { + owners.borrow_mut().retain(|owner| { + if !is_dead(*owner) { + return true; + } + visit_cells(*owner as *mut ClosureHeader, |_, cell| cells.push(cell)); + false + }); + }); + for cell in cells { + super::box_captures::decrement_cell_capture_count(cell, 1); + } +} + +#[cfg(test)] +pub(super) fn clear() { + BOX_LAYOUT_OWNERS.with(|owners| owners.borrow_mut().clear()); + FUNCTION_BOX_LAYOUTS.with(|layouts| layouts.borrow_mut().clear()); +} + +#[cfg(feature = "keepalive-anchors")] +#[used(compiler)] +static KEEP_REGISTER_BOX_LAYOUT: unsafe extern "C" fn(*mut ClosureHeader, *const u64, u32) = + js_closure_register_box_layout; + +#[cfg(test)] +mod tests { + use super::*; + use crate::closure::{box_capture_count, js_closure_alloc, js_closure_set_capture_bits}; + use crate::r#box::{js_box_alloc_bits, js_box_scope_release, test_clear_box_registry}; + + fn is_registered_box_ptr(cell: *mut crate::r#box::Box) -> bool { + crate::r#box::registered_box_capture_addr(cell as usize).is_some() + } + + fn closure( + func: usize, + captures: u32, + slots: &[(u32, usize)], + mask: &[u64], + ) -> *mut ClosureHeader { + let owner = js_closure_alloc(func as *const u8, captures); + for &(index, cell) in slots { + js_closure_set_capture_bits(owner, index, cell as u64); + } + unsafe { + js_closure_register_box_layout(owner, mask.as_ptr(), mask.len() as u32); + } + owner + } + + #[test] + fn shared_box_layout_wide_sparse_duplicate_edges_and_singleton() { + test_clear_box_registry(); + let cell = js_box_alloc_bits(42); + // Slot 1 contains the SAME pointer-shaped word but is not a box edge. + let slots = [(0, cell as usize), (1, cell as usize), (65, cell as usize)]; + let mask = [1, 2]; + let a = closure(123, 66, &slots, &mask); + let b = closure(123, 66, &slots, &mask); + unsafe { + js_closure_register_box_layout(a, mask.as_ptr(), 2); + } + assert_eq!(box_capture_count(cell as usize), 4); + assert!(!super::super::box_captures::has_dynamic_box_captures(a)); + FUNCTION_BOX_LAYOUTS.with(|layouts| assert_eq!(layouts.borrow().len(), 1)); + BOX_LAYOUT_OWNERS.with(|owners| assert_eq!(owners.borrow().len(), 2)); + js_box_scope_release(cell); + prune(&|owner| owner == a as usize); + assert_eq!(box_capture_count(cell as usize), 2); + assert!(is_registered_box_ptr(cell)); + prune(&|owner| owner == b as usize); + assert!(!is_registered_box_ptr(cell)); + } + + #[test] + fn shared_box_layout_move_clone_and_dynamic_replacement() { + test_clear_box_registry(); + let first = js_box_alloc_bits(41); + let second = js_box_alloc_bits(42); + let a = closure(123, 1, &[(0, first as usize)], &[1]); + let b = js_closure_alloc(123 as *const u8, 1); + js_closure_set_capture_bits(b, 0, first as u64); + crate::closure::clone_closure_box_captures(a, b); + assert_eq!(box_capture_count(first as usize), 2); + let moved = js_closure_alloc(123 as *const u8, 1); + js_closure_set_capture_bits(moved, 0, first as u64); + crate::closure::closure_box_captures_owner_moved(a as usize, moved as usize); + prune(&|owner| owner == a as usize); + assert_eq!(box_capture_count(first as usize), 2); + crate::closure::js_closure_set_box_capture_ptr(b, 0, second as i64); + assert_eq!(box_capture_count(first as usize), 1); + assert_eq!(box_capture_count(second as usize), 1); + js_box_scope_release(first); + js_box_scope_release(second); + crate::closure::prune_dead_closure_box_capture_owners(&|_| true); + assert!(!is_registered_box_ptr(first)); + assert!(!is_registered_box_ptr(second)); + } + + #[test] + fn shared_box_layout_rebound_clone_uses_actual_destination_edges() { + test_clear_box_registry(); + let cell = js_box_alloc_bits(42); + let source = closure(123, 2, &[(0, cell as usize), (1, cell as usize)], &[3]); + let dest = js_closure_alloc(123 as *const u8, 2); + js_closure_set_capture_bits(dest, 0, crate::value::TAG_UNDEFINED); + js_closure_set_capture_bits(dest, 1, cell as u64); + crate::closure::clone_closure_box_captures(source, dest); + assert_eq!(box_capture_count(cell as usize), 3); + js_box_scope_release(cell); + crate::closure::prune_dead_closure_box_capture_owners(&|owner| owner == source as usize); + assert_eq!(box_capture_count(cell as usize), 1); + crate::closure::prune_dead_closure_box_capture_owners(&|owner| owner == dest as usize); + assert!(!is_registered_box_ptr(cell)); + } + #[test] + fn shared_box_layout_traces_released_payload_through_live_closure() { + test_clear_box_registry(); + let cell = js_box_alloc_bits(crate::value::TAG_UNDEFINED as i64); + let owner = closure(123, 1, &[(0, cell as usize)], &[1]); + let bits = crate::value::js_nanbox_pointer(owner as i64).to_bits(); + crate::r#box::js_box_set_bits(cell, bits as i64); + js_box_scope_release(cell); + crate::gc::begin_full_trace(); + let mut slots = Vec::new(); + crate::closure::visit_closure_box_payload_slots_mut(owner as usize, |slot| { + slots.push(slot as usize) + }); + crate::gc::finish_full_trace(); + assert!( + slots.contains(&(cell as usize)), + "live closure must reach the released box payload" + ); + prune(&|_| true); + assert!(!is_registered_box_ptr(cell)); + } +} diff --git a/crates/perry-runtime/src/closure/mod.rs b/crates/perry-runtime/src/closure/mod.rs index e3846ffa55..6e68e92910 100644 --- a/crates/perry-runtime/src/closure/mod.rs +++ b/crates/perry-runtime/src/closure/mod.rs @@ -7,6 +7,8 @@ mod alloc; mod box_captures; +mod box_layout; +pub use box_layout::js_closure_register_box_layout; mod dispatch; mod dynamic_props; mod registry; diff --git a/scripts/check_runtime_symbols.sh b/scripts/check_runtime_symbols.sh index 9307f677ac..1b6c1e5062 100755 --- a/scripts/check_runtime_symbols.sh +++ b/scripts/check_runtime_symbols.sh @@ -54,6 +54,7 @@ SENTINELS=( js_closure_get_capture_bits js_closure_set_capture_bits js_closure_set_box_capture_ptr + js_closure_register_box_layout js_closure_resolve_arrow_direct_call js_register_closure_trusted_direct # #9188: codegen switched module init from the copying spellings to these. diff --git a/scripts/gc_root_dominance_check.py b/scripts/gc_root_dominance_check.py index 093368a9cb..f79eba0e89 100755 --- a/scripts/gc_root_dominance_check.py +++ b/scripts/gc_root_dominance_check.py @@ -505,6 +505,7 @@ def build_cfg(f): # verified non-allocating bookkeeping stores/reads (perry-runtime) "js_closure_set_capture_bits", # closure/alloc.rs:477 raw slot write + layout note "js_closure_set_box_capture_ptr", # declared box edge + same raw slot write + "js_closure_register_box_layout", # Rust-owned bitmap/edge-count bookkeeping; no GC allocation "js_closure_get_capture_bits", # closure/alloc.rs:463 raw slot read "js_closure_set_capture_ptr", "js_closure_get_capture_ptr", "js_box_set_bits", "js_box_set_bits_trusted_no_barrier", diff --git a/test-files/test_gap_closure_shared_box_layout.ts b/test-files/test_gap_closure_shared_box_layout.ts new file mode 100644 index 0000000000..a9e3331ee7 --- /dev/null +++ b/test-files/test_gap_closure_shared_box_layout.ts @@ -0,0 +1,162 @@ +// Shared box layouts must preserve distinct instances and captures past slot 63. +const maybeGc = (globalThis as any).gc; +function makeWide(seed: number) { + let v0 = seed + 0; + let v1 = seed + 1; + let v2 = seed + 2; + let v3 = seed + 3; + let v4 = seed + 4; + let v5 = seed + 5; + let v6 = seed + 6; + let v7 = seed + 7; + let v8 = seed + 8; + let v9 = seed + 9; + let v10 = seed + 10; + let v11 = seed + 11; + let v12 = seed + 12; + let v13 = seed + 13; + let v14 = seed + 14; + let v15 = seed + 15; + let v16 = seed + 16; + let v17 = seed + 17; + let v18 = seed + 18; + let v19 = seed + 19; + let v20 = seed + 20; + let v21 = seed + 21; + let v22 = seed + 22; + let v23 = seed + 23; + let v24 = seed + 24; + let v25 = seed + 25; + let v26 = seed + 26; + let v27 = seed + 27; + let v28 = seed + 28; + let v29 = seed + 29; + let v30 = seed + 30; + let v31 = seed + 31; + let v32 = seed + 32; + let v33 = seed + 33; + let v34 = seed + 34; + let v35 = seed + 35; + let v36 = seed + 36; + let v37 = seed + 37; + let v38 = seed + 38; + let v39 = seed + 39; + let v40 = seed + 40; + let v41 = seed + 41; + let v42 = seed + 42; + let v43 = seed + 43; + let v44 = seed + 44; + let v45 = seed + 45; + let v46 = seed + 46; + let v47 = seed + 47; + let v48 = seed + 48; + let v49 = seed + 49; + let v50 = seed + 50; + let v51 = seed + 51; + let v52 = seed + 52; + let v53 = seed + 53; + let v54 = seed + 54; + let v55 = seed + 55; + let v56 = seed + 56; + let v57 = seed + 57; + let v58 = seed + 58; + let v59 = seed + 59; + let v60 = seed + 60; + let v61 = seed + 61; + let v62 = seed + 62; + let v63 = seed + 63; + let v64 = seed + 64; + let v65 = seed + 65; + return (step: number) => { + v0 += step; + v1 += step; + v2 += step; + v3 += step; + v4 += step; + v5 += step; + v6 += step; + v7 += step; + v8 += step; + v9 += step; + v10 += step; + v11 += step; + v12 += step; + v13 += step; + v14 += step; + v15 += step; + v16 += step; + v17 += step; + v18 += step; + v19 += step; + v20 += step; + v21 += step; + v22 += step; + v23 += step; + v24 += step; + v25 += step; + v26 += step; + v27 += step; + v28 += step; + v29 += step; + v30 += step; + v31 += step; + v32 += step; + v33 += step; + v34 += step; + v35 += step; + v36 += step; + v37 += step; + v38 += step; + v39 += step; + v40 += step; + v41 += step; + v42 += step; + v43 += step; + v44 += step; + v45 += step; + v46 += step; + v47 += step; + v48 += step; + v49 += step; + v50 += step; + v51 += step; + v52 += step; + v53 += step; + v54 += step; + v55 += step; + v56 += step; + v57 += step; + v58 += step; + v59 += step; + v60 += step; + v61 += step; + v62 += step; + v63 += step; + v64 += step; + v65 += step; + return v0 + v1 + v2 + v3 + v4 + v5 + v6 + v7 + v8 + v9 + v10 + v11 + v12 + v13 + v14 + v15 + v16 + v17 + v18 + v19 + v20 + v21 + v22 + v23 + v24 + v25 + v26 + v27 + v28 + v29 + v30 + v31 + v32 + v33 + v34 + v35 + v36 + v37 + v38 + v39 + v40 + v41 + v42 + v43 + v44 + v45 + v46 + v47 + v48 + v49 + v50 + v51 + v52 + v53 + v54 + v55 + v56 + v57 + v58 + v59 + v60 + v61 + v62 + v63 + v64 + v65; + }; +} +const closures: Array<(step: number) => number> = []; +for (let i = 0; i < 20; i++) closures.push(makeWide(i)); +if (typeof maybeGc === "function") maybeGc(); +let total = 0; +for (let i = 0; i < closures.length; i++) total += closures[i](i); +console.log("wide first", total); +if (typeof maybeGc === "function") maybeGc(); +total = 0; +for (const fn of closures) total += fn(1); +console.log("wide second", total); + +async function escaped() { + let payload = { text: "kept" }; + await Promise.resolve(); + return () => { + payload = { text: payload.text + "!" }; + return payload.text; + }; +} +escaped().then((fn) => { + if (typeof maybeGc === "function") maybeGc(); + console.log("async", fn(), fn()); +});