diff --git a/quest/m2/perf/README.md b/quest/m2/perf/README.md index 8d8a6649f9c..a7e1a17843c 100644 --- a/quest/m2/perf/README.md +++ b/quest/m2/perf/README.md @@ -91,7 +91,6 @@ revisions on the same host and worker count, and profiles with - [An idle export poll is cheap](/quest/m2/perf/export-idle-poll.md) - an export poll nothing woke skips the track fill and catalog work it repeats today - [Mesh session cost](/quest/m2/perf/mesh-session-cost.md) - moq-net's per-session announce, route sync, and scope work drops, swept over sessions and announces - [Cluster route memory](/quest/m2/perf/cluster-route-memory.md) - a full-mesh cluster's ~3.7 KB per route per broadcast shrinks, the in-memory hop is profiled, and `bench-profile lock` tolerates lost records -- [Lock-free single-delivery writes](/quest/m2/perf/single-delivery-write.md) - a frame with one delivery writes without a lock, recovering #6922's remaining FETCH cost - [Small hot-path waste](/quest/m2/perf/hot-path-waste.md) - one track-name copy per lite subscription, no full-history `#drift` walk, and one header buffer per JS object - [Track-state reads skip when nothing changed](/quest/m2/perf/cursor-version-skip.md) - `Cursor::sync`, `probe` and `GroupExpiry` skip unchanged reads, then group reads move onto a Watch - [An idle TS export with a delay costs what one without does](/quest/m2/perf/ts-idle-jitter.md) - the jitter scan and memcmp stop leading the idle poll diff --git a/quest/m2/perf/single-delivery-write.md b/quest/m2/perf/single-delivery-write.md deleted file mode 100644 index 99de0b6b146..00000000000 --- a/quest/m2/perf/single-delivery-write.md +++ /dev/null @@ -1,31 +0,0 @@ -# [M] A frame with one delivery writes without a lock - -## Goal - -Writing a frame that has a single delivery takes no lock and no overlap -check, so 256-frame lite FETCH costs no more instructions than before -[moq-dev/moq#6922](https://github.com/moq-dev/moq/pull/6922), while a second -copy attaching mid-write stays correct. - -## Plan - -Found 2026-10-28 by [moq-dev/moq#7141](https://github.com/moq-dev/moq/pull/7141), -which brought allocations back to base and recovered about half of #6922's -cost. About +3.6% instructions remains on 256-frame lite FETCH, from #6922's -per-frame locking and overlap checks when writing. - -Decided 2026-10-28 (maintainer): accept the residual in #7141 and explore a -lock-free path here, in m2. - -- The risk is a second copy attaching to a frame while it is still being - written. Rule that race out before relying on it. -- Keep the shared reservation and canonical-content rules #6922 added. -- A measured no-win abandons the quest. - -Also in scope (decided 2026-10-29, from [moq-dev/moq#7139](https://github.com/moq-dev/moq/pull/7139)): -after #7139, batched `write_frames` still costs about 20% more instructions per -frame than before #6922 (459 against 380; 6 to 12% wall clock), from the same -canonical-storage design. Measure the `group_write_frames/batch32/*` cases -beside lite FETCH, and fix both paths together where they share the cost. - -Public API: none. Wire: none. diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 9940487c7eb..acc087ad273 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -459,7 +459,9 @@ impl Drop for Reservation { /// /// The delivery that creates the content keeps its progress in the content itself, so /// a frame with one delivery, nearly every frame, costs one allocation. Only a fork -/// allocates a [`View`] of its own. +/// allocates a [`View`] of its own. Until anything else attaches to the content, that +/// delivery is its only writer, so it publishes without the publication lock or an +/// overlap check (see [`FrameContent::attach`]). #[derive(Clone)] pub(super) struct FrameBuf { content: Arc, @@ -484,6 +486,13 @@ struct FrameContent { // freezes as itself and leaves this empty. frozen: OnceLock, publication: Mutex, + // Set by the creating delivery around each write it makes without the publication + // lock, and by `attach`, after which every write takes the lock. + writing: AtomicBool, + attached: AtomicBool, + // Whether the creating delivery still holds the admission it took with the content, + // so attaching counts it among the publication's producers. + creator_admitted: AtomicBool, bill: OnceLock, identity: OnceLock, } @@ -531,6 +540,9 @@ impl Shared { storage: OnceLock::from(FrameStorage::Shared(payload)), frozen: OnceLock::new(), publication: Mutex::default(), + writing: AtomicBool::new(false), + attached: AtomicBool::new(false), + creator_admitted: AtomicBool::new(false), bill: OnceLock::new(), identity: OnceLock::new(), })) @@ -539,6 +551,10 @@ impl Shared { SharedWeak(Arc::downgrade(&self.0)) } pub(super) fn fork(&self) -> FrameBuf { + if !self.0.attached.load(Ordering::Acquire) { + self.0 + .attach(&mut self.0.publication.lock().unwrap_or_else(PoisonError::into_inner)); + } FrameBuf { content: self.0.clone(), fork: Some(Arc::default()), @@ -562,7 +578,8 @@ impl Shared { self.0.bill.get().map_or(0, cache::Extra::size) } pub(super) fn identify(&self, owner: Arc, key: group::ContentKey) -> bool { - let _publication = self.0.publication.lock().unwrap_or_else(PoisonError::into_inner); + let mut publication = self.0.publication.lock().unwrap_or_else(PoisonError::into_inner); + self.0.attach(&mut publication); if self .0 .identity @@ -582,7 +599,9 @@ impl Shared { impl FrameContent { fn bill_content(&self, bill: cache::Extra) { let content = self; - let _publication = content.publication.lock().unwrap_or_else(PoisonError::into_inner); + let mut publication = content.publication.lock().unwrap_or_else(PoisonError::into_inner); + // Billing reads the published bytes, which the creator's lock-free writes would race. + content.attach(&mut publication); content.bill.get_or_init(|| { let written = self.written.load(Ordering::Relaxed) as u64; // A lazy owner may take already-billed payload from the original delivery. @@ -597,6 +616,30 @@ impl FrameContent { }); } + /// Let anything besides the creating delivery reach the content: another delivery, its + /// cache bill, or an admission taken through the publication. The creator's write in + /// flight finishes first, and every later one publishes under the lock. + /// + /// Caller holds `publication`, so attachers are serialized. + fn attach(&self, publication: &mut Publication) { + if self.attached.load(Ordering::Relaxed) { + return; + } + // With the creator's SeqCst store to `writing` and load of `attached`, at least one + // side sees the other's store: either the creator takes the lock, or this waits. + self.attached.store(true, Ordering::SeqCst); + // A lock-free write is a bounded copy that never waits on anything, so this ends + // once its thread runs again. + while self.writing.load(Ordering::SeqCst) { + std::thread::yield_now(); + } + // The creator's admission joins the shared count; its reservation stays with it + // until it ends, and passes to any producer still active then. + if self.creator_admitted.load(Ordering::Relaxed) { + publication.producers += 1; + } + } + /// Caller holds publication so attachment and growth update one aggregate. fn add_charge(&self, bytes: u64) { if let Some(bill) = self.bill.get() { @@ -681,34 +724,17 @@ struct Publication { reserved: Option, } -/// One producer's hold on its canonical frame's admission. The reservation returns to -/// the budget once no producer of the frame remains active. +/// One producer's hold on its frame's admission, ended by [`FrameBuf::release`]. The +/// reservation returns to the budget once no producer of the frame remains active. /// -/// The admission lives in the frame's [`Publication`] rather than an allocation of its -/// own: every streamed frame takes one, and nearly all have a single producer. -pub(super) struct Admission(Arc); - -impl Admission { - /// Whether the frame holds a share of the budget. - #[cfg(test)] - fn reserved(&self) -> bool { - self.0 - .publication - .lock() - .unwrap_or_else(PoisonError::into_inner) - .reserved - .is_some() - } -} - -impl Drop for Admission { - fn drop(&mut self) { - let mut publication = self.0.publication.lock().unwrap_or_else(PoisonError::into_inner); - publication.producers -= 1; - if publication.producers == 0 { - publication.reserved = None; - } - } +/// No admission allocates: every streamed frame takes one, and nearly all have a single +/// producer. +pub(super) enum Admission { + /// The creating delivery's, taken with its content. It keeps its reservation, and + /// counts among the publication's producers only once something attaches. + Creator(Option), + /// Counted in the [`Publication`], which holds the reservation. + Shared, } enum FrameStorage { @@ -896,6 +922,20 @@ impl FrameBuf { Self::with_growth(size, true) } + /// A buffer for a frame of `size` bytes with its creating delivery's admission, + /// allocated at that size on the first write while `budget` covers it, and grown as + /// bytes arrive otherwise. + pub(super) fn admitted(size: usize, budget: &Budget) -> (Self, Option) { + if size == 0 { + return (Self::new(0), None); + } + let reserved = budget.reserve(size.min(SEGMENT)); + let buf = Self::with_growth(size, reserved.is_none()); + // Nothing else holds the content yet. + buf.content.creator_admitted.store(true, Ordering::Relaxed); + (buf, Some(Admission::Creator(reserved))) + } + fn with_growth(size: usize, grow: bool) -> Self { Self { content: Arc::new(FrameContent { @@ -906,6 +946,9 @@ impl FrameBuf { storage: OnceLock::new(), frozen: OnceLock::new(), publication: Mutex::default(), + writing: AtomicBool::new(false), + attached: AtomicBool::new(false), + creator_admitted: AtomicBool::new(false), bill: OnceLock::new(), identity: OnceLock::new(), }), @@ -977,6 +1020,7 @@ impl FrameBuf { pub(super) fn admit(&self, budget: &Budget) -> Option { let content = &self.content; let mut publication = content.publication.lock().unwrap_or_else(PoisonError::into_inner); + content.attach(&mut publication); if self.shared_written(Ordering::Acquire) == self.size() { return None; } @@ -990,8 +1034,45 @@ impl FrameBuf { publication.reserved = reserved; } publication.producers += 1; - drop(publication); - Some(Admission(content.clone())) + Some(Admission::Shared) + } + + /// End `admission`, which this delivery took. + pub(super) fn release(&self, admission: Admission) { + let content = &self.content; + let reserved = match admission { + Admission::Creator(reserved) => { + // Alone, the creator's reservation is the frame's; see `write_chunk`. + content.writing.store(true, Ordering::SeqCst); + if !content.attached.load(Ordering::SeqCst) { + content.creator_admitted.store(false, Ordering::Relaxed); + content.writing.store(false, Ordering::Release); + return; + } + content.writing.store(false, Ordering::Release); + reserved + } + Admission::Shared => None, + }; + let mut publication = content.publication.lock().unwrap_or_else(PoisonError::into_inner); + publication.producers -= 1; + if publication.producers == 0 { + publication.reserved = None; + } else if publication.reserved.is_none() { + // Producers still active keep the creator's reservation. + publication.reserved = reserved; + } + } + + /// Whether the frame's active producers hold a share of the budget. + #[cfg(test)] + fn reserved(&self) -> bool { + self.content + .publication + .lock() + .unwrap_or_else(PoisonError::into_inner) + .reserved + .is_some() } /// The frame's declared size. @@ -1022,7 +1103,8 @@ impl FrameBuf { }) } - /// Safety: caller holds the publication mutex and `new_written` is `<= size`. + /// Safety: caller is the only publisher, holding the publication mutex or writing + /// alone as the creator, and `new_written` is `<= size`. unsafe fn store_written(&self, new_written: usize) { let content = &self.content; let before = content.written.load(Ordering::Relaxed); @@ -1038,6 +1120,17 @@ impl FrameBuf { fn write_chunk(&self, chunk: B) -> Result<()> { let content = &self.content; + if self.fork.is_none() { + // The SeqCst store then load pairs with `FrameContent::attach`'s: a write that + // misses the attachment finishes before the attacher continues. + content.writing.store(true, Ordering::SeqCst); + if !content.attached.load(Ordering::SeqCst) { + let written = self.write_alone(chunk); + content.writing.store(false, Ordering::Release); + return written; + } + content.writing.store(false, Ordering::Release); + } let _publication = content.publication.lock().unwrap_or_else(PoisonError::into_inner); let start = self.written(Ordering::Relaxed); let end = start @@ -1074,7 +1167,32 @@ impl FrameBuf { Ok(()) } - /// Append after the shared prefix while holding the publication mutex. + /// Publish the creating delivery's chunk while nothing else can reach the content: its + /// progress is the content's, so no published byte overlaps the chunk. + fn write_alone(&self, chunk: B) -> Result<()> { + let content = &self.content; + let start = content.written.load(Ordering::Relaxed); + let end = start + .checked_add(chunk.as_ref().len()) + .filter(|end| *end <= self.size()) + .ok_or(Error::WrongSize)?; + if end == start { + return Ok(()); + } + match start == 0 && end == self.size() { + true => match self.try_set_bytes(chunk.into_bytes()) { + // Safety: the creator writes alone; the full size was checked. + Ok(()) => unsafe { self.store_written(end) }, + Err(chunk) => self.append_suffix(&chunk), + }, + false => self.append_suffix(chunk.as_ref()), + } + content.received.store(end, Ordering::Release); + Ok(()) + } + + /// Append after the shared prefix as its only publisher: holding the publication + /// mutex, or the creator writing alone. fn append_suffix(&self, src: &[u8]) { if src.is_empty() { return; @@ -1091,8 +1209,8 @@ impl FrameBuf { shared: Mutex::default(), }, }); - // Safety (every write below): the publication mutex makes this the sole writer; - // the caller checked the remaining size, and consumers only read `[..written]`. + // Safety (every write below): the caller is the sole writer, checked the remaining + // size, and consumers only read `[..written]`. match storage { // Only reachable if the frame is already complete via shared storage, which // `Producer::write` rejects for a non-empty chunk. Nothing to copy. @@ -1108,7 +1226,7 @@ impl FrameBuf { segments.append(prev, src, size, grow); } } - // Safety: the publication mutex excludes other writers, and the caller checked the size. + // Safety: the caller is the sole writer, and checked the size. unsafe { self.store_written(prev + src.len()) }; } @@ -1173,7 +1291,7 @@ pub struct Producer { // chunk's bytes. Empty (no-op) for an untagged group. stats: stats::Meter, // The frame's share of the origin's [`Budget`], if it got one, held until it ends. - _reserved: Option, + admission: Option, } impl std::ops::Deref for Producer { @@ -1185,7 +1303,7 @@ impl std::ops::Deref for Producer { } impl Producer { - pub(super) fn new(group: group::Producer, buf: FrameBuf, info: Info, reserved: Option) -> Self { + pub(super) fn new(group: group::Producer, buf: FrameBuf, info: Info, admission: Option) -> Self { Self { group: Some(group), buf, @@ -1193,7 +1311,7 @@ impl Producer { done: false, cancel_on_drop: false, stats: stats::Meter::default(), - _reserved: reserved, + admission, } } @@ -1255,11 +1373,7 @@ impl Producer { } // Ingress payload: count the chunk's bytes as they're written. self.stats.bytes(len as u64); - if len == self.buf.size() && self.buf.written(Ordering::Acquire) == 0 { - self.buf.write_chunk(chunk) - } else { - self.buf.append(chunk.as_ref()) - } + self.buf.write_chunk(chunk) } /// Poll until the group's writer may take more of this frame from upstream; see @@ -1313,27 +1427,28 @@ impl Producer { impl Drop for Producer { fn drop(&mut self) { - if self.done { - return; - } // An unfinished frame leaves the group stream broken; fail the group so // consumers surface an error instead of hanging on the partial forever. // A group already aborted (superseded, evicted, cancelled) carries its own // reason, so cutting its in-flight frame short is expected. - let Some(group) = self.group.as_mut() else { - return; - }; - if !group.is_aborted() && !self.cancel_on_drop { - tracing::warn!( - group = group.sequence, - "frame::Producer dropped before writing all bytes" - ); + if !self.done + && let Some(group) = self.group.as_mut() + { + if !group.is_aborted() && !self.cancel_on_drop { + tracing::warn!( + group = group.sequence, + "frame::Producer dropped before writing all bytes" + ); + } + group.frame_abort(if self.cancel_on_drop { + Error::Cancel + } else { + Error::Dropped + }); + } + if let Some(admission) = self.admission.take() { + self.buf.release(admission); } - group.frame_abort(if self.cancel_on_drop { - Error::Cancel - } else { - Error::Dropped - }); } } @@ -1846,11 +1961,9 @@ mod tests { #[test] fn admission_ends_with_active_producers_not_cached_views() { let budget = Budget::new(SEGMENT); - let a = FrameBuf::new(SEGMENT + 1); + let (a, a_active) = FrameBuf::admitted(SEGMENT + 1, &budget); let b = a.fork(); - let a_active = a.admit(&budget).unwrap(); let b_active = b.admit(&budget).unwrap(); - assert!(Arc::ptr_eq(&a_active.0, &b_active.0)); assert!( budget.reserve(1).is_none(), "the second producer shares the first's reservation" @@ -1861,27 +1974,87 @@ mod tests { replay.admit(&budget).is_none(), "complete content reserves nothing on replay" ); - drop(a_active); + a.release(a_active.unwrap()); assert!( budget.reserve(1).is_none(), "another active delivery owns the same reservation" ); - drop(b_active); + b.release(b_active); let returned = budget.reserve(SEGMENT).expect("cached views keep no admission alive"); assert_eq!(b.written(Ordering::Acquire), 0); assert_eq!(replay.written(Ordering::Acquire), 0); drop(returned); } + /// The creator writes and ends alone without the publication, and a delivery that + /// attaches later still shares its reservation. + #[test] + fn a_creator_alone_publishes_and_releases_without_attaching() { + let budget = Budget::new(8); + let (a, active) = FrameBuf::admitted(8, &budget); + assert!(budget.reserve(1).is_none(), "the creator reserved its frame"); + a.append(b"abcd").unwrap(); + assert_eq!(a.shared_written(Ordering::Acquire), 4); + a.release(active.unwrap()); + assert!(!a.content.attached.load(Ordering::Relaxed)); + drop(budget.reserve(8).expect("the creator's reservation returned")); + + let (a, active) = FrameBuf::admitted(8, &budget); + a.append(b"abcd").unwrap(); + let b = a.fork(); + let copy = b.admit(&budget).unwrap(); + assert!(budget.reserve(1).is_none(), "the attached copy shares the reservation"); + b.release(copy); + assert!( + budget.reserve(1).is_none(), + "the attached creator still holds its reservation" + ); + a.append(b"efgh").unwrap(); + assert!(matches!(b.append(b"abcdefgx"), Err(Error::ProtocolViolation))); + a.release(active.unwrap()); + drop(budget.reserve(8).expect("the last producer returned it")); + } + + /// A delivery attaching while the creator streams without the lock never overlaps a + /// write in flight: both see one payload, and a mismatch is still caught. + #[test] + fn a_delivery_attaching_mid_write_publishes_one_payload() { + const SIZE: usize = SEGMENT + 4093; + let payload: Vec<_> = (0..SIZE).map(|i| (i % 251) as u8).collect(); + for grow in [false, true] { + let budget = Budget::new(if grow { 0 } else { SEGMENT }); + let (a, active) = FrameBuf::admitted(SIZE, &budget); + let attached = std::sync::Barrier::new(2); + std::thread::scope(|scope| { + scope.spawn(|| { + for chunk in payload.chunks(97) { + a.append(chunk).unwrap(); + if a.written(Ordering::Relaxed) == 97 * 64 { + attached.wait(); + } + } + }); + attached.wait(); + let b = a.fork(); + for chunk in payload.chunks(1021) { + b.append(chunk).unwrap(); + } + assert_eq!(b.written(Ordering::Acquire), SIZE); + }); + a.release(active.unwrap()); + assert_eq!(a.freeze().contiguous(0), payload); + assert_eq!(a.allocated(), SIZE); + } + } + #[test] fn unadmitted_deliveries_share_the_same_growing_frame() { let budget = Budget::new(0); let a = FrameBuf::new(SEGMENT); let b = a.fork(); - let left = a.admit(&budget).unwrap(); - let right = b.admit(&budget).unwrap(); - assert!(Arc::ptr_eq(&left.0, &right.0)); - assert!(!left.reserved()); + let _left = a.admit(&budget).unwrap(); + let _right = b.admit(&budget).unwrap(); + assert!(!a.reserved()); a.append(b"a").unwrap(); assert_eq!(a.allocated(), 1, "failed admission grows only with received bytes"); b.append(b"ab").unwrap(); @@ -1925,7 +2098,7 @@ mod tests { ); a.append(b"efgh").unwrap(); assert_eq!(pool.used(), baseline + metadata + 8); - drop(active); + a.release(active); drop(a); assert_eq!(pool.used(), baseline + metadata + 8, "the fork still holds the content"); drop(b); @@ -1953,14 +2126,13 @@ mod tests { let first = a.admit(&budget).unwrap(); let copy = a.fork(); let second = copy.admit(&budget).unwrap(); - assert!(Arc::ptr_eq(&first.0, &second.0)); assert_eq!( pool.used(), baseline + metadata, "an admission lives in its content, allocating nothing" ); - drop(first); - drop(second); + a.release(first); + copy.release(second); let resumed = a.admit(&budget).unwrap(); assert_eq!( pool.used(), @@ -1971,7 +2143,7 @@ mod tests { assert_eq!(a.content_size(), (FrameBuf::CONTENT_HEAP + 8) as u64); assert_eq!(a.view_size(), 0, "the creating delivery's view is part of its content"); assert_eq!(copy.view_size(), 0, "a fork starts without a delivery charge"); - drop(resumed); + a.release(resumed); drop(a); drop(copy); assert_eq!(pool.used(), baseline); diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index cd84338be76..f56b940553b 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -1756,7 +1756,10 @@ impl Producer { arrived: super::clock::now(), }; let mut entry = Entry::plain(frame, inflated); - if let Err(err) = state.canonicalize(state.next_index, &mut entry) { + // Checked here too, so a group without shared content skips the call. + if state.content.is_some() + && let Err(err) = state.canonicalize(state.next_index, &mut entry) + { return Err(self.abort_conflict(state, err)); } let now = state.commit(entry); @@ -1934,15 +1937,14 @@ impl Producer { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn create_object(mut self, frame: frame::Header) -> Result { let meter = self.chunk_meter(frame.inflated); - let (info, buf) = match self.open_frame(frame) { + let (info, buf, admission) = match self.open_frame(frame) { Ok(opened) => opened, Err(err) => { let _ = self.handle.abort(err.clone()); return Err(err); } }; - let reserved = buf.admit(self.handle.cache.frames()); - Ok(frame::Producer::new(self, buf, info, reserved).with_meter(meter)) + Ok(frame::Producer::new(self, buf, info, admission).with_meter(meter)) } /// The meter a streamed frame's chunks bump: none for a plain frame a compressed group @@ -1959,7 +1961,7 @@ impl Producer { /// /// Its bytes are charged to the cache as they are written, not here: the declared /// size is only a promise until they arrive. - fn open_frame(&mut self, header: frame::Header) -> Result<(frame::Info, FrameBuf)> { + fn open_frame(&mut self, header: frame::Header) -> Result<(frame::Info, FrameBuf, Option)> { let frame = header.info; let timestamp = on_track(frame.timestamp, self.handle.track.timescale)?; if frame.size > MAX_CACHE_BYTES || header.inflated.is_some_and(|inflated| inflated > MAX_CACHE_BYTES) { @@ -1994,7 +1996,7 @@ impl Producer { if !state.trim(frame.size.saturating_add(metadata_bytes)) { return Err(self.abort_too_large(state)); } - let buf = match state.content.as_ref().filter(|_| !scratch) { + let (buf, admission) = match state.content.as_ref().filter(|_| !scratch) { Some(content) => { let key = Content::key(state.next_index, header.fields.as_deref(), state.wire().is_some()); let row = content.row( @@ -2009,9 +2011,10 @@ impl Producer { )?; let buf = row.fork(); buf.bill_view(state.charge.extra()); - buf + let admission = buf.admit(self.handle.cache.frames()); + (buf, admission) } - None => FrameBuf::new(frame.size as usize), + None => FrameBuf::admitted(frame.size as usize, self.handle.cache.frames()), }; let now = state.charge.record_write(); state.partial = Some(Partial { @@ -2067,6 +2070,7 @@ impl Producer { timestamp, }, buf, + admission, )) } @@ -2131,7 +2135,7 @@ impl Producer { inflated, end: 0, }; - if !owned { + if !owned && state.content.is_some() { state.canonicalize(state.next_index - 1, &mut entry)?; } // The slot was reserved and charged by `charge_partial` above.