Rollup of 10 perf-sensitive pull requests

compiler/rustc_ast/src/util/literal.rs

@@ -15,11 +15,31 @@ use crate::token::{self, Token};
 // Escapes a string, represented as a symbol. Reuses the original symbol,
 // avoiding interning, if no changes are required.
 pub fn escape_string_symbol(symbol: Symbol) -> Symbol {
-    // Don't use escape_default() here, because using it in conjunction with to_string()
-    // is slow.
     let s = symbol.as_str();
-    let mut escaped = String::with_capacity(s.len());
-    for c in s.chars() {
+
+    fn requires_escape(b: &u8) -> bool {
+        match *b {
+            b'\\' | b'\'' | b'"' => true,
+            b'\x20'..=b'\x7e' => false,
+            _ => true,
+        }
+    }
+
+    // Fast-path: if we don't need escaping, just return the original symbol
+    let Some(position) = s.as_bytes().iter().position(requires_escape) else {
+        return symbol;
+    };
+
+    // At this point we know that we need to escape something in `suffix`
+    let (prefix, suffix) = s.split_at(position);
+
+    // We set the capacity to the original size + 1, because the resulting string will be at least
+    // one character larger than the original, because of escaping.
+    let mut escaped = String::with_capacity(s.len() + 1);
+    escaped.push_str(prefix);
+
+    // Don't use escape_default() here, because using it is slower than escaping manually.
+    for c in suffix.chars() {
         match c {
             '\t' => escaped.push_str("\\t"),
             '\r' => escaped.push_str("\\r"),
@@ -31,7 +51,7 @@ pub fn escape_string_symbol(symbol: Symbol) -> Symbol {
             c => write!(escaped, "\\u{{{:x}}}", c as u32).unwrap(),
         }
     }
-    if s == escaped { symbol } else { Symbol::intern(&escaped) }
+    Symbol::intern(&escaped)
 }
 
 // Escapes a char.

compiler/rustc_borrowck/src/lib.rs

@@ -2497,13 +2497,14 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
         // partial initialization, do not complain about mutability
         // errors except for actual mutation (as opposed to an attempt
         // to do a partial initialization).
-        let previously_initialized = self.is_local_ever_initialized(place.local, state);
+        let previously_initialized = state.ever_inits.contains(place.local);
 
         // at this point, we have set up the error reporting state.
-        if let Some(init_index) = previously_initialized {
+        if previously_initialized {
             if let (AccessKind::Mutate, Some(_)) = (error_access, place.as_local()) {
                 // If this is a mutate access to an immutable local variable with no projections
                 // report the error as an illegal reassignment
+                let init_index = self.first_reaching_init(place.local, location).unwrap();
                 let init = &self.move_data.inits[init_index];
                 let assigned_span = init.span(self.body);
                 self.report_illegal_reassignment((place, span), assigned_span, place);
@@ -2516,10 +2517,14 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
         }
     }
 
-    fn is_local_ever_initialized(&self, local: Local, state: &BorrowckDomain) -> Option<InitIndex> {
+    /// Returns the first init of `local` (in gather order) that may have executed on some path
+    /// reaching `location` without an intervening `StorageDead(local)`.
+    fn first_reaching_init(&self, local: Local, location: Location) -> Option<InitIndex> {
         let mpi = self.move_data.rev_lookup.find_local(local)?;
-        let ii = &self.move_data.init_path_map[mpi];
-        ii.into_iter().find(|&&index| state.ever_inits.contains(index)).copied()
+        self.move_data.init_path_map[mpi].iter().copied().find(|&ii| {
+            let init = self.move_data.inits[ii];
+            EverInitializedPlaces::init_reaches_location(self.body, local, init, location)
+        })
     }
 
     /// Adds the place into the used mutable variables set
@@ -2530,7 +2535,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
                 // mutated, then it is justified to be annotated with the `mut`
                 // keyword, since the mutation may be a possible reassignment.
                 if is_local_mutation_allowed != LocalMutationIsAllowed::Yes
-                    && self.is_local_ever_initialized(local, state).is_some()
+                    && state.ever_inits.contains(local)
                 {
                     self.used_mut.insert(local);
                 }

compiler/rustc_const_eval/src/interpret/call.rs

@@ -315,7 +315,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             self.storage_live_dyn(local, meta)?;
         }
         // Now we can finally actually evaluate the callee place.
-        let callee_arg = self.eval_place(*callee_arg)?;
+        let callee_arg =
+            self.eval_place(*callee_arg, /* skip_validity_for_simple_deref */ false)?;
         // We allow some transmutes here.
         // FIXME: Depending on the PassMode, this should reset some padding to uninitialized. (This
         // is true for all `copy_op`, but there are a lot of special cases for argument passing
@@ -498,7 +499,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     // This argument is a VaList holding the remaining caller-side arguments.
                     ecx.storage_live(local)?;
 
-                    let place = ecx.eval_place(dest)?;
+                    let place =
+                        ecx.eval_place(dest, /* skip_validity_for_simple_deref */ false)?;
                     let mplace = ecx.force_allocation(&place)?;
 
                     // Consume the remaining arguments by putting them into the variable argument

compiler/rustc_const_eval/src/interpret/place.rs

@@ -479,54 +479,57 @@ where
         trace!("deref to {} on {:?}", val.layout.ty, *val);
         let mplace = self.imm_ptr_to_mplace(&val)?;
 
-        // This is conceptually a typed load from `src` to get the pointer. Most of the time when
-        // we do typed loads for primitive operations, all relevant invariants are checked
-        // implicitly, e.g. when we call `to_bool()` on a Boolean.
-        // But here, we do need to specifically check for metadata validity, null, alignment, and
-        // dereferenceability, or they will not be checked anywhere at all.
-        // This duplicates some of the logic in the validity check, but so far we found no
-        // good way to share that logic.
-        if ptr_ty.is_ref() || ptr_ty.is_box() {
-            let kind = if ptr_ty.is_ref() { "reference" } else { "box" };
-
-            // Null check.
-            let scalar_ptr = Scalar::from_maybe_pointer(mplace.ptr(), self);
-            if self.scalar_may_be_null(scalar_ptr)? {
-                let maybe = !M::Provenance::OFFSET_IS_ADDR && matches!(scalar_ptr, Scalar::Ptr(..));
-                throw_ub_format!(
-                    "dereferencing a {maybe}null {kind}",
-                    maybe = if maybe { "maybe-" } else { "" }
-                );
-            }
+        if M::enforce_validity(self, val.layout) {
+            // This is conceptually a typed load from `src` to get the pointer. Most of the time when
+            // we do typed loads for primitive operations, all relevant invariants are checked
+            // implicitly, e.g. when we call `to_bool()` on a Boolean.
+            // But here, we do need to specifically check for metadata validity, null, alignment, and
+            // dereferenceability, or they will not be checked anywhere at all.
+            // This duplicates some of the logic in the validity check, but so far we found no
+            // good way to share that logic.
+            if ptr_ty.is_ref() || ptr_ty.is_box() {
+                let kind = if ptr_ty.is_ref() { "reference" } else { "box" };
+
+                // Null check.
+                let scalar_ptr = Scalar::from_maybe_pointer(mplace.ptr(), self);
+                if self.scalar_may_be_null(scalar_ptr)? {
+                    let maybe =
+                        !M::Provenance::OFFSET_IS_ADDR && matches!(scalar_ptr, Scalar::Ptr(..));
+                    throw_ub_format!(
+                        "dereferencing a {maybe}null {kind}",
+                        maybe = if maybe { "maybe-" } else { "" }
+                    );
+                }
 
-            // Dereferencability and alignment check. This also implicitly checks metadata validity.
-            let (size, align) = self
-                .size_and_align_of_val(&mplace)?
-                .unwrap_or_else(|| (mplace.layout.size, mplace.layout.align.abi));
-            self.check_ptr_access(mplace.ptr(), size, CheckInAllocMsg::Dereferenceable(kind))?;
-            self.check_ptr_align(mplace.ptr(), align).map_err_kind(|err| {
+                // Dereferencability and alignment check. This also implicitly checks metadata validity.
+                let (size, align) = self
+                    .size_and_align_of_val(&mplace)?
+                    .unwrap_or_else(|| (mplace.layout.size, mplace.layout.align.abi));
+                self.check_ptr_access(mplace.ptr(), size, CheckInAllocMsg::Dereferenceable(kind))?;
+                self.check_ptr_align(mplace.ptr(), align).map_err_kind(|err| {
                 let err_ub!(AlignmentCheckFailed(Misalignment { required, has }, _msg)) = err else { bug!() };
                 err_ub_format!(
                     "encountered an unaligned {kind} (required {required_bytes} byte alignment but found {found_bytes})",
                     required_bytes = required.bytes(),
                     found_bytes = has.bytes()
                 )
             })?;
-        } else {
-            assert!(ptr_ty.is_raw_ptr());
-            // For raw pointers, the validity invariant is pretty weak, but we do require the vtable
-            // to make sense, so we do have to check that if there is one.
-            if mplace.layout.is_unsized() {
-                let tail = self.tcx.struct_tail_for_codegen(mplace.layout.ty, self.typing_env);
-                match tail.kind() {
-                    ty::Dynamic(data, _) => {
-                        let vtable = mplace.meta().unwrap_meta().to_pointer(self)?;
-                        self.get_ptr_vtable_ty(vtable, Some(data))?;
-                    }
-                    ty::Slice(..) | ty::Str | ty::Foreign(..) => {
-                        // Nothing to check (`read_immediate` already ensured initialization).
+            } else {
+                assert!(ptr_ty.is_raw_ptr());
+                // For raw pointers, the validity invariant is pretty weak, but we do require the vtable
+                // to make sense, so we do have to check that if there is one.
+                if mplace.layout.is_unsized() {
+                    let tail = self.tcx.struct_tail_for_codegen(mplace.layout.ty, self.typing_env);
+                    match tail.kind() {
+                        ty::Dynamic(data, _) => {
+                            let vtable = mplace.meta().unwrap_meta().to_pointer(self)?;
+                            self.get_ptr_vtable_ty(vtable, Some(data))?;
+                        }
+                        ty::Slice(..) | ty::Str | ty::Foreign(..) => {
+                            // Nothing to check (`read_immediate` already ensured initialization).
+                        }
+                        _ => bug!("Unexpected unsized type tail: {:?}", tail),
                     }
-                    _ => bug!("Unexpected unsized type tail: {:?}", tail),
                 }
             }
         }
@@ -590,15 +593,27 @@ where
 
     /// Computes a place. You should only use this if you intend to write into this
     /// place; for reading, a more efficient alternative is `eval_place_to_op`.
+    ///
+    /// If `skip_validity_for_simple_deref` is true, then we do not check validity of the inner
+    /// pointer for places of the form `*ptr`. The caller must justify why that is okay.
     #[instrument(skip(self), level = "trace")]
     pub fn eval_place(
         &self,
         mir_place: mir::Place<'tcx>,
+        skip_validity_for_simple_deref: bool,
     ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
         let _trace =
             enter_trace_span!(M, step::eval_place, ?mir_place, tracing_separate_thread = Empty);
 
         let mut place = self.local_to_place(mir_place.local)?;
+        if skip_validity_for_simple_deref
+            && mir_place.projection.as_slice() == &[mir::ProjectionElem::Deref]
+        {
+            // We want to skip the checks in `deref_pointer`.
+            let val = self.read_immediate(&place)?;
+            let place = self.imm_ptr_to_mplace(&val)?;
+            return interp_ok(place.into());
+        }
         // Using `try_fold` turned out to be bad for performance, hence the loop.
         for elem in mir_place.projection.iter() {
             place = self.project(&place, elem)?

compiler/rustc_const_eval/src/interpret/projection.rs

@@ -414,7 +414,7 @@ where
             UnwrapUnsafeBinder(target) => base.transmute(self.layout_of(target)?, self)?,
             Field(field, _) => self.project_field(base, field)?,
             Downcast(_, variant) => self.project_downcast(base, variant)?,
-            Deref => self.deref_pointer(&base.to_op(self)?)?.into(),
+            Deref => self.deref_pointer(base)?.into(),
             Index(local) => {
                 let layout = self.layout_of(self.tcx.types.usize)?;
                 let n = self.local_to_op(local, Some(layout))?;

compiler/rustc_const_eval/src/interpret/step.rs

@@ -93,7 +93,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             Assign((place, rvalue)) => self.eval_rvalue_into_place(rvalue, *place)?,
 
             SetDiscriminant { place, variant_index } => {
-                let dest = self.eval_place(**place)?;
+                let dest =
+                    self.eval_place(**place, /* skip_validity_for_simple_deref */ false)?;
                 self.write_discriminant(*variant_index, &dest)?;
             }
 
@@ -115,7 +116,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
 
             // Evaluate the place expression, without reading from it.
             PlaceMention(place) => {
-                let _ = self.eval_place(**place)?;
+                let _ =
+                    self.eval_place(**place, /* skip_validity_for_simple_deref */ false)?;
             }
 
             // This exists purely to guide borrowck lifetime inference, and does not have
@@ -159,7 +161,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         rvalue: &mir::Rvalue<'tcx>,
         place: mir::Place<'tcx>,
     ) -> InterpResult<'tcx> {
-        let dest = self.eval_place(place)?;
+        // We can skip validity because we'll write to the place which checks everything we care
+        // about for references, and the pointee must be sized so there's nothing to check for raw
+        // pointers.
+        let dest = self.eval_place(place, /* skip_validity_for_simple_deref */ true)?;
         // FIXME: ensure some kind of non-aliasing between LHS and RHS?
         // Also see https://github.com/rust-lang/rust/issues/68364.
 
@@ -206,7 +211,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             }
 
             Ref(_, borrow_kind, place) => {
-                let src = self.eval_place(place)?;
+                // `x = &*ptr` does not need a validity check on `ptr` because we will already
+                // check `x` below.
+                let src = self.eval_place(place, /* skip_validity_for_simple_deref */ true)?;
                 let place = self.force_allocation(&src)?;
                 let mut val = ImmTy::from_immediate(place.to_ref(self), dest.layout);
                 // A fresh reference was created, make sure it gets retagged with the right mode.
@@ -251,7 +258,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     false
                 };
 
-                let src = self.eval_place(place)?;
+                let src =
+                    self.eval_place(place, /* skip_validity_for_simple_deref */ false)?;
                 let place = self.force_allocation(&src)?;
                 let mut val = ImmTy::from_immediate(place.to_ref(self), dest.layout);
                 if !place_base_raw && !kind.is_fake() {
@@ -403,7 +411,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 FnArg::Copy(op)
             }
             mir::Operand::Move(place) => {
-                let place = self.eval_place(*place)?;
+                // We will read from this place, which checks everything there is to check,
+                // so we can skip the extra validity check here.
+                let place =
+                    self.eval_place(*place, /* skip_validity_for_simple_deref */ true)?;
                 if move_definitely_disjoint {
                     // We still have to ensure that no *other* pointers are used to access this place,
                     // so *if* it is in memory then we have to treat it as `InPlace`.
@@ -553,7 +564,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 let old_loc = self.frame().loc;
 
                 // Evaluation order consistent with assignment: destination first.
-                let dest_place = self.eval_place(destination)?;
+                let dest_place =
+                    self.eval_place(destination, /* skip_validity_for_simple_deref */ false)?;
                 let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =
                     self.eval_callee_and_args(terminator, func, args, &destination)?;
 
@@ -602,7 +614,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     drop.is_none(),
                     "Async Drop must be expanded or reset to sync in runtime MIR"
                 );
-                let place = self.eval_place(place)?;
+                let place =
+                    self.eval_place(place, /* skip_validity_for_simple_deref */ false)?;
                 let instance = {
                     let _trace =
                         enter_trace_span!(M, resolve::resolve_drop_glue, ty = ?place.layout.ty);

compiler/rustc_hir_analysis/src/check/compare_impl_item.rs

@@ -23,8 +23,9 @@ use rustc_span::{BytePos, DUMMY_SP, Span};
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
 use rustc_trait_selection::infer::InferCtxtExt;
 use rustc_trait_selection::regions::InferCtxtRegionExt;
+use rustc_trait_selection::solve::NextSolverError;
 use rustc_trait_selection::traits::{
-    self, FulfillmentError, ObligationCause, ObligationCauseCode, ObligationCtxt,
+    self, FromSolverError, FulfillmentError, ObligationCause, ObligationCauseCode, ObligationCtxt,
 };
 use tracing::{debug, instrument};
 
@@ -802,7 +803,8 @@ struct ImplTraitInTraitCollector<'a, 'tcx, E> {
 
 impl<'a, 'tcx, E> ImplTraitInTraitCollector<'a, 'tcx, E>
 where
-    E: 'tcx,
+    E: FromSolverError<'tcx, NextSolverError<'tcx>>
+        + FromSolverError<'tcx, traits::OldSolverError<'tcx>>,
 {
     fn new(
         ocx: &'a ObligationCtxt<'a, 'tcx, E>,
@@ -816,7 +818,8 @@ where
 
 impl<'tcx, E> TypeFolder<TyCtxt<'tcx>> for ImplTraitInTraitCollector<'_, 'tcx, E>
 where
-    E: 'tcx,
+    E: FromSolverError<'tcx, NextSolverError<'tcx>>
+        + FromSolverError<'tcx, traits::OldSolverError<'tcx>>,
 {
     fn cx(&self) -> TyCtxt<'tcx> {
         self.ocx.infcx.tcx