Skip to content

Commit 0bc61de

Browse files
scottmcmscottmcm
committed
Let intrinsics use the SSA operand path
1 parent d458387 commit 0bc61de

19 files changed

Lines changed: 376 additions & 266 deletions

File tree

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -694,13 +694,12 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
694694
self.context.new_rvalue_from_int(self.int_type, 0)
695695
}
696696

697-
fn va_start(&mut self, _va_list: RValue<'gcc>) -> RValue<'gcc> {
697+
fn va_start(&mut self, _va_list: RValue<'gcc>) {
698698
unimplemented!();
699699
}
700700

701-
fn va_end(&mut self, _va_list: RValue<'gcc>) -> RValue<'gcc> {
701+
fn va_end(&mut self, _va_list: RValue<'gcc>) {
702702
// FIXME(antoyo): implement.
703-
self.context.new_rvalue_from_int(self.int_type, 0)
704703
}
705704
}
706705

‎compiler/rustc_codegen_llvm/src/context.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ use rustc_session::Session;
2525
use rustc_session::config::{
2626
BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, FunctionReturn, PAuthKey, PacRet,
2727
};
28-
use rustc_span::{DUMMY_SP, Span, Spanned, Symbol};
28+
use rustc_span::{DUMMY_SP, Span, Spanned, Symbol, sym};
2929
use rustc_symbol_mangling::mangle_internal_symbol;
3030
use rustc_target::spec::{
3131
Arch, CfgAbi, Env, HasTargetSpec, Os, RelocModel, SmallDataThresholdSupport, Target, TlsModel,
@@ -937,6 +937,17 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
937937
None
938938
}
939939
}
940+
941+
fn intrinsic_call_expects_place_always(&self, name: Symbol) -> bool {
942+
matches!(
943+
name,
944+
sym::autodiff
945+
| sym::catch_unwind
946+
| sym::volatile_load
947+
| sym::unaligned_volatile_load
948+
| sym::black_box
949+
)
950+
}
940951
}
941952

942953
impl<'ll> CodegenCx<'ll, '_> {

‎compiler/rustc_codegen_llvm/src/intrinsic.rs‎

Lines changed: 83 additions & 52 deletions
Large diffs are not rendered by default.

‎compiler/rustc_codegen_ssa/src/mir/analyze.rs‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ use rustc_index::bit_set::DenseBitSet;
77
use rustc_index::{IndexSlice, IndexVec};
88
use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor};
99
use rustc_middle::mir::{self, DefLocation, Location, TerminatorKind, traversal};
10-
use rustc_middle::ty::layout::LayoutOf;
11-
use rustc_middle::{bug, span_bug};
10+
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
11+
use rustc_middle::{bug, span_bug, ty};
1212
use tracing::debug;
1313

1414
use super::FunctionCx;
@@ -55,7 +55,7 @@ pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
5555
non_ssa_locals
5656
}
5757

58-
#[derive(Copy, Clone, PartialEq, Eq)]
58+
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
5959
enum LocalKind {
6060
ZST,
6161
/// A local that requires an alloca.
@@ -195,12 +195,20 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer
195195
match context {
196196
PlaceContext::MutatingUse(MutatingUseContext::Call) => {
197197
let call = location.block;
198-
let TerminatorKind::Call { target, .. } =
199-
self.fx.mir.basic_blocks[call].terminator().kind
198+
let TerminatorKind::Call { target, func, .. } =
199+
&self.fx.mir.basic_blocks[call].terminator().kind
200200
else {
201201
bug!()
202202
};
203-
self.define(local, DefLocation::CallReturn { call, target });
203+
let tcx = self.fx.cx.tcx();
204+
let func_ty = func.ty(&self.fx.mir.local_decls, tcx);
205+
if let ty::FnDef(def_id, _args) = *func_ty.kind()
206+
&& let Some(intrinsic) = tcx.intrinsic(def_id)
207+
&& self.fx.cx.intrinsic_call_expects_place_always(intrinsic.name)
208+
{
209+
self.locals[local] = LocalKind::Memory;
210+
}
211+
self.define(local, DefLocation::CallReturn { call, target: *target });
204212
}
205213

206214
PlaceContext::NonUse(_)

‎compiler/rustc_codegen_ssa/src/mir/block.rs‎

Lines changed: 52 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use super::{CachedLlbb, FunctionCx, LocalRef};
2323
use crate::base::{self, is_call_from_compiler_builtins_to_upstream_monomorphization};
2424
use crate::common::{self, IntPredicate};
2525
use crate::errors::CompilerBuiltinsCannotCall;
26+
use crate::mir::IntrinsicResult;
2627
use crate::traits::*;
2728
use crate::{MemFlags, meth};
2829

@@ -945,32 +946,31 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
945946
let result_layout =
946947
self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref()));
947948

948-
let (result, store_in_local) = if result_layout.is_zst() {
949-
(
950-
PlaceRef::new_sized(bx.const_undef(bx.type_ptr()), result_layout),
951-
None,
952-
)
953-
} else if let Some(local) = destination.as_local() {
954-
match self.locals[local] {
955-
LocalRef::Place(dest) => (dest, None),
956-
LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
957-
LocalRef::PendingOperand => {
958-
// Currently, intrinsics always need a location to store
959-
// the result, so we create a temporary `alloca` for the
960-
// result.
961-
let tmp = PlaceRef::alloca(bx, result_layout);
962-
tmp.storage_live(bx);
963-
(tmp, Some(local))
949+
let (result_place, store_in_local) =
950+
if let Some(local) = destination.as_local() {
951+
match self.locals[local] {
952+
LocalRef::Place(dest) => (Some(dest.val), None),
953+
LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
954+
LocalRef::PendingOperand => (None, Some(local)),
955+
LocalRef::Operand(_) => {
956+
if result_layout.is_zst() {
957+
let place = PlaceRef::new_sized(
958+
bx.const_undef(bx.type_ptr()),
959+
result_layout,
960+
);
961+
(Some(place.val), None)
962+
} else {
963+
bug!("place local already assigned to");
964+
}
965+
}
964966
}
965-
LocalRef::Operand(_) => {
966-
bug!("place local already assigned to");
967-
}
968-
}
969-
} else {
970-
(self.codegen_place(bx, destination.as_ref()), None)
971-
};
967+
} else {
968+
(Some(self.codegen_place(bx, destination.as_ref()).val), None)
969+
};
972970

973-
if result.val.align < result.layout.align.abi {
971+
if let Some(place) = result_place
972+
&& place.align < result_layout.align.abi
973+
{
974974
// Currently, MIR code generation does not create calls
975975
// that store directly to fields of packed structs (in
976976
// fact, the calls it creates write only to temps).
@@ -983,24 +983,45 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
983983
let args: Vec<_> =
984984
args.iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect();
985985

986-
match self.codegen_intrinsic_call(bx, instance, &args, result, source_info)
987-
{
988-
Ok(()) => {
989-
if let Some(local) = store_in_local {
990-
let op = bx.load_operand(result);
991-
result.storage_dead(bx);
986+
let intrinsic_result = self.codegen_intrinsic_call(
987+
bx,
988+
instance,
989+
&args,
990+
result_layout,
991+
result_place,
992+
source_info,
993+
);
994+
995+
if let IntrinsicResult::Operand(op_val) = intrinsic_result {
996+
match (result_place, store_in_local) {
997+
(None, Some(local)) => {
998+
let op = OperandRef {
999+
val: op_val,
1000+
layout: result_layout,
1001+
move_annotation: None,
1002+
};
9921003
self.overwrite_local(local, LocalRef::Operand(op));
9931004
self.debug_introduce_local(bx, local);
9941005
}
1006+
(Some(place_val), None) => {
1007+
let dest = PlaceRef { val: place_val, layout: result_layout };
1008+
op_val.store(bx, dest);
1009+
}
1010+
_ => bug!(),
1011+
}
1012+
}
9951013

1014+
match intrinsic_result {
1015+
IntrinsicResult::Operand(_) | IntrinsicResult::WroteIntoPlace => {
9961016
return if let Some(target) = target {
9971017
helper.funclet_br(self, bx, target, mergeable_succ)
9981018
} else {
9991019
bx.unreachable();
10001020
MergingSucc::False
10011021
};
10021022
}
1003-
Err(instance) => {
1023+
IntrinsicResult::Err(_) => return MergingSucc::False,
1024+
IntrinsicResult::Fallback(instance) => {
10041025
if intrinsic.must_be_overridden {
10051026
span_bug!(
10061027
fn_span,

0 commit comments

Comments
 (0)