diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index e4ac89a7bec..b1adaa193b3 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -21,6 +21,7 @@ mod simd; pub(crate) use cpuid::codegen_cpuid_call; pub(crate) use llvm::codegen_llvm_intrinsic_call; +use rustc_middle::ty::layout::HasParamEnv; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::subst::SubstsRef; use rustc_span::symbol::{kw, sym, Symbol}; @@ -659,7 +660,9 @@ fn codegen_regular_intrinsic_call<'tcx>( return; } - if intrinsic == sym::assert_zero_valid && !fx.tcx.permits_zero_init(layout) { + if intrinsic == sym::assert_zero_valid + && !fx.tcx.permits_zero_init(fx.param_env().and(layout)) + { with_no_trimmed_paths!({ crate::base::codegen_panic( fx, @@ -674,7 +677,7 @@ fn codegen_regular_intrinsic_call<'tcx>( } if intrinsic == sym::assert_mem_uninitialized_valid - && !fx.tcx.permits_uninit_init(layout) + && !fx.tcx.permits_uninit_init(fx.param_env().and(layout)) { with_no_trimmed_paths!({ crate::base::codegen_panic( diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 978aff511bf..c73f415ad8f 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -678,8 +678,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let layout = bx.layout_of(ty); let do_panic = match intrinsic { Inhabited => layout.abi.is_uninhabited(), - ZeroValid => !bx.tcx().permits_zero_init(layout), - MemUninitializedValid => !bx.tcx().permits_uninit_init(layout), + ZeroValid => !bx.tcx().permits_zero_init(bx.param_env().and(layout)), + MemUninitializedValid => !bx.tcx().permits_uninit_init(bx.param_env().and(layout)), }; Some(if do_panic { let msg_str = with_no_visible_paths!({ diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index cc7b6c91b60..f87e4fbc1a1 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -447,7 +447,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } if intrinsic_name == sym::assert_zero_valid { - let should_panic = !self.tcx.permits_zero_init(layout); + let should_panic = !self.tcx.permits_zero_init(self.param_env.and(layout)); if should_panic { M::abort( @@ -461,7 +461,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } if intrinsic_name == sym::assert_mem_uninitialized_valid { - let should_panic = !self.tcx.permits_uninit_init(layout); + let should_panic = !self.tcx.permits_uninit_init(self.param_env.and(layout)); if should_panic { M::abort( diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs index 57b91df2d07..51624a0c6c8 100644 --- a/compiler/rustc_const_eval/src/lib.rs +++ b/compiler/rustc_const_eval/src/lib.rs @@ -58,7 +58,12 @@ pub fn provide(providers: &mut Providers) { let (param_env, value) = param_env_and_value.into_parts(); const_eval::deref_mir_constant(tcx, param_env, value) }; - providers.permits_uninit_init = - |tcx, ty| util::might_permit_raw_init(tcx, ty, InitKind::UninitMitigated0x01Fill); - providers.permits_zero_init = |tcx, ty| util::might_permit_raw_init(tcx, ty, InitKind::Zero); + providers.permits_uninit_init = |tcx, param_env_and_ty| { + let (param_env, ty) = param_env_and_ty.into_parts(); + util::might_permit_raw_init(tcx, param_env, ty, InitKind::UninitMitigated0x01Fill) + }; + providers.permits_zero_init = |tcx, param_env_and_ty| { + let (param_env, ty) = param_env_and_ty.into_parts(); + util::might_permit_raw_init(tcx, param_env, ty, InitKind::Zero) + }; } diff --git a/compiler/rustc_const_eval/src/util/might_permit_raw_init.rs b/compiler/rustc_const_eval/src/util/might_permit_raw_init.rs index 4ce107ea68d..48961b7aac6 100644 --- a/compiler/rustc_const_eval/src/util/might_permit_raw_init.rs +++ b/compiler/rustc_const_eval/src/util/might_permit_raw_init.rs @@ -20,13 +20,14 @@ use crate::interpret::{InterpCx, MemoryKind, OpTy}; /// to the full uninit check). pub fn might_permit_raw_init<'tcx>( tcx: TyCtxt<'tcx>, + param_env: ParamEnv<'tcx>, ty: TyAndLayout<'tcx>, kind: InitKind, ) -> bool { if tcx.sess.opts.unstable_opts.strict_init_checks { might_permit_raw_init_strict(ty, tcx, kind) } else { - let layout_cx = LayoutCx { tcx, param_env: ParamEnv::reveal_all() }; + let layout_cx = LayoutCx { tcx, param_env }; might_permit_raw_init_lax(ty, &layout_cx, kind) } } diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 7db86c8d0d4..b1e0d380d04 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -2116,12 +2116,12 @@ rustc_queries! { separate_provide_extern } - query permits_uninit_init(key: TyAndLayout<'tcx>) -> bool { - desc { "checking to see if `{}` permits being left uninit", key.ty } + query permits_uninit_init(key: ty::ParamEnvAnd<'tcx, TyAndLayout<'tcx>>) -> bool { + desc { "checking to see if `{}` permits being left uninit", key.value.ty } } - query permits_zero_init(key: TyAndLayout<'tcx>) -> bool { - desc { "checking to see if `{}` permits being left zeroed", key.ty } + query permits_zero_init(key: ty::ParamEnvAnd<'tcx, TyAndLayout<'tcx>>) -> bool { + desc { "checking to see if `{}` permits being left zeroed", key.value.ty } } query compare_impl_const( diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 7d4d35b7fdf..2de886a3e81 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -11,6 +11,7 @@ use crate::ty::{self, AliasTy, InferConst, Lift, Term, TermKind, Ty, TyCtxt}; use rustc_data_structures::functor::IdFunctor; use rustc_hir::def::Namespace; use rustc_index::vec::{Idx, IndexVec}; +use rustc_target::abi::TyAndLayout; use std::fmt; use std::mem::ManuallyDrop; @@ -853,3 +854,9 @@ impl<'tcx> TypeSuperVisitable<'tcx> for ty::UnevaluatedConst<'tcx> { self.substs.visit_with(visitor) } } + +impl<'tcx> TypeVisitable<'tcx> for TyAndLayout<'tcx, Ty<'tcx>> { + fn visit_with>(&self, visitor: &mut V) -> ControlFlow { + visitor.visit_ty(self.ty) + } +} diff --git a/compiler/rustc_mir_transform/src/instcombine.rs b/compiler/rustc_mir_transform/src/instcombine.rs index 2f3c65869ef..e1faa7a08d9 100644 --- a/compiler/rustc_mir_transform/src/instcombine.rs +++ b/compiler/rustc_mir_transform/src/instcombine.rs @@ -6,7 +6,8 @@ use rustc_middle::mir::{ BinOp, Body, Constant, ConstantKind, LocalDecls, Operand, Place, ProjectionElem, Rvalue, SourceInfo, Statement, StatementKind, Terminator, TerminatorKind, UnOp, }; -use rustc_middle::ty::{self, TyCtxt}; +use rustc_middle::ty::{self, layout::TyAndLayout, ParamEnv, ParamEnvAnd, SubstsRef, Ty, TyCtxt}; +use rustc_span::symbol::{sym, Symbol}; pub struct InstCombine; @@ -16,7 +17,11 @@ impl<'tcx> MirPass<'tcx> for InstCombine { } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let ctx = InstCombineContext { tcx, local_decls: &body.local_decls }; + let ctx = InstCombineContext { + tcx, + local_decls: &body.local_decls, + param_env: tcx.param_env_reveal_all_normalized(body.source.def_id()), + }; for block in body.basic_blocks.as_mut() { for statement in block.statements.iter_mut() { match statement.kind { @@ -33,6 +38,10 @@ impl<'tcx> MirPass<'tcx> for InstCombine { &mut block.terminator.as_mut().unwrap(), &mut block.statements, ); + ctx.combine_intrinsic_assert( + &mut block.terminator.as_mut().unwrap(), + &mut block.statements, + ); } } } @@ -40,6 +49,7 @@ impl<'tcx> MirPass<'tcx> for InstCombine { struct InstCombineContext<'tcx, 'a> { tcx: TyCtxt<'tcx>, local_decls: &'a LocalDecls<'tcx>, + param_env: ParamEnv<'tcx>, } impl<'tcx> InstCombineContext<'tcx, '_> { @@ -200,4 +210,76 @@ impl<'tcx> InstCombineContext<'tcx, '_> { }); terminator.kind = TerminatorKind::Goto { target: destination_block }; } + + fn combine_intrinsic_assert( + &self, + terminator: &mut Terminator<'tcx>, + _statements: &mut Vec>, + ) { + let TerminatorKind::Call { func, target, .. } = &mut terminator.kind else { return; }; + let Some(target_block) = target else { return; }; + let func_ty = func.ty(self.local_decls, self.tcx); + let Some((intrinsic_name, substs)) = resolve_rust_intrinsic(self.tcx, func_ty) else { + return; + }; + // The intrinsics we are interested in have one generic parameter + if substs.is_empty() { + return; + } + let ty = substs.type_at(0); + + // Check this is a foldable intrinsic before we query the layout of our generic parameter + let Some(assert_panics) = intrinsic_assert_panics(intrinsic_name) else { return; }; + let Ok(layout) = self.tcx.layout_of(self.param_env.and(ty)) else { return; }; + if assert_panics(self.tcx, self.param_env.and(layout)) { + // If we know the assert panics, indicate to later opts that the call diverges + *target = None; + } else { + // If we know the assert does not panic, turn the call into a Goto + terminator.kind = TerminatorKind::Goto { target: *target_block }; + } + } +} + +fn intrinsic_assert_panics<'tcx>( + intrinsic_name: Symbol, +) -> Option, ParamEnvAnd<'tcx, TyAndLayout<'tcx>>) -> bool> { + fn inhabited_predicate<'tcx>( + _tcx: TyCtxt<'tcx>, + param_env_and_layout: ParamEnvAnd<'tcx, TyAndLayout<'tcx>>, + ) -> bool { + let (_param_env, layout) = param_env_and_layout.into_parts(); + layout.abi.is_uninhabited() + } + fn zero_valid_predicate<'tcx>( + tcx: TyCtxt<'tcx>, + param_env_and_layout: ParamEnvAnd<'tcx, TyAndLayout<'tcx>>, + ) -> bool { + !tcx.permits_zero_init(param_env_and_layout) + } + fn mem_uninitialized_valid_predicate<'tcx>( + tcx: TyCtxt<'tcx>, + param_env_and_layout: ParamEnvAnd<'tcx, TyAndLayout<'tcx>>, + ) -> bool { + !tcx.permits_uninit_init(param_env_and_layout) + } + + match intrinsic_name { + sym::assert_inhabited => Some(inhabited_predicate), + sym::assert_zero_valid => Some(zero_valid_predicate), + sym::assert_mem_uninitialized_valid => Some(mem_uninitialized_valid_predicate), + _ => None, + } +} + +fn resolve_rust_intrinsic<'tcx>( + tcx: TyCtxt<'tcx>, + func_ty: Ty<'tcx>, +) -> Option<(Symbol, SubstsRef<'tcx>)> { + if let ty::FnDef(def_id, substs) = *func_ty.kind() { + if tcx.is_intrinsic(def_id) { + return Some((tcx.item_name(def_id), substs)); + } + } + None } diff --git a/tests/mir-opt/intrinsic_asserts.generic.InstCombine.diff b/tests/mir-opt/intrinsic_asserts.generic.InstCombine.diff new file mode 100644 index 00000000000..8ff64c1ea15 --- /dev/null +++ b/tests/mir-opt/intrinsic_asserts.generic.InstCombine.diff @@ -0,0 +1,42 @@ +- // MIR for `generic` before InstCombine ++ // MIR for `generic` after InstCombine + + fn generic() -> () { + let mut _0: (); // return place in scope 0 at $DIR/intrinsic_asserts.rs:+0:21: +0:21 + let _1: (); // in scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:46 + let _2: (); // in scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:47 + let _3: (); // in scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:60 + + bb0: { + StorageLive(_1); // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:46 + _1 = assert_inhabited::() -> bb1; // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:46 + // mir::Constant + // + span: $DIR/intrinsic_asserts.rs:25:5: 25:44 + // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_inhabited::}, val: Value() } + } + + bb1: { + StorageDead(_1); // scope 0 at $DIR/intrinsic_asserts.rs:+1:46: +1:47 + StorageLive(_2); // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:47 + _2 = assert_zero_valid::() -> bb2; // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:47 + // mir::Constant + // + span: $DIR/intrinsic_asserts.rs:26:5: 26:45 + // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_zero_valid::}, val: Value() } + } + + bb2: { + StorageDead(_2); // scope 0 at $DIR/intrinsic_asserts.rs:+2:47: +2:48 + StorageLive(_3); // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:60 + _3 = assert_mem_uninitialized_valid::() -> bb3; // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:60 + // mir::Constant + // + span: $DIR/intrinsic_asserts.rs:27:5: 27:58 + // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_mem_uninitialized_valid::}, val: Value() } + } + + bb3: { + StorageDead(_3); // scope 0 at $DIR/intrinsic_asserts.rs:+3:60: +3:61 + nop; // scope 0 at $DIR/intrinsic_asserts.rs:+0:21: +4:2 + return; // scope 0 at $DIR/intrinsic_asserts.rs:+4:2: +4:2 + } + } + diff --git a/tests/mir-opt/intrinsic_asserts.panics.InstCombine.diff b/tests/mir-opt/intrinsic_asserts.panics.InstCombine.diff new file mode 100644 index 00000000000..ddc01590315 --- /dev/null +++ b/tests/mir-opt/intrinsic_asserts.panics.InstCombine.diff @@ -0,0 +1,47 @@ +- // MIR for `panics` before InstCombine ++ // MIR for `panics` after InstCombine + + fn panics() -> () { + let mut _0: (); // return place in scope 0 at $DIR/intrinsic_asserts.rs:+0:17: +0:17 + let _1: (); // in scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:50 + let _2: (); // in scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:49 + let _3: (); // in scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:62 + + bb0: { + StorageLive(_1); // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:50 +- _1 = assert_inhabited::() -> bb1; // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:50 ++ _1 = assert_inhabited::(); // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:50 + // mir::Constant + // + span: $DIR/intrinsic_asserts.rs:17:5: 17:48 + // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_inhabited::}, val: Value() } + } + + bb1: { + StorageDead(_1); // scope 0 at $DIR/intrinsic_asserts.rs:+1:50: +1:51 + StorageLive(_2); // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:49 +- _2 = assert_zero_valid::<&u8>() -> bb2; // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:49 ++ _2 = assert_zero_valid::<&u8>(); // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:49 + // mir::Constant + // + span: $DIR/intrinsic_asserts.rs:18:5: 18:47 + // + user_ty: UserType(0) + // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_zero_valid::<&u8>}, val: Value() } + } + + bb2: { + StorageDead(_2); // scope 0 at $DIR/intrinsic_asserts.rs:+2:49: +2:50 + StorageLive(_3); // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:62 +- _3 = assert_mem_uninitialized_valid::<&u8>() -> bb3; // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:62 ++ _3 = assert_mem_uninitialized_valid::<&u8>(); // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:62 + // mir::Constant + // + span: $DIR/intrinsic_asserts.rs:19:5: 19:60 + // + user_ty: UserType(1) + // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_mem_uninitialized_valid::<&u8>}, val: Value() } + } + + bb3: { + StorageDead(_3); // scope 0 at $DIR/intrinsic_asserts.rs:+3:62: +3:63 + nop; // scope 0 at $DIR/intrinsic_asserts.rs:+0:17: +4:2 + return; // scope 0 at $DIR/intrinsic_asserts.rs:+4:2: +4:2 + } + } + diff --git a/tests/mir-opt/intrinsic_asserts.removable.InstCombine.diff b/tests/mir-opt/intrinsic_asserts.removable.InstCombine.diff new file mode 100644 index 00000000000..568fbf1a0d6 --- /dev/null +++ b/tests/mir-opt/intrinsic_asserts.removable.InstCombine.diff @@ -0,0 +1,45 @@ +- // MIR for `removable` before InstCombine ++ // MIR for `removable` after InstCombine + + fn removable() -> () { + let mut _0: (); // return place in scope 0 at $DIR/intrinsic_asserts.rs:+0:20: +0:20 + let _1: (); // in scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:47 + let _2: (); // in scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:48 + let _3: (); // in scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:61 + + bb0: { + StorageLive(_1); // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:47 +- _1 = assert_inhabited::<()>() -> bb1; // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:47 +- // mir::Constant +- // + span: $DIR/intrinsic_asserts.rs:7:5: 7:45 +- // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_inhabited::<()>}, val: Value() } ++ goto -> bb1; // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:47 + } + + bb1: { + StorageDead(_1); // scope 0 at $DIR/intrinsic_asserts.rs:+1:47: +1:48 + StorageLive(_2); // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:48 +- _2 = assert_zero_valid::() -> bb2; // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:48 +- // mir::Constant +- // + span: $DIR/intrinsic_asserts.rs:8:5: 8:46 +- // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_zero_valid::}, val: Value() } ++ goto -> bb2; // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:48 + } + + bb2: { + StorageDead(_2); // scope 0 at $DIR/intrinsic_asserts.rs:+2:48: +2:49 + StorageLive(_3); // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:61 +- _3 = assert_mem_uninitialized_valid::() -> bb3; // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:61 +- // mir::Constant +- // + span: $DIR/intrinsic_asserts.rs:9:5: 9:59 +- // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_mem_uninitialized_valid::}, val: Value() } ++ goto -> bb3; // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:61 + } + + bb3: { + StorageDead(_3); // scope 0 at $DIR/intrinsic_asserts.rs:+3:61: +3:62 + nop; // scope 0 at $DIR/intrinsic_asserts.rs:+0:20: +4:2 + return; // scope 0 at $DIR/intrinsic_asserts.rs:+4:2: +4:2 + } + } + diff --git a/tests/mir-opt/intrinsic_asserts.rs b/tests/mir-opt/intrinsic_asserts.rs new file mode 100644 index 00000000000..8fb99cdf6e0 --- /dev/null +++ b/tests/mir-opt/intrinsic_asserts.rs @@ -0,0 +1,28 @@ +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +// All these assertions pass, so all the intrinsic calls should be deleted. +// EMIT_MIR intrinsic_asserts.removable.InstCombine.diff +pub fn removable() { + core::intrinsics::assert_inhabited::<()>(); + core::intrinsics::assert_zero_valid::(); + core::intrinsics::assert_mem_uninitialized_valid::(); +} + +enum Never {} + +// These assertions all diverge, so their target blocks should become None. +// EMIT_MIR intrinsic_asserts.panics.InstCombine.diff +pub fn panics() { + core::intrinsics::assert_inhabited::(); + core::intrinsics::assert_zero_valid::<&u8>(); + core::intrinsics::assert_mem_uninitialized_valid::<&u8>(); +} + +// Whether or not these asserts pass isn't known, so they shouldn't be modified. +// EMIT_MIR intrinsic_asserts.generic.InstCombine.diff +pub fn generic() { + core::intrinsics::assert_inhabited::(); + core::intrinsics::assert_zero_valid::(); + core::intrinsics::assert_mem_uninitialized_valid::(); +}