Auto merge of #105582 - saethlin:instcombine-assert-inhabited, r=cjgillot

InstCombine away intrinsic validity assertions

This optimization (currently) fires 246 times on the standard library. It seems to fire hardly at all on the big crates in the benchmark suite. Interesting.

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -21,6 +21,7 @@ mod simd;
 pub(crate) use cpuid::codegen_cpuid_call;
 pub(crate) use llvm::codegen_llvm_intrinsic_call;
 
+use rustc_middle::ty::layout::HasParamEnv;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_span::symbol::{kw, sym, Symbol};
@@ -659,7 +660,9 @@ fn codegen_regular_intrinsic_call<'tcx>(
                 return;
             }
 
-            if intrinsic == sym::assert_zero_valid && !fx.tcx.permits_zero_init(layout) {
+            if intrinsic == sym::assert_zero_valid
+                && !fx.tcx.permits_zero_init(fx.param_env().and(layout))
+            {
                 with_no_trimmed_paths!({
                     crate::base::codegen_panic(
                         fx,
@@ -674,7 +677,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             }
 
             if intrinsic == sym::assert_mem_uninitialized_valid
-                && !fx.tcx.permits_uninit_init(layout)
+                && !fx.tcx.permits_uninit_init(fx.param_env().and(layout))
             {
                 with_no_trimmed_paths!({
                     crate::base::codegen_panic(

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -678,8 +678,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             let layout = bx.layout_of(ty);
             let do_panic = match intrinsic {
                 Inhabited => layout.abi.is_uninhabited(),
-                ZeroValid => !bx.tcx().permits_zero_init(layout),
-                MemUninitializedValid => !bx.tcx().permits_uninit_init(layout),
+                ZeroValid => !bx.tcx().permits_zero_init(bx.param_env().and(layout)),
+                MemUninitializedValid => !bx.tcx().permits_uninit_init(bx.param_env().and(layout)),
             };
             Some(if do_panic {
                 let msg_str = with_no_visible_paths!({

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -447,7 +447,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 }
 
                 if intrinsic_name == sym::assert_zero_valid {
-                    let should_panic = !self.tcx.permits_zero_init(layout);
+                    let should_panic = !self.tcx.permits_zero_init(self.param_env.and(layout));
 
                     if should_panic {
                         M::abort(
@@ -461,7 +461,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 }
 
                 if intrinsic_name == sym::assert_mem_uninitialized_valid {
-                    let should_panic = !self.tcx.permits_uninit_init(layout);
+                    let should_panic = !self.tcx.permits_uninit_init(self.param_env.and(layout));
 
                     if should_panic {
                         M::abort(

compiler/rustc_const_eval/src/lib.rs

@@ -58,7 +58,12 @@ pub fn provide(providers: &mut Providers) {
         let (param_env, value) = param_env_and_value.into_parts();
         const_eval::deref_mir_constant(tcx, param_env, value)
     };
-    providers.permits_uninit_init =
-        |tcx, ty| util::might_permit_raw_init(tcx, ty, InitKind::UninitMitigated0x01Fill);
-    providers.permits_zero_init = |tcx, ty| util::might_permit_raw_init(tcx, ty, InitKind::Zero);
+    providers.permits_uninit_init = |tcx, param_env_and_ty| {
+        let (param_env, ty) = param_env_and_ty.into_parts();
+        util::might_permit_raw_init(tcx, param_env, ty, InitKind::UninitMitigated0x01Fill)
+    };
+    providers.permits_zero_init = |tcx, param_env_and_ty| {
+        let (param_env, ty) = param_env_and_ty.into_parts();
+        util::might_permit_raw_init(tcx, param_env, ty, InitKind::Zero)
+    };
 }

compiler/rustc_const_eval/src/util/might_permit_raw_init.rs

@@ -20,13 +20,14 @@ use crate::interpret::{InterpCx, MemoryKind, OpTy};
 /// to the full uninit check).
 pub fn might_permit_raw_init<'tcx>(
     tcx: TyCtxt<'tcx>,
+    param_env: ParamEnv<'tcx>,
     ty: TyAndLayout<'tcx>,
     kind: InitKind,
 ) -> bool {
     if tcx.sess.opts.unstable_opts.strict_init_checks {
         might_permit_raw_init_strict(ty, tcx, kind)
     } else {
-        let layout_cx = LayoutCx { tcx, param_env: ParamEnv::reveal_all() };
+        let layout_cx = LayoutCx { tcx, param_env };
         might_permit_raw_init_lax(ty, &layout_cx, kind)
     }
 }

compiler/rustc_middle/src/query/mod.rs

@@ -2116,12 +2116,12 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    query permits_uninit_init(key: TyAndLayout<'tcx>) -> bool {
-        desc { "checking to see if `{}` permits being left uninit", key.ty }
+    query permits_uninit_init(key: ty::ParamEnvAnd<'tcx, TyAndLayout<'tcx>>) -> bool {
+        desc { "checking to see if `{}` permits being left uninit", key.value.ty }
     }
 
-    query permits_zero_init(key: TyAndLayout<'tcx>) -> bool {
-        desc { "checking to see if `{}` permits being left zeroed", key.ty }
+    query permits_zero_init(key: ty::ParamEnvAnd<'tcx, TyAndLayout<'tcx>>) -> bool {
+        desc { "checking to see if `{}` permits being left zeroed", key.value.ty }
     }
 
     query compare_impl_const(

compiler/rustc_middle/src/ty/structural_impls.rs

@@ -11,6 +11,7 @@ use crate::ty::{self, AliasTy, InferConst, Lift, Term, TermKind, Ty, TyCtxt};
 use rustc_data_structures::functor::IdFunctor;
 use rustc_hir::def::Namespace;
 use rustc_index::vec::{Idx, IndexVec};
+use rustc_target::abi::TyAndLayout;
 
 use std::fmt;
 use std::mem::ManuallyDrop;
@@ -853,3 +854,9 @@ impl<'tcx> TypeSuperVisitable<'tcx> for ty::UnevaluatedConst<'tcx> {
         self.substs.visit_with(visitor)
     }
 }
+
+impl<'tcx> TypeVisitable<'tcx> for TyAndLayout<'tcx, Ty<'tcx>> {
+    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        visitor.visit_ty(self.ty)
+    }
+}

compiler/rustc_mir_transform/src/instcombine.rs

@@ -6,7 +6,8 @@ use rustc_middle::mir::{
     BinOp, Body, Constant, ConstantKind, LocalDecls, Operand, Place, ProjectionElem, Rvalue,
     SourceInfo, Statement, StatementKind, Terminator, TerminatorKind, UnOp,
 };
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::{self, layout::TyAndLayout, ParamEnv, ParamEnvAnd, SubstsRef, Ty, TyCtxt};
+use rustc_span::symbol::{sym, Symbol};
 
 pub struct InstCombine;
 
@@ -16,7 +17,11 @@ impl<'tcx> MirPass<'tcx> for InstCombine {
     }
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        let ctx = InstCombineContext { tcx, local_decls: &body.local_decls };
+        let ctx = InstCombineContext {
+            tcx,
+            local_decls: &body.local_decls,
+            param_env: tcx.param_env_reveal_all_normalized(body.source.def_id()),
+        };
         for block in body.basic_blocks.as_mut() {
             for statement in block.statements.iter_mut() {
                 match statement.kind {
@@ -33,6 +38,10 @@ impl<'tcx> MirPass<'tcx> for InstCombine {
                 &mut block.terminator.as_mut().unwrap(),
                 &mut block.statements,
             );
+            ctx.combine_intrinsic_assert(
+                &mut block.terminator.as_mut().unwrap(),
+                &mut block.statements,
+            );
         }
     }
 }
@@ -40,6 +49,7 @@ impl<'tcx> MirPass<'tcx> for InstCombine {
 struct InstCombineContext<'tcx, 'a> {
     tcx: TyCtxt<'tcx>,
     local_decls: &'a LocalDecls<'tcx>,
+    param_env: ParamEnv<'tcx>,
 }
 
 impl<'tcx> InstCombineContext<'tcx, '_> {
@@ -200,4 +210,76 @@ impl<'tcx> InstCombineContext<'tcx, '_> {
         });
         terminator.kind = TerminatorKind::Goto { target: destination_block };
     }
+
+    fn combine_intrinsic_assert(
+        &self,
+        terminator: &mut Terminator<'tcx>,
+        _statements: &mut Vec<Statement<'tcx>>,
+    ) {
+        let TerminatorKind::Call { func, target, .. } = &mut terminator.kind  else { return; };
+        let Some(target_block) = target else { return; };
+        let func_ty = func.ty(self.local_decls, self.tcx);
+        let Some((intrinsic_name, substs)) = resolve_rust_intrinsic(self.tcx, func_ty) else {
+            return;
+        };
+        // The intrinsics we are interested in have one generic parameter
+        if substs.is_empty() {
+            return;
+        }
+        let ty = substs.type_at(0);
+
+        // Check this is a foldable intrinsic before we query the layout of our generic parameter
+        let Some(assert_panics) = intrinsic_assert_panics(intrinsic_name) else { return; };
+        let Ok(layout) = self.tcx.layout_of(self.param_env.and(ty)) else { return; };
+        if assert_panics(self.tcx, self.param_env.and(layout)) {
+            // If we know the assert panics, indicate to later opts that the call diverges
+            *target = None;
+        } else {
+            // If we know the assert does not panic, turn the call into a Goto
+            terminator.kind = TerminatorKind::Goto { target: *target_block };
+        }
+    }
+}
+
+fn intrinsic_assert_panics<'tcx>(
+    intrinsic_name: Symbol,
+) -> Option<fn(TyCtxt<'tcx>, ParamEnvAnd<'tcx, TyAndLayout<'tcx>>) -> bool> {
+    fn inhabited_predicate<'tcx>(
+        _tcx: TyCtxt<'tcx>,
+        param_env_and_layout: ParamEnvAnd<'tcx, TyAndLayout<'tcx>>,
+    ) -> bool {
+        let (_param_env, layout) = param_env_and_layout.into_parts();
+        layout.abi.is_uninhabited()
+    }
+    fn zero_valid_predicate<'tcx>(
+        tcx: TyCtxt<'tcx>,
+        param_env_and_layout: ParamEnvAnd<'tcx, TyAndLayout<'tcx>>,
+    ) -> bool {
+        !tcx.permits_zero_init(param_env_and_layout)
+    }
+    fn mem_uninitialized_valid_predicate<'tcx>(
+        tcx: TyCtxt<'tcx>,
+        param_env_and_layout: ParamEnvAnd<'tcx, TyAndLayout<'tcx>>,
+    ) -> bool {
+        !tcx.permits_uninit_init(param_env_and_layout)
+    }
+
+    match intrinsic_name {
+        sym::assert_inhabited => Some(inhabited_predicate),
+        sym::assert_zero_valid => Some(zero_valid_predicate),
+        sym::assert_mem_uninitialized_valid => Some(mem_uninitialized_valid_predicate),
+        _ => None,
+    }
+}
+
+fn resolve_rust_intrinsic<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    func_ty: Ty<'tcx>,
+) -> Option<(Symbol, SubstsRef<'tcx>)> {
+    if let ty::FnDef(def_id, substs) = *func_ty.kind() {
+        if tcx.is_intrinsic(def_id) {
+            return Some((tcx.item_name(def_id), substs));
+        }
+    }
+    None
 }

tests/mir-opt/intrinsic_asserts.generic.InstCombine.diff

@@ -0,0 +1,42 @@
+- // MIR for `generic` before InstCombine
++ // MIR for `generic` after InstCombine
+  
+  fn generic() -> () {
+      let mut _0: ();                      // return place in scope 0 at $DIR/intrinsic_asserts.rs:+0:21: +0:21
+      let _1: ();                          // in scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:46
+      let _2: ();                          // in scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:47
+      let _3: ();                          // in scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:60
+  
+      bb0: {
+          StorageLive(_1);                 // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:46
+          _1 = assert_inhabited::<T>() -> bb1; // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:46
+                                           // mir::Constant
+                                           // + span: $DIR/intrinsic_asserts.rs:25:5: 25:44
+                                           // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_inhabited::<T>}, val: Value(<ZST>) }
+      }
+  
+      bb1: {
+          StorageDead(_1);                 // scope 0 at $DIR/intrinsic_asserts.rs:+1:46: +1:47
+          StorageLive(_2);                 // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:47
+          _2 = assert_zero_valid::<T>() -> bb2; // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:47
+                                           // mir::Constant
+                                           // + span: $DIR/intrinsic_asserts.rs:26:5: 26:45
+                                           // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_zero_valid::<T>}, val: Value(<ZST>) }
+      }
+  
+      bb2: {
+          StorageDead(_2);                 // scope 0 at $DIR/intrinsic_asserts.rs:+2:47: +2:48
+          StorageLive(_3);                 // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:60
+          _3 = assert_mem_uninitialized_valid::<T>() -> bb3; // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:60
+                                           // mir::Constant
+                                           // + span: $DIR/intrinsic_asserts.rs:27:5: 27:58
+                                           // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_mem_uninitialized_valid::<T>}, val: Value(<ZST>) }
+      }
+  
+      bb3: {
+          StorageDead(_3);                 // scope 0 at $DIR/intrinsic_asserts.rs:+3:60: +3:61
+          nop;                             // scope 0 at $DIR/intrinsic_asserts.rs:+0:21: +4:2
+          return;                          // scope 0 at $DIR/intrinsic_asserts.rs:+4:2: +4:2
+      }
+  }
+  

tests/mir-opt/intrinsic_asserts.panics.InstCombine.diff

@@ -0,0 +1,47 @@
+- // MIR for `panics` before InstCombine
++ // MIR for `panics` after InstCombine
+  
+  fn panics() -> () {
+      let mut _0: ();                      // return place in scope 0 at $DIR/intrinsic_asserts.rs:+0:17: +0:17
+      let _1: ();                          // in scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:50
+      let _2: ();                          // in scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:49
+      let _3: ();                          // in scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:62
+  
+      bb0: {
+          StorageLive(_1);                 // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:50
+-         _1 = assert_inhabited::<Never>() -> bb1; // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:50
++         _1 = assert_inhabited::<Never>(); // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:50
+                                           // mir::Constant
+                                           // + span: $DIR/intrinsic_asserts.rs:17:5: 17:48
+                                           // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_inhabited::<Never>}, val: Value(<ZST>) }
+      }
+  
+      bb1: {
+          StorageDead(_1);                 // scope 0 at $DIR/intrinsic_asserts.rs:+1:50: +1:51
+          StorageLive(_2);                 // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:49
+-         _2 = assert_zero_valid::<&u8>() -> bb2; // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:49
++         _2 = assert_zero_valid::<&u8>(); // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:49
+                                           // mir::Constant
+                                           // + span: $DIR/intrinsic_asserts.rs:18:5: 18:47
+                                           // + user_ty: UserType(0)
+                                           // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_zero_valid::<&u8>}, val: Value(<ZST>) }
+      }
+  
+      bb2: {
+          StorageDead(_2);                 // scope 0 at $DIR/intrinsic_asserts.rs:+2:49: +2:50
+          StorageLive(_3);                 // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:62
+-         _3 = assert_mem_uninitialized_valid::<&u8>() -> bb3; // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:62
++         _3 = assert_mem_uninitialized_valid::<&u8>(); // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:62
+                                           // mir::Constant
+                                           // + span: $DIR/intrinsic_asserts.rs:19:5: 19:60
+                                           // + user_ty: UserType(1)
+                                           // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_mem_uninitialized_valid::<&u8>}, val: Value(<ZST>) }
+      }
+  
+      bb3: {
+          StorageDead(_3);                 // scope 0 at $DIR/intrinsic_asserts.rs:+3:62: +3:63
+          nop;                             // scope 0 at $DIR/intrinsic_asserts.rs:+0:17: +4:2
+          return;                          // scope 0 at $DIR/intrinsic_asserts.rs:+4:2: +4:2
+      }
+  }
+  

tests/mir-opt/intrinsic_asserts.removable.InstCombine.diff

@@ -0,0 +1,45 @@
+- // MIR for `removable` before InstCombine
++ // MIR for `removable` after InstCombine
+  
+  fn removable() -> () {
+      let mut _0: ();                      // return place in scope 0 at $DIR/intrinsic_asserts.rs:+0:20: +0:20
+      let _1: ();                          // in scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:47
+      let _2: ();                          // in scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:48
+      let _3: ();                          // in scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:61
+  
+      bb0: {
+          StorageLive(_1);                 // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:47
+-         _1 = assert_inhabited::<()>() -> bb1; // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:47
+-                                          // mir::Constant
+-                                          // + span: $DIR/intrinsic_asserts.rs:7:5: 7:45
+-                                          // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_inhabited::<()>}, val: Value(<ZST>) }
++         goto -> bb1;                     // scope 0 at $DIR/intrinsic_asserts.rs:+1:5: +1:47
+      }
+  
+      bb1: {
+          StorageDead(_1);                 // scope 0 at $DIR/intrinsic_asserts.rs:+1:47: +1:48
+          StorageLive(_2);                 // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:48
+-         _2 = assert_zero_valid::<u8>() -> bb2; // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:48
+-                                          // mir::Constant
+-                                          // + span: $DIR/intrinsic_asserts.rs:8:5: 8:46
+-                                          // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_zero_valid::<u8>}, val: Value(<ZST>) }
++         goto -> bb2;                     // scope 0 at $DIR/intrinsic_asserts.rs:+2:5: +2:48
+      }
+  
+      bb2: {
+          StorageDead(_2);                 // scope 0 at $DIR/intrinsic_asserts.rs:+2:48: +2:49
+          StorageLive(_3);                 // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:61
+-         _3 = assert_mem_uninitialized_valid::<u8>() -> bb3; // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:61
+-                                          // mir::Constant
+-                                          // + span: $DIR/intrinsic_asserts.rs:9:5: 9:59
+-                                          // + literal: Const { ty: extern "rust-intrinsic" fn() {assert_mem_uninitialized_valid::<u8>}, val: Value(<ZST>) }
++         goto -> bb3;                     // scope 0 at $DIR/intrinsic_asserts.rs:+3:5: +3:61
+      }
+  
+      bb3: {
+          StorageDead(_3);                 // scope 0 at $DIR/intrinsic_asserts.rs:+3:61: +3:62
+          nop;                             // scope 0 at $DIR/intrinsic_asserts.rs:+0:20: +4:2
+          return;                          // scope 0 at $DIR/intrinsic_asserts.rs:+4:2: +4:2
+      }
+  }
+  

tests/mir-opt/intrinsic_asserts.rs

@@ -0,0 +1,28 @@
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+// All these assertions pass, so all the intrinsic calls should be deleted.
+// EMIT_MIR intrinsic_asserts.removable.InstCombine.diff
+pub fn removable() {
+    core::intrinsics::assert_inhabited::<()>();
+    core::intrinsics::assert_zero_valid::<u8>();
+    core::intrinsics::assert_mem_uninitialized_valid::<u8>();
+}
+
+enum Never {}
+
+// These assertions all diverge, so their target blocks should become None.
+// EMIT_MIR intrinsic_asserts.panics.InstCombine.diff
+pub fn panics() {
+    core::intrinsics::assert_inhabited::<Never>();
+    core::intrinsics::assert_zero_valid::<&u8>();
+    core::intrinsics::assert_mem_uninitialized_valid::<&u8>();
+}
+
+// Whether or not these asserts pass isn't known, so they shouldn't be modified.
+// EMIT_MIR intrinsic_asserts.generic.InstCombine.diff
+pub fn generic<T>() {
+    core::intrinsics::assert_inhabited::<T>();
+    core::intrinsics::assert_zero_valid::<T>();
+    core::intrinsics::assert_mem_uninitialized_valid::<T>();
+}