From f2871a9ce552cd21b1c7b8df69eefe06af6d59da Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 6 Feb 2019 11:57:11 +1100 Subject: [PATCH] Make `intern_lazy_const` actually intern its argument. Currently it just unconditionally allocates it in the arena. For a "Clean Check" build of the the `packed-simd` benchmark, this change reduces both the `max-rss` and `faults` counts by 59%; it slightly (~3%) increases the instruction counts but the `wall-time` is unchanged. For the same builds of a few other benchmarks, `max-rss` and `faults` drop by 1--5%, but instruction counts and `wall-time` changes are in the noise. Fixes #57432, fixes #57829. --- src/librustc/mir/mod.rs | 2 +- src/librustc/traits/project.rs | 4 ++-- src/librustc/traits/query/normalize.rs | 4 ++-- src/librustc/ty/codec.rs | 2 +- src/librustc/ty/context.rs | 22 ++++++++++--------- src/librustc/ty/structural_impls.rs | 2 +- src/librustc_mir/build/expr/as_rvalue.rs | 2 +- src/librustc_mir/build/matches/test.rs | 2 +- src/librustc_mir/build/misc.rs | 2 +- src/librustc_mir/hair/cx/expr.rs | 12 +++++----- src/librustc_mir/hair/cx/mod.rs | 6 ++--- src/librustc_mir/shim.rs | 6 ++--- src/librustc_mir/transform/elaborate_drops.rs | 2 +- src/librustc_mir/transform/generator.rs | 4 ++-- src/librustc_mir/util/elaborate_drops.rs | 2 +- src/librustc_typeck/astconv.rs | 2 +- src/librustc_typeck/check/mod.rs | 2 +- 17 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/librustc/mir/mod.rs b/src/librustc/mir/mod.rs index 82083b4f699..eca01d78c2a 100644 --- a/src/librustc/mir/mod.rs +++ b/src/librustc/mir/mod.rs @@ -2154,7 +2154,7 @@ impl<'tcx> Operand<'tcx> { span, ty, user_ty: None, - literal: tcx.intern_lazy_const( + literal: tcx.mk_lazy_const( ty::LazyConst::Evaluated(ty::Const::zero_sized(ty)), ), }) diff --git a/src/librustc/traits/project.rs b/src/librustc/traits/project.rs index bec45046cb9..21151276e72 100644 --- a/src/librustc/traits/project.rs +++ b/src/librustc/traits/project.rs @@ -408,7 +408,7 @@ impl<'a, 'b, 'gcx, 'tcx> TypeFolder<'gcx, 'tcx> for AssociatedTypeNormalizer<'a, if let Ok(evaluated) = tcx.const_eval(param_env.and(cid)) { let substs = tcx.lift_to_global(&substs).unwrap(); let evaluated = evaluated.subst(tcx, substs); - return tcx.intern_lazy_const(ty::LazyConst::Evaluated(evaluated)); + return tcx.mk_lazy_const(ty::LazyConst::Evaluated(evaluated)); } } } else { @@ -420,7 +420,7 @@ impl<'a, 'b, 'gcx, 'tcx> TypeFolder<'gcx, 'tcx> for AssociatedTypeNormalizer<'a, promoted: None }; if let Ok(evaluated) = tcx.const_eval(param_env.and(cid)) { - return tcx.intern_lazy_const(ty::LazyConst::Evaluated(evaluated)); + return tcx.mk_lazy_const(ty::LazyConst::Evaluated(evaluated)); } } } diff --git a/src/librustc/traits/query/normalize.rs b/src/librustc/traits/query/normalize.rs index be05445cfc6..9f55bb95d25 100644 --- a/src/librustc/traits/query/normalize.rs +++ b/src/librustc/traits/query/normalize.rs @@ -203,7 +203,7 @@ impl<'cx, 'gcx, 'tcx> TypeFolder<'gcx, 'tcx> for QueryNormalizer<'cx, 'gcx, 'tcx if let Ok(evaluated) = tcx.const_eval(param_env.and(cid)) { let substs = tcx.lift_to_global(&substs).unwrap(); let evaluated = evaluated.subst(tcx, substs); - return tcx.intern_lazy_const(ty::LazyConst::Evaluated(evaluated)); + return tcx.mk_lazy_const(ty::LazyConst::Evaluated(evaluated)); } } } else { @@ -215,7 +215,7 @@ impl<'cx, 'gcx, 'tcx> TypeFolder<'gcx, 'tcx> for QueryNormalizer<'cx, 'gcx, 'tcx promoted: None, }; if let Ok(evaluated) = tcx.const_eval(param_env.and(cid)) { - return tcx.intern_lazy_const(ty::LazyConst::Evaluated(evaluated)); + return tcx.mk_lazy_const(ty::LazyConst::Evaluated(evaluated)); } } } diff --git a/src/librustc/ty/codec.rs b/src/librustc/ty/codec.rs index e0e4d9c362a..a4a24718527 100644 --- a/src/librustc/ty/codec.rs +++ b/src/librustc/ty/codec.rs @@ -252,7 +252,7 @@ pub fn decode_lazy_const<'a, 'tcx, D>(decoder: &mut D) where D: TyDecoder<'a, 'tcx>, 'tcx: 'a, { - Ok(decoder.tcx().intern_lazy_const(Decodable::decode(decoder)?)) + Ok(decoder.tcx().mk_lazy_const(Decodable::decode(decoder)?)) } #[inline] diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index b379b5ba024..87cf52af1d1 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -127,6 +127,7 @@ pub struct CtxtInterners<'tcx> { goal: InternedSet<'tcx, GoalKind<'tcx>>, goal_list: InternedSet<'tcx, List>>, projs: InternedSet<'tcx, List>>, + lazy_const: InternedSet<'tcx, LazyConst<'tcx>>, } impl<'gcx: 'tcx, 'tcx> CtxtInterners<'tcx> { @@ -144,6 +145,7 @@ impl<'gcx: 'tcx, 'tcx> CtxtInterners<'tcx> { goal: Default::default(), goal_list: Default::default(), projs: Default::default(), + lazy_const: Default::default(), } } @@ -1096,10 +1098,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { self.global_arenas.adt_def.alloc(def) } - pub fn intern_const_alloc( - self, - alloc: Allocation, - ) -> &'gcx Allocation { + pub fn intern_const_alloc(self, alloc: Allocation) -> &'gcx Allocation { self.allocation_interner.borrow_mut().intern(alloc, |alloc| { self.global_arenas.const_allocs.alloc(alloc) }) @@ -1119,10 +1118,6 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { }) } - pub fn intern_lazy_const(self, c: ty::LazyConst<'tcx>) -> &'tcx ty::LazyConst<'tcx> { - self.global_interners.arena.alloc(c) - } - pub fn intern_layout(self, layout: LayoutDetails) -> &'gcx LayoutDetails { self.layout_interner.borrow_mut().intern(layout, |layout| { self.global_arenas.layout.alloc(layout) @@ -2271,6 +2266,12 @@ impl<'tcx: 'lcx, 'lcx> Borrow> for Interned<'tcx, GoalKind<'tcx>> } } +impl<'tcx: 'lcx, 'lcx> Borrow> for Interned<'tcx, LazyConst<'tcx>> { + fn borrow<'a>(&'a self) -> &'a LazyConst<'lcx> { + &self.0 + } +} + impl<'tcx: 'lcx, 'lcx> Borrow<[ExistentialPredicate<'lcx>]> for Interned<'tcx, List>> { fn borrow<'a>(&'a self) -> &'a [ExistentialPredicate<'lcx>] { @@ -2377,7 +2378,8 @@ pub fn keep_local<'tcx, T: ty::TypeFoldable<'tcx>>(x: &T) -> bool { direct_interners!('tcx, region: mk_region(|r: &RegionKind| r.keep_in_local_tcx()) -> RegionKind, - goal: mk_goal(|c: &GoalKind<'_>| keep_local(c)) -> GoalKind<'tcx> + goal: mk_goal(|c: &GoalKind<'_>| keep_local(c)) -> GoalKind<'tcx>, + lazy_const: mk_lazy_const(|c: &LazyConst<'_>| keep_local(&c)) -> LazyConst<'tcx> ); macro_rules! slice_interners { @@ -2562,7 +2564,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { #[inline] pub fn mk_array(self, ty: Ty<'tcx>, n: u64) -> Ty<'tcx> { - self.mk_ty(Array(ty, self.intern_lazy_const( + self.mk_ty(Array(ty, self.mk_lazy_const( ty::LazyConst::Evaluated(ty::Const::from_usize(self.global_tcx(), n)) ))) } diff --git a/src/librustc/ty/structural_impls.rs b/src/librustc/ty/structural_impls.rs index 28f5a65374d..f89a6ed2e76 100644 --- a/src/librustc/ty/structural_impls.rs +++ b/src/librustc/ty/structural_impls.rs @@ -1042,7 +1042,7 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::LazyConst<'tcx> { ty::LazyConst::Unevaluated(*def_id, substs.fold_with(folder)) } }; - folder.tcx().intern_lazy_const(new) + folder.tcx().mk_lazy_const(new) } fn fold_with<'gcx: 'tcx, F: TypeFolder<'gcx, 'tcx>>(&self, folder: &mut F) -> Self { diff --git a/src/librustc_mir/build/expr/as_rvalue.rs b/src/librustc_mir/build/expr/as_rvalue.rs index 3de2f475786..508f6555f4a 100644 --- a/src/librustc_mir/build/expr/as_rvalue.rs +++ b/src/librustc_mir/build/expr/as_rvalue.rs @@ -268,7 +268,7 @@ impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> { span: expr_span, ty: this.hir.tcx().types.u32, user_ty: None, - literal: this.hir.tcx().intern_lazy_const(ty::LazyConst::Evaluated( + literal: this.hir.tcx().mk_lazy_const(ty::LazyConst::Evaluated( ty::Const::from_bits( this.hir.tcx(), 0, diff --git a/src/librustc_mir/build/matches/test.rs b/src/librustc_mir/build/matches/test.rs index 696c173b048..6e836f7059b 100644 --- a/src/librustc_mir/build/matches/test.rs +++ b/src/librustc_mir/build/matches/test.rs @@ -302,7 +302,7 @@ impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> { } let eq_def_id = self.hir.tcx().lang_items().eq_trait().unwrap(); let (mty, method) = self.hir.trait_method(eq_def_id, "eq", ty, &[ty.into()]); - let method = self.hir.tcx().intern_lazy_const(ty::LazyConst::Evaluated(method)); + let method = self.hir.tcx().mk_lazy_const(ty::LazyConst::Evaluated(method)); let re_erased = self.hir.tcx().types.re_erased; // take the argument by reference diff --git a/src/librustc_mir/build/misc.rs b/src/librustc_mir/build/misc.rs index c849c022428..a7b201fc0db 100644 --- a/src/librustc_mir/build/misc.rs +++ b/src/librustc_mir/build/misc.rs @@ -33,7 +33,7 @@ impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> { span, ty, user_ty: None, - literal: self.hir.tcx().intern_lazy_const(ty::LazyConst::Evaluated(literal)), + literal: self.hir.tcx().mk_lazy_const(ty::LazyConst::Evaluated(literal)), }; Operand::Constant(constant) } diff --git a/src/librustc_mir/hair/cx/expr.rs b/src/librustc_mir/hair/cx/expr.rs index 8d64c9e9ada..88512fede33 100644 --- a/src/librustc_mir/hair/cx/expr.rs +++ b/src/librustc_mir/hair/cx/expr.rs @@ -342,7 +342,7 @@ fn make_mirror_unadjusted<'a, 'gcx, 'tcx>(cx: &mut Cx<'a, 'gcx, 'tcx>, } hir::ExprKind::Lit(ref lit) => ExprKind::Literal { - literal: cx.tcx.intern_lazy_const(ty::LazyConst::Evaluated( + literal: cx.tcx.mk_lazy_const(ty::LazyConst::Evaluated( cx.const_eval_literal(&lit.node, expr_ty, lit.span, false) )), user_ty: None, @@ -442,7 +442,7 @@ fn make_mirror_unadjusted<'a, 'gcx, 'tcx>(cx: &mut Cx<'a, 'gcx, 'tcx>, } else { if let hir::ExprKind::Lit(ref lit) = arg.node { ExprKind::Literal { - literal: cx.tcx.intern_lazy_const(ty::LazyConst::Evaluated( + literal: cx.tcx.mk_lazy_const(ty::LazyConst::Evaluated( cx.const_eval_literal(&lit.node, expr_ty, lit.span, true) )), user_ty: None, @@ -702,7 +702,7 @@ fn make_mirror_unadjusted<'a, 'gcx, 'tcx>(cx: &mut Cx<'a, 'gcx, 'tcx>, ty: var_ty, span: expr.span, kind: ExprKind::Literal { - literal: cx.tcx.intern_lazy_const(literal), + literal: cx.tcx.mk_lazy_const(literal), user_ty: None }, }.to_ref(); @@ -856,7 +856,7 @@ fn method_callee<'a, 'gcx, 'tcx>( ty, span, kind: ExprKind::Literal { - literal: cx.tcx().intern_lazy_const(ty::LazyConst::Evaluated( + literal: cx.tcx().mk_lazy_const(ty::LazyConst::Evaluated( ty::Const::zero_sized(ty) )), user_ty, @@ -918,7 +918,7 @@ fn convert_path_expr<'a, 'gcx, 'tcx>(cx: &mut Cx<'a, 'gcx, 'tcx>, let user_ty = user_substs_applied_to_def(cx, expr.hir_id, &def); debug!("convert_path_expr: user_ty={:?}", user_ty); ExprKind::Literal { - literal: cx.tcx.intern_lazy_const(ty::LazyConst::Evaluated(ty::Const::zero_sized( + literal: cx.tcx.mk_lazy_const(ty::LazyConst::Evaluated(ty::Const::zero_sized( cx.tables().node_id_to_type(expr.hir_id), ))), user_ty, @@ -930,7 +930,7 @@ fn convert_path_expr<'a, 'gcx, 'tcx>(cx: &mut Cx<'a, 'gcx, 'tcx>, let user_ty = user_substs_applied_to_def(cx, expr.hir_id, &def); debug!("convert_path_expr: (const) user_ty={:?}", user_ty); ExprKind::Literal { - literal: cx.tcx.intern_lazy_const(ty::LazyConst::Unevaluated(def_id, substs)), + literal: cx.tcx.mk_lazy_const(ty::LazyConst::Unevaluated(def_id, substs)), user_ty, } }, diff --git a/src/librustc_mir/hair/cx/mod.rs b/src/librustc_mir/hair/cx/mod.rs index f514cac6326..a726b87afa8 100644 --- a/src/librustc_mir/hair/cx/mod.rs +++ b/src/librustc_mir/hair/cx/mod.rs @@ -110,7 +110,7 @@ impl<'a, 'gcx, 'tcx> Cx<'a, 'gcx, 'tcx> { } pub fn usize_literal(&mut self, value: u64) -> &'tcx ty::LazyConst<'tcx> { - self.tcx.intern_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_usize(self.tcx, value))) + self.tcx.mk_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_usize(self.tcx, value))) } pub fn bool_ty(&mut self) -> Ty<'tcx> { @@ -122,11 +122,11 @@ impl<'a, 'gcx, 'tcx> Cx<'a, 'gcx, 'tcx> { } pub fn true_literal(&mut self) -> &'tcx ty::LazyConst<'tcx> { - self.tcx.intern_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bool(self.tcx, true))) + self.tcx.mk_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bool(self.tcx, true))) } pub fn false_literal(&mut self) -> &'tcx ty::LazyConst<'tcx> { - self.tcx.intern_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bool(self.tcx, false))) + self.tcx.mk_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bool(self.tcx, false))) } pub fn const_eval_literal( diff --git a/src/librustc_mir/shim.rs b/src/librustc_mir/shim.rs index 751815eab28..15796186063 100644 --- a/src/librustc_mir/shim.rs +++ b/src/librustc_mir/shim.rs @@ -459,7 +459,7 @@ impl<'a, 'tcx> CloneShimBuilder<'a, 'tcx> { span: self.span, ty: func_ty, user_ty: None, - literal: tcx.intern_lazy_const(ty::LazyConst::Evaluated( + literal: tcx.mk_lazy_const(ty::LazyConst::Evaluated( ty::Const::zero_sized(func_ty), )), }); @@ -521,7 +521,7 @@ impl<'a, 'tcx> CloneShimBuilder<'a, 'tcx> { span: self.span, ty: self.tcx.types.usize, user_ty: None, - literal: self.tcx.intern_lazy_const(ty::LazyConst::Evaluated( + literal: self.tcx.mk_lazy_const(ty::LazyConst::Evaluated( ty::Const::from_usize(self.tcx, value), )), } @@ -759,7 +759,7 @@ fn build_call_shim<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, span, ty, user_ty: None, - literal: tcx.intern_lazy_const(ty::LazyConst::Evaluated( + literal: tcx.mk_lazy_const(ty::LazyConst::Evaluated( ty::Const::zero_sized(ty) )), }), diff --git a/src/librustc_mir/transform/elaborate_drops.rs b/src/librustc_mir/transform/elaborate_drops.rs index 06e16de8b43..9b75a70ff41 100644 --- a/src/librustc_mir/transform/elaborate_drops.rs +++ b/src/librustc_mir/transform/elaborate_drops.rs @@ -533,7 +533,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { span, ty: self.tcx.types.bool, user_ty: None, - literal: self.tcx.intern_lazy_const(ty::LazyConst::Evaluated( + literal: self.tcx.mk_lazy_const(ty::LazyConst::Evaluated( ty::Const::from_bool(self.tcx, val), )), }))) diff --git a/src/librustc_mir/transform/generator.rs b/src/librustc_mir/transform/generator.rs index f5cc6a43e28..2a1188ed870 100644 --- a/src/librustc_mir/transform/generator.rs +++ b/src/librustc_mir/transform/generator.rs @@ -198,7 +198,7 @@ impl<'a, 'tcx> TransformVisitor<'a, 'tcx> { span: source_info.span, ty: self.tcx.types.u32, user_ty: None, - literal: self.tcx.intern_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bits( + literal: self.tcx.mk_lazy_const(ty::LazyConst::Evaluated(ty::Const::from_bits( self.tcx, state_disc.into(), ty::ParamEnv::empty().and(self.tcx.types.u32) @@ -731,7 +731,7 @@ fn insert_panic_block<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, span: mir.span, ty: tcx.types.bool, user_ty: None, - literal: tcx.intern_lazy_const(ty::LazyConst::Evaluated( + literal: tcx.mk_lazy_const(ty::LazyConst::Evaluated( ty::Const::from_bool(tcx, false), )), }), diff --git a/src/librustc_mir/util/elaborate_drops.rs b/src/librustc_mir/util/elaborate_drops.rs index 8b55a4424ae..eedc42927c1 100644 --- a/src/librustc_mir/util/elaborate_drops.rs +++ b/src/librustc_mir/util/elaborate_drops.rs @@ -963,7 +963,7 @@ impl<'l, 'b, 'tcx, D> DropCtxt<'l, 'b, 'tcx, D> span: self.source_info.span, ty: self.tcx().types.usize, user_ty: None, - literal: self.tcx().intern_lazy_const(ty::LazyConst::Evaluated( + literal: self.tcx().mk_lazy_const(ty::LazyConst::Evaluated( ty::Const::from_usize(self.tcx(), val.into()) )), }) diff --git a/src/librustc_typeck/astconv.rs b/src/librustc_typeck/astconv.rs index 8da0b6dcbea..757385aeb3e 100644 --- a/src/librustc_typeck/astconv.rs +++ b/src/librustc_typeck/astconv.rs @@ -1793,7 +1793,7 @@ impl<'o, 'gcx: 'tcx, 'tcx> dyn AstConv<'gcx, 'tcx> + 'o { let length_def_id = tcx.hir().local_def_id(length.id); let substs = Substs::identity_for_item(tcx, length_def_id); let length = ty::LazyConst::Unevaluated(length_def_id, substs); - let length = tcx.intern_lazy_const(length); + let length = tcx.mk_lazy_const(length); let array_ty = tcx.mk_ty(ty::Array(self.ast_ty_to_ty(&ty), length)); self.normalize_ty(ast_ty.span, array_ty) } diff --git a/src/librustc_typeck/check/mod.rs b/src/librustc_typeck/check/mod.rs index 3e2a9d720f1..fb8f6088121 100644 --- a/src/librustc_typeck/check/mod.rs +++ b/src/librustc_typeck/check/mod.rs @@ -4597,7 +4597,7 @@ impl<'a, 'gcx, 'tcx> FnCtxt<'a, 'gcx, 'tcx> { if element_ty.references_error() { tcx.types.err } else if let Ok(count) = count { - tcx.mk_ty(ty::Array(t, tcx.intern_lazy_const(ty::LazyConst::Evaluated(count)))) + tcx.mk_ty(ty::Array(t, tcx.mk_lazy_const(ty::LazyConst::Evaluated(count)))) } else { tcx.types.err }