From 86d8644c1bfa4b8feddfa588e8e599e420acd040 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Fri, 26 Jun 2020 13:20:41 +0100 Subject: [PATCH] Optimise fast path of checked_ops with `unlikely` --- src/libcore/num/mod.rs | 70 ++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/src/libcore/num/mod.rs b/src/libcore/num/mod.rs index d36da90f2ad..60c496ba2d3 100644 --- a/src/libcore/num/mod.rs +++ b/src/libcore/num/mod.rs @@ -21,6 +21,21 @@ macro_rules! try_opt { }; } +#[cfg(bootstrap)] +macro_rules! unlikely { + ($e: expr) => { + $e + }; +} + +#[cfg(not(bootstrap))] +#[allow_internal_unstable(const_likely)] +macro_rules! unlikely { + ($e: expr) => { + intrinsics::unlikely($e) + }; +} + macro_rules! impl_nonzero_fmt { ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => { $( @@ -745,7 +760,7 @@ $EndFeature, " #[inline] pub const fn checked_add(self, rhs: Self) -> Option { let (a, b) = self.overflowing_add(rhs); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -787,7 +802,7 @@ $EndFeature, " #[inline] pub const fn checked_sub(self, rhs: Self) -> Option { let (a, b) = self.overflowing_sub(rhs); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -829,7 +844,7 @@ $EndFeature, " #[inline] pub const fn checked_mul(self, rhs: Self) -> Option { let (a, b) = self.overflowing_mul(rhs); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -871,7 +886,7 @@ $EndFeature, " without modifying the original"] #[inline] pub const fn checked_div(self, rhs: Self) -> Option { - if rhs == 0 || (self == Self::MIN && rhs == -1) { + if unlikely!(rhs == 0 || (self == Self::MIN && rhs == -1)) { None } else { // SAFETY: div by zero and by INT_MIN have been checked above @@ -900,7 +915,7 @@ assert_eq!((1", stringify!($SelfT), ").checked_div_euclid(0), None); without modifying the original"] #[inline] pub const fn checked_div_euclid(self, rhs: Self) -> Option { - if rhs == 0 || (self == Self::MIN && rhs == -1) { + if unlikely!(rhs == 0 || (self == Self::MIN && rhs == -1)) { None } else { Some(self.div_euclid(rhs)) @@ -929,7 +944,7 @@ $EndFeature, " without modifying the original"] #[inline] pub const fn checked_rem(self, rhs: Self) -> Option { - if rhs == 0 || (self == Self::MIN && rhs == -1) { + if unlikely!(rhs == 0 || (self == Self::MIN && rhs == -1)) { None } else { // SAFETY: div by zero and by INT_MIN have been checked above @@ -957,7 +972,7 @@ assert_eq!(", stringify!($SelfT), "::MIN.checked_rem_euclid(-1), None); without modifying the original"] #[inline] pub const fn checked_rem_euclid(self, rhs: Self) -> Option { - if rhs == 0 || (self == Self::MIN && rhs == -1) { + if unlikely!(rhs == 0 || (self == Self::MIN && rhs == -1)) { None } else { Some(self.rem_euclid(rhs)) @@ -983,7 +998,7 @@ $EndFeature, " #[inline] pub const fn checked_neg(self) -> Option { let (a, b) = self.overflowing_neg(); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -1007,7 +1022,7 @@ $EndFeature, " #[inline] pub const fn checked_shl(self, rhs: u32) -> Option { let (a, b) = self.overflowing_shl(rhs); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -1031,7 +1046,7 @@ $EndFeature, " #[inline] pub const fn checked_shr(self, rhs: u32) -> Option { let (a, b) = self.overflowing_shr(rhs); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -1738,7 +1753,7 @@ $EndFeature, " #[must_use = "this returns the result of the operation, \ without modifying the original"] pub const fn overflowing_div(self, rhs: Self) -> (Self, bool) { - if self == Self::MIN && rhs == -1 { + if unlikely!(self == Self::MIN && rhs == -1) { (self, true) } else { (self / rhs, false) @@ -1771,7 +1786,7 @@ assert_eq!(", stringify!($SelfT), "::MIN.overflowing_div_euclid(-1), (", stringi #[must_use = "this returns the result of the operation, \ without modifying the original"] pub const fn overflowing_div_euclid(self, rhs: Self) -> (Self, bool) { - if self == Self::MIN && rhs == -1 { + if unlikely!(self == Self::MIN && rhs == -1) { (self, true) } else { (self.div_euclid(rhs), false) @@ -1805,7 +1820,7 @@ $EndFeature, " #[must_use = "this returns the result of the operation, \ without modifying the original"] pub const fn overflowing_rem(self, rhs: Self) -> (Self, bool) { - if self == Self::MIN && rhs == -1 { + if unlikely!(self == Self::MIN && rhs == -1) { (0, true) } else { (self % rhs, false) @@ -1838,7 +1853,7 @@ assert_eq!(", stringify!($SelfT), "::MIN.overflowing_rem_euclid(-1), (0, true)); without modifying the original"] #[inline] pub const fn overflowing_rem_euclid(self, rhs: Self) -> (Self, bool) { - if self == Self::MIN && rhs == -1 { + if unlikely!(self == Self::MIN && rhs == -1) { (0, true) } else { (self.rem_euclid(rhs), false) @@ -1869,7 +1884,7 @@ assert_eq!(", stringify!($SelfT), "::MIN.overflowing_neg(), (", stringify!($Self #[allow(unused_attributes)] #[cfg_attr(bootstrap, allow_internal_unstable(const_if_match))] pub const fn overflowing_neg(self) -> (Self, bool) { - if self == Self::MIN { + if unlikely!(self == Self::MIN) { (Self::MIN, true) } else { (-self, false) @@ -2981,7 +2996,7 @@ assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add(3), None);", $EndFeat #[inline] pub const fn checked_add(self, rhs: Self) -> Option { let (a, b) = self.overflowing_add(rhs); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -3021,7 +3036,7 @@ assert_eq!(0", stringify!($SelfT), ".checked_sub(1), None);", $EndFeature, " #[inline] pub const fn checked_sub(self, rhs: Self) -> Option { let (a, b) = self.overflowing_sub(rhs); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -3061,7 +3076,7 @@ assert_eq!(", stringify!($SelfT), "::MAX.checked_mul(2), None);", $EndFeature, " #[inline] pub const fn checked_mul(self, rhs: Self) -> Option { let (a, b) = self.overflowing_mul(rhs); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -3100,11 +3115,12 @@ assert_eq!(1", stringify!($SelfT), ".checked_div(0), None);", $EndFeature, " without modifying the original"] #[inline] pub const fn checked_div(self, rhs: Self) -> Option { - match rhs { - 0 => None, + if unlikely!(rhs == 0) { + None + } else { // SAFETY: div by zero has been checked above and unsigned types have no other // failure modes for division - rhs => Some(unsafe { intrinsics::unchecked_div(self, rhs) }), + Some(unsafe { intrinsics::unchecked_div(self, rhs) }) } } } @@ -3127,7 +3143,7 @@ assert_eq!(1", stringify!($SelfT), ".checked_div_euclid(0), None); without modifying the original"] #[inline] pub const fn checked_div_euclid(self, rhs: Self) -> Option { - if rhs == 0 { + if unlikely!(rhs == 0) { None } else { Some(self.div_euclid(rhs)) @@ -3154,7 +3170,7 @@ assert_eq!(5", stringify!($SelfT), ".checked_rem(0), None);", $EndFeature, " without modifying the original"] #[inline] pub const fn checked_rem(self, rhs: Self) -> Option { - if rhs == 0 { + if unlikely!(rhs == 0) { None } else { // SAFETY: div by zero has been checked above and unsigned types have no other @@ -3182,7 +3198,7 @@ assert_eq!(5", stringify!($SelfT), ".checked_rem_euclid(0), None); without modifying the original"] #[inline] pub const fn checked_rem_euclid(self, rhs: Self) -> Option { - if rhs == 0 { + if unlikely!(rhs == 0) { None } else { Some(self.rem_euclid(rhs)) @@ -3209,7 +3225,7 @@ assert_eq!(1", stringify!($SelfT), ".checked_neg(), None);", $EndFeature, " #[inline] pub const fn checked_neg(self) -> Option { let (a, b) = self.overflowing_neg(); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -3232,7 +3248,7 @@ assert_eq!(0x10", stringify!($SelfT), ".checked_shl(129), None);", $EndFeature, #[inline] pub const fn checked_shl(self, rhs: u32) -> Option { let (a, b) = self.overflowing_shl(rhs); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } } @@ -3255,7 +3271,7 @@ assert_eq!(0x10", stringify!($SelfT), ".checked_shr(129), None);", $EndFeature, #[inline] pub const fn checked_shr(self, rhs: u32) -> Option { let (a, b) = self.overflowing_shr(rhs); - if b {None} else {Some(a)} + if unlikely!(b) {None} else {Some(a)} } }