Rollup merge of #73938 - nbdd0121:checked_opt, r=nagisa
Optimise fast path of checked_ops with `unlikely` This PR marks paths returning `None` in checked_ops as unlikely to improvde codegen. Fixes #73731
This commit is contained in:
commit
4f536f2c36
1 changed files with 43 additions and 27 deletions
|
@ -21,6 +21,21 @@ macro_rules! try_opt {
|
|||
};
|
||||
}
|
||||
|
||||
#[cfg(bootstrap)]
|
||||
macro_rules! unlikely {
|
||||
($e: expr) => {
|
||||
$e
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(not(bootstrap))]
|
||||
#[allow_internal_unstable(const_likely)]
|
||||
macro_rules! unlikely {
|
||||
($e: expr) => {
|
||||
intrinsics::unlikely($e)
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! impl_nonzero_fmt {
|
||||
( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => {
|
||||
$(
|
||||
|
@ -746,7 +761,7 @@ $EndFeature, "
|
|||
#[inline]
|
||||
pub const fn checked_add(self, rhs: Self) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_add(rhs);
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -790,7 +805,7 @@ $EndFeature, "
|
|||
#[inline]
|
||||
pub const fn checked_sub(self, rhs: Self) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_sub(rhs);
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -834,7 +849,7 @@ $EndFeature, "
|
|||
#[inline]
|
||||
pub const fn checked_mul(self, rhs: Self) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_mul(rhs);
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -878,7 +893,7 @@ $EndFeature, "
|
|||
without modifying the original"]
|
||||
#[inline]
|
||||
pub const fn checked_div(self, rhs: Self) -> Option<Self> {
|
||||
if rhs == 0 || (self == Self::MIN && rhs == -1) {
|
||||
if unlikely!(rhs == 0 || (self == Self::MIN && rhs == -1)) {
|
||||
None
|
||||
} else {
|
||||
// SAFETY: div by zero and by INT_MIN have been checked above
|
||||
|
@ -907,7 +922,7 @@ assert_eq!((1", stringify!($SelfT), ").checked_div_euclid(0), None);
|
|||
without modifying the original"]
|
||||
#[inline]
|
||||
pub const fn checked_div_euclid(self, rhs: Self) -> Option<Self> {
|
||||
if rhs == 0 || (self == Self::MIN && rhs == -1) {
|
||||
if unlikely!(rhs == 0 || (self == Self::MIN && rhs == -1)) {
|
||||
None
|
||||
} else {
|
||||
Some(self.div_euclid(rhs))
|
||||
|
@ -936,7 +951,7 @@ $EndFeature, "
|
|||
without modifying the original"]
|
||||
#[inline]
|
||||
pub const fn checked_rem(self, rhs: Self) -> Option<Self> {
|
||||
if rhs == 0 || (self == Self::MIN && rhs == -1) {
|
||||
if unlikely!(rhs == 0 || (self == Self::MIN && rhs == -1)) {
|
||||
None
|
||||
} else {
|
||||
// SAFETY: div by zero and by INT_MIN have been checked above
|
||||
|
@ -964,7 +979,7 @@ assert_eq!(", stringify!($SelfT), "::MIN.checked_rem_euclid(-1), None);
|
|||
without modifying the original"]
|
||||
#[inline]
|
||||
pub const fn checked_rem_euclid(self, rhs: Self) -> Option<Self> {
|
||||
if rhs == 0 || (self == Self::MIN && rhs == -1) {
|
||||
if unlikely!(rhs == 0 || (self == Self::MIN && rhs == -1)) {
|
||||
None
|
||||
} else {
|
||||
Some(self.rem_euclid(rhs))
|
||||
|
@ -990,7 +1005,7 @@ $EndFeature, "
|
|||
#[inline]
|
||||
pub const fn checked_neg(self) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_neg();
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1014,7 +1029,7 @@ $EndFeature, "
|
|||
#[inline]
|
||||
pub const fn checked_shl(self, rhs: u32) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_shl(rhs);
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1038,7 +1053,7 @@ $EndFeature, "
|
|||
#[inline]
|
||||
pub const fn checked_shr(self, rhs: u32) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_shr(rhs);
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1745,7 +1760,7 @@ $EndFeature, "
|
|||
#[must_use = "this returns the result of the operation, \
|
||||
without modifying the original"]
|
||||
pub const fn overflowing_div(self, rhs: Self) -> (Self, bool) {
|
||||
if self == Self::MIN && rhs == -1 {
|
||||
if unlikely!(self == Self::MIN && rhs == -1) {
|
||||
(self, true)
|
||||
} else {
|
||||
(self / rhs, false)
|
||||
|
@ -1778,7 +1793,7 @@ assert_eq!(", stringify!($SelfT), "::MIN.overflowing_div_euclid(-1), (", stringi
|
|||
#[must_use = "this returns the result of the operation, \
|
||||
without modifying the original"]
|
||||
pub const fn overflowing_div_euclid(self, rhs: Self) -> (Self, bool) {
|
||||
if self == Self::MIN && rhs == -1 {
|
||||
if unlikely!(self == Self::MIN && rhs == -1) {
|
||||
(self, true)
|
||||
} else {
|
||||
(self.div_euclid(rhs), false)
|
||||
|
@ -1812,7 +1827,7 @@ $EndFeature, "
|
|||
#[must_use = "this returns the result of the operation, \
|
||||
without modifying the original"]
|
||||
pub const fn overflowing_rem(self, rhs: Self) -> (Self, bool) {
|
||||
if self == Self::MIN && rhs == -1 {
|
||||
if unlikely!(self == Self::MIN && rhs == -1) {
|
||||
(0, true)
|
||||
} else {
|
||||
(self % rhs, false)
|
||||
|
@ -1845,7 +1860,7 @@ assert_eq!(", stringify!($SelfT), "::MIN.overflowing_rem_euclid(-1), (0, true));
|
|||
without modifying the original"]
|
||||
#[inline]
|
||||
pub const fn overflowing_rem_euclid(self, rhs: Self) -> (Self, bool) {
|
||||
if self == Self::MIN && rhs == -1 {
|
||||
if unlikely!(self == Self::MIN && rhs == -1) {
|
||||
(0, true)
|
||||
} else {
|
||||
(self.rem_euclid(rhs), false)
|
||||
|
@ -1876,7 +1891,7 @@ assert_eq!(", stringify!($SelfT), "::MIN.overflowing_neg(), (", stringify!($Self
|
|||
#[allow(unused_attributes)]
|
||||
#[cfg_attr(bootstrap, allow_internal_unstable(const_if_match))]
|
||||
pub const fn overflowing_neg(self) -> (Self, bool) {
|
||||
if self == Self::MIN {
|
||||
if unlikely!(self == Self::MIN) {
|
||||
(Self::MIN, true)
|
||||
} else {
|
||||
(-self, false)
|
||||
|
@ -2988,7 +3003,7 @@ assert_eq!((", stringify!($SelfT), "::MAX - 2).checked_add(3), None);", $EndFeat
|
|||
#[inline]
|
||||
pub const fn checked_add(self, rhs: Self) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_add(rhs);
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3030,7 +3045,7 @@ assert_eq!(0", stringify!($SelfT), ".checked_sub(1), None);", $EndFeature, "
|
|||
#[inline]
|
||||
pub const fn checked_sub(self, rhs: Self) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_sub(rhs);
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3072,7 +3087,7 @@ assert_eq!(", stringify!($SelfT), "::MAX.checked_mul(2), None);", $EndFeature, "
|
|||
#[inline]
|
||||
pub const fn checked_mul(self, rhs: Self) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_mul(rhs);
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3113,11 +3128,12 @@ assert_eq!(1", stringify!($SelfT), ".checked_div(0), None);", $EndFeature, "
|
|||
without modifying the original"]
|
||||
#[inline]
|
||||
pub const fn checked_div(self, rhs: Self) -> Option<Self> {
|
||||
match rhs {
|
||||
0 => None,
|
||||
if unlikely!(rhs == 0) {
|
||||
None
|
||||
} else {
|
||||
// SAFETY: div by zero has been checked above and unsigned types have no other
|
||||
// failure modes for division
|
||||
rhs => Some(unsafe { intrinsics::unchecked_div(self, rhs) }),
|
||||
Some(unsafe { intrinsics::unchecked_div(self, rhs) })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -3140,7 +3156,7 @@ assert_eq!(1", stringify!($SelfT), ".checked_div_euclid(0), None);
|
|||
without modifying the original"]
|
||||
#[inline]
|
||||
pub const fn checked_div_euclid(self, rhs: Self) -> Option<Self> {
|
||||
if rhs == 0 {
|
||||
if unlikely!(rhs == 0) {
|
||||
None
|
||||
} else {
|
||||
Some(self.div_euclid(rhs))
|
||||
|
@ -3167,7 +3183,7 @@ assert_eq!(5", stringify!($SelfT), ".checked_rem(0), None);", $EndFeature, "
|
|||
without modifying the original"]
|
||||
#[inline]
|
||||
pub const fn checked_rem(self, rhs: Self) -> Option<Self> {
|
||||
if rhs == 0 {
|
||||
if unlikely!(rhs == 0) {
|
||||
None
|
||||
} else {
|
||||
// SAFETY: div by zero has been checked above and unsigned types have no other
|
||||
|
@ -3195,7 +3211,7 @@ assert_eq!(5", stringify!($SelfT), ".checked_rem_euclid(0), None);
|
|||
without modifying the original"]
|
||||
#[inline]
|
||||
pub const fn checked_rem_euclid(self, rhs: Self) -> Option<Self> {
|
||||
if rhs == 0 {
|
||||
if unlikely!(rhs == 0) {
|
||||
None
|
||||
} else {
|
||||
Some(self.rem_euclid(rhs))
|
||||
|
@ -3222,7 +3238,7 @@ assert_eq!(1", stringify!($SelfT), ".checked_neg(), None);", $EndFeature, "
|
|||
#[inline]
|
||||
pub const fn checked_neg(self) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_neg();
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3245,7 +3261,7 @@ assert_eq!(0x10", stringify!($SelfT), ".checked_shl(129), None);", $EndFeature,
|
|||
#[inline]
|
||||
pub const fn checked_shl(self, rhs: u32) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_shl(rhs);
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3268,7 +3284,7 @@ assert_eq!(0x10", stringify!($SelfT), ".checked_shr(129), None);", $EndFeature,
|
|||
#[inline]
|
||||
pub const fn checked_shr(self, rhs: u32) -> Option<Self> {
|
||||
let (a, b) = self.overflowing_shr(rhs);
|
||||
if b {None} else {Some(a)}
|
||||
if unlikely!(b) {None} else {Some(a)}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue