From 68e0ea9d47f797f815225e4f2fbd9bb1cde6e19e Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Fri, 23 Mar 2018 01:30:23 -0700 Subject: [PATCH 1/5] Introduce unsafe offset_from on pointers Adds intrinsics::exact_div to take advantage of the unsafe, which reduces the implementation from ```asm sub rcx, rdx mov rax, rcx sar rax, 63 shr rax, 62 lea rax, [rax + rcx] sar rax, 2 ret ``` down to ```asm sub rcx, rdx sar rcx, 2 mov rax, rcx ret ``` (for `*const i32`) --- src/libcore/intrinsics.rs | 10 ++ src/libcore/ptr.rs | 207 +++++++++++++++++++++++++ src/librustc_llvm/ffi.rs | 5 + src/librustc_trans/builder.rs | 7 + src/librustc_trans/intrinsic.rs | 8 +- src/librustc_typeck/check/intrinsic.rs | 2 +- 6 files changed, 237 insertions(+), 2 deletions(-) diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs index 830ebad0654..3b740adc468 100644 --- a/src/libcore/intrinsics.rs +++ b/src/libcore/intrinsics.rs @@ -1314,6 +1314,11 @@ extern "rust-intrinsic" { /// [`std::u32::overflowing_mul`](../../std/primitive.u32.html#method.overflowing_mul) pub fn mul_with_overflow(x: T, y: T) -> (T, bool); + /// Performs an exact division, resulting in undefined behavior where + /// `x % y != 0` or `y == 0` or `x == T::min_value() && y == -1` + #[cfg(not(stage0))] + pub fn exact_div(x: T, y: T) -> T; + /// Performs an unchecked division, resulting in undefined behavior /// where y = 0 or x = `T::min_value()` and y = -1 pub fn unchecked_div(x: T, y: T) -> T; @@ -1396,3 +1401,8 @@ extern "rust-intrinsic" { /// Probably will never become stable. pub fn nontemporal_store(ptr: *mut T, val: T); } + +#[cfg(stage0)] +pub unsafe fn exact_div(a: T, b: T) -> T { + unchecked_div(a, b) +} diff --git a/src/libcore/ptr.rs b/src/libcore/ptr.rs index 6270e5892b3..cbd45bb6a39 100644 --- a/src/libcore/ptr.rs +++ b/src/libcore/ptr.rs @@ -700,6 +700,114 @@ impl *const T { } } + /// Calculates the distance between two pointers. The returned value is in + /// units of T: the distance in bytes is divided by `mem::size_of::()`. + /// + /// This function is the inverse of [`offset`]. + /// + /// [`offset`]: #method.offset + /// [`wrapping_offset_from`]: #method.wrapping_offset_from + /// + /// # Safety + /// + /// If any of the following conditions are violated, the result is Undefined + /// Behavior: + /// + /// * Both the starting and other pointer must be either in bounds or one + /// byte past the end of the same allocated object. + /// + /// * The distance between the pointers, **in bytes**, cannot overflow an `isize`. + /// + /// * The distance between the pointers, in bytes, must be an exact multiple + /// of the size of `T` and `T` must not be a Zero-Sized Type ("ZST"). + /// + /// * The distance being in bounds cannot rely on "wrapping around" the address space. + /// + /// The compiler and standard library generally try to ensure allocations + /// never reach a size where an offset is a concern. For instance, `Vec` + /// and `Box` ensure they never allocate more than `isize::MAX` bytes, so + /// `ptr_into_vec.offset_from(vec.as_ptr())` is always safe. + /// + /// Most platforms fundamentally can't even construct such an allocation. + /// For instance, no known 64-bit platform can ever serve a request + /// for 263 bytes due to page-table limitations or splitting the address space. + /// However, some 32-bit and 16-bit platforms may successfully serve a request for + /// more than `isize::MAX` bytes with things like Physical Address + /// Extension. As such, memory acquired directly from allocators or memory + /// mapped files *may* be too large to handle with this function. + /// + /// Consider using [`wrapping_offset_from`] instead if these constraints are + /// difficult to satisfy. The only advantage of this method is that it + /// enables more aggressive compiler optimizations. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(ptr_offset_from)] + /// + /// let a = [0; 5]; + /// let ptr1: *const i32 = &a[1]; + /// let ptr2: *const i32 = &a[3]; + /// unsafe { + /// assert_eq!(ptr2.offset_from(ptr1), 2); + /// assert_eq!(ptr1.offset_from(ptr2), -2); + /// assert_eq!(ptr1.offset(2), ptr2); + /// assert_eq!(ptr2.offset(-2), ptr1); + /// } + /// ``` + #[unstable(feature = "ptr_offset_from", issue = "41079")] + #[inline] + pub unsafe fn offset_from(self, other: *const T) -> isize where T: Sized { + let pointee_size = mem::size_of::(); + assert!(0 < pointee_size && pointee_size <= isize::max_value() as usize); + + // FIXME: can this be nuw/nsw? + let d = isize::wrapping_sub(self as _, other as _); + intrinsics::exact_div(d, pointee_size as _) + } + + /// Calculates the distance between two pointers. The returned value is in + /// units of T: the distance in bytes is divided by `mem::size_of::()`. + /// + /// If the address different between the two pointers is not a multiple of + /// `mem::size_of::()` then the result of the division is rounded towards + /// zero. + /// + /// # Panics + /// + /// This function panics if `T` is a zero-sized typed. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(ptr_wrapping_offset_from)] + /// + /// let a = [0; 5]; + /// let ptr1: *const i32 = &a[1]; + /// let ptr2: *const i32 = &a[3]; + /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); + /// assert_eq!(ptr1.wrapping_offset_from(ptr2), -2); + /// assert_eq!(ptr1.wrapping_offset(2), ptr2); + /// assert_eq!(ptr2.wrapping_offset(-2), ptr1); + /// + /// let ptr1: *const i32 = 3 as _; + /// let ptr2: *const i32 = 13 as _; + /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); + /// ``` + #[unstable(feature = "ptr_wrapping_offset_from", issue = "41079")] + #[inline] + pub fn wrapping_offset_from(self, other: *const T) -> isize where T: Sized { + let pointee_size = mem::size_of::(); + assert!(0 < pointee_size && pointee_size <= isize::max_value() as usize); + + let d = isize::wrapping_sub(self as _, other as _); + d.wrapping_div(pointee_size as _) + } + /// Calculates the offset from a pointer (convenience for `.offset(count as isize)`). /// /// `count` is in units of T; e.g. a `count` of 3 represents a pointer @@ -1347,6 +1455,105 @@ impl *mut T { } } + /// Calculates the distance between two pointers. The returned value is in + /// units of T: the distance in bytes is divided by `mem::size_of::()`. + /// + /// This function is the inverse of [`offset`]. + /// + /// [`offset`]: #method.offset-1 + /// [`wrapping_offset_from`]: #method.wrapping_offset_from-1 + /// + /// # Safety + /// + /// If any of the following conditions are violated, the result is Undefined + /// Behavior: + /// + /// * Both the starting and other pointer must be either in bounds or one + /// byte past the end of the same allocated object. + /// + /// * The distance between the pointers, **in bytes**, cannot overflow an `isize`. + /// + /// * The distance between the pointers, in bytes, must be an exact multiple + /// of the size of `T` and `T` must not be a Zero-Sized Type ("ZST"). + /// + /// * The distance being in bounds cannot rely on "wrapping around" the address space. + /// + /// The compiler and standard library generally try to ensure allocations + /// never reach a size where an offset is a concern. For instance, `Vec` + /// and `Box` ensure they never allocate more than `isize::MAX` bytes, so + /// `ptr_into_vec.offset_from(vec.as_ptr())` is always safe. + /// + /// Most platforms fundamentally can't even construct such an allocation. + /// For instance, no known 64-bit platform can ever serve a request + /// for 263 bytes due to page-table limitations or splitting the address space. + /// However, some 32-bit and 16-bit platforms may successfully serve a request for + /// more than `isize::MAX` bytes with things like Physical Address + /// Extension. As such, memory acquired directly from allocators or memory + /// mapped files *may* be too large to handle with this function. + /// + /// Consider using [`wrapping_offset_from`] instead if these constraints are + /// difficult to satisfy. The only advantage of this method is that it + /// enables more aggressive compiler optimizations. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(ptr_offset_from)] + /// + /// let a = [0; 5]; + /// let ptr1: *mut i32 = &mut a[1]; + /// let ptr2: *mut i32 = &mut a[3]; + /// unsafe { + /// assert_eq!(ptr2.offset_from(ptr1), 2); + /// assert_eq!(ptr1.offset_from(ptr2), -2); + /// assert_eq!(ptr1.offset(2), ptr2); + /// assert_eq!(ptr2.offset(-2), ptr1); + /// } + /// ``` + #[unstable(feature = "ptr_offset_from", issue = "41079")] + #[inline] + pub unsafe fn offset_from(self, other: *const T) -> isize where T: Sized { + (self as *const T).offset_from(other) + } + + /// Calculates the distance between two pointers. The returned value is in + /// units of T: the distance in bytes is divided by `mem::size_of::()`. + /// + /// If the address different between the two pointers is not a multiple of + /// `mem::size_of::()` then the result of the division is rounded towards + /// zero. + /// + /// # Panics + /// + /// This function panics if `T` is a zero-sized typed. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// #![feature(ptr_wrapping_offset_from)] + /// + /// let a = [0; 5]; + /// let ptr1: *mut i32 = &mut a[1]; + /// let ptr2: *mut i32 = &mut a[3]; + /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); + /// assert_eq!(ptr1.wrapping_offset_from(ptr2), -2); + /// assert_eq!(ptr1.wrapping_offset(2), ptr2); + /// assert_eq!(ptr2.wrapping_offset(-2), ptr1); + /// + /// let ptr1: *mut i32 = 3 as _; + /// let ptr2: *mut i32 = 13 as _; + /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2); + /// ``` + #[unstable(feature = "ptr_wrapping_offset_from", issue = "41079")] + #[inline] + pub fn wrapping_offset_from(self, other: *const T) -> isize where T: Sized { + (self as *const T).wrapping_offset_from(other) + } + /// Computes the byte offset that needs to be applied in order to /// make the pointer aligned to `align`. /// If it is not possible to align the pointer, the implementation returns diff --git a/src/librustc_llvm/ffi.rs b/src/librustc_llvm/ffi.rs index c0cdd212770..403fe4731f1 100644 --- a/src/librustc_llvm/ffi.rs +++ b/src/librustc_llvm/ffi.rs @@ -935,6 +935,11 @@ extern "C" { RHS: ValueRef, Name: *const c_char) -> ValueRef; + pub fn LLVMBuildExactUDiv(B: BuilderRef, + LHS: ValueRef, + RHS: ValueRef, + Name: *const c_char) + -> ValueRef; pub fn LLVMBuildSDiv(B: BuilderRef, LHS: ValueRef, RHS: ValueRef, diff --git a/src/librustc_trans/builder.rs b/src/librustc_trans/builder.rs index 91eabb9998f..5e2d32b3596 100644 --- a/src/librustc_trans/builder.rs +++ b/src/librustc_trans/builder.rs @@ -344,6 +344,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } } + pub fn exactudiv(&self, lhs: ValueRef, rhs: ValueRef) -> ValueRef { + self.count_insn("exactudiv"); + unsafe { + llvm::LLVMBuildExactUDiv(self.llbuilder, lhs, rhs, noname()) + } + } + pub fn sdiv(&self, lhs: ValueRef, rhs: ValueRef) -> ValueRef { self.count_insn("sdiv"); unsafe { diff --git a/src/librustc_trans/intrinsic.rs b/src/librustc_trans/intrinsic.rs index c3de9e0ffcc..ca5b48be4d5 100644 --- a/src/librustc_trans/intrinsic.rs +++ b/src/librustc_trans/intrinsic.rs @@ -289,7 +289,7 @@ pub fn trans_intrinsic_call<'a, 'tcx>(bx: &Builder<'a, 'tcx>, "ctlz" | "ctlz_nonzero" | "cttz" | "cttz_nonzero" | "ctpop" | "bswap" | "bitreverse" | "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" | "overflowing_add" | "overflowing_sub" | "overflowing_mul" | - "unchecked_div" | "unchecked_rem" | "unchecked_shl" | "unchecked_shr" => { + "unchecked_div" | "unchecked_rem" | "unchecked_shl" | "unchecked_shr" | "exact_div" => { let ty = arg_tys[0]; match int_type_width_signed(ty, cx) { Some((width, signed)) => @@ -343,6 +343,12 @@ pub fn trans_intrinsic_call<'a, 'tcx>(bx: &Builder<'a, 'tcx>, "overflowing_add" => bx.add(args[0].immediate(), args[1].immediate()), "overflowing_sub" => bx.sub(args[0].immediate(), args[1].immediate()), "overflowing_mul" => bx.mul(args[0].immediate(), args[1].immediate()), + "exact_div" => + if signed { + bx.exactsdiv(args[0].immediate(), args[1].immediate()) + } else { + bx.exactudiv(args[0].immediate(), args[1].immediate()) + }, "unchecked_div" => if signed { bx.sdiv(args[0].immediate(), args[1].immediate()) diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs index 99707a4a3c0..a4e9967daa6 100644 --- a/src/librustc_typeck/check/intrinsic.rs +++ b/src/librustc_typeck/check/intrinsic.rs @@ -283,7 +283,7 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, (1, vec![param(0), param(0)], tcx.intern_tup(&[param(0), tcx.types.bool])), - "unchecked_div" | "unchecked_rem" => + "unchecked_div" | "unchecked_rem" | "exact_div" => (1, vec![param(0), param(0)], param(0)), "unchecked_shl" | "unchecked_shr" => (1, vec![param(0), param(0)], param(0)), From d6926ca12daa51fd786f233019ac539d1a732493 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Fri, 23 Mar 2018 23:47:22 -0700 Subject: [PATCH 2/5] Add a codegen test for exact_div intrinsic --- src/test/codegen/exact_div.rs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/test/codegen/exact_div.rs diff --git a/src/test/codegen/exact_div.rs b/src/test/codegen/exact_div.rs new file mode 100644 index 00000000000..9ba6c0c0006 --- /dev/null +++ b/src/test/codegen/exact_div.rs @@ -0,0 +1,30 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-flags: -C no-prepopulate-passes + +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +use std::intrinsics::exact_div; + +// CHECK-LABEL: @exact_sdiv +#[no_mangle] +pub unsafe fn exact_sdiv(x: i32, y: i32) -> i32 { +// CHECK: sdiv exact + exact_div(x, y) +} + +// CHECK-LABEL: @exact_udiv +#[no_mangle] +pub unsafe fn exact_udiv(x: u32, y: u32) -> u32 { +// CHECK: udiv exact + exact_div(x, y) +} From 02b5851258c5a0db32d1c735d1c29e5e56f45ed1 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sat, 24 Mar 2018 19:15:12 -0700 Subject: [PATCH 3/5] Polyfill LLVMBuildExactUDiv It was added 32 days after LLVM 3.9 shipped. --- src/rustllvm/RustWrapper.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp index a5644d6f9e2..e815d151aeb 100644 --- a/src/rustllvm/RustWrapper.cpp +++ b/src/rustllvm/RustWrapper.cpp @@ -1492,3 +1492,11 @@ LLVMRustBuildVectorReduceFMax(LLVMBuilderRef, LLVMValueRef, bool) { return nullptr; } #endif + +#if LLVM_VERSION_LT(4, 0) +extern "C" LLVMValueRef +LLVMBuildExactUDiv(LLVMBuilderRef B, LLVMValueRef LHS, + LLVMValueRef RHS, const char *Name) { + return wrap(unwrap(B)->CreateExactUDiv(unwrap(LHS), unwrap(RHS), Name)); +} +#endif From 4a097ea53251e59063cf5bdc9901f0313664f90e Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sat, 24 Mar 2018 20:37:31 -0700 Subject: [PATCH 4/5] Documentation and naming improvements --- src/libcore/ptr.rs | 48 +++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/src/libcore/ptr.rs b/src/libcore/ptr.rs index cbd45bb6a39..3fc8421d3b0 100644 --- a/src/libcore/ptr.rs +++ b/src/libcore/ptr.rs @@ -669,7 +669,7 @@ impl *const T { /// `mem::size_of::()` then the result of the division is rounded towards /// zero. /// - /// This function returns `None` if `T` is a zero-sized typed. + /// This function returns `None` if `T` is a zero-sized type. /// /// # Examples /// @@ -719,7 +719,7 @@ impl *const T { /// * The distance between the pointers, **in bytes**, cannot overflow an `isize`. /// /// * The distance between the pointers, in bytes, must be an exact multiple - /// of the size of `T` and `T` must not be a Zero-Sized Type ("ZST"). + /// of the size of `T`. /// /// * The distance being in bounds cannot rely on "wrapping around" the address space. /// @@ -740,6 +740,10 @@ impl *const T { /// difficult to satisfy. The only advantage of this method is that it /// enables more aggressive compiler optimizations. /// + /// # Panics + /// + /// This function panics if `T` is a Zero-Sized Type ("ZST"). + /// /// # Examples /// /// Basic usage: @@ -759,12 +763,14 @@ impl *const T { /// ``` #[unstable(feature = "ptr_offset_from", issue = "41079")] #[inline] - pub unsafe fn offset_from(self, other: *const T) -> isize where T: Sized { + pub unsafe fn offset_from(self, origin: *const T) -> isize where T: Sized { let pointee_size = mem::size_of::(); assert!(0 < pointee_size && pointee_size <= isize::max_value() as usize); - // FIXME: can this be nuw/nsw? - let d = isize::wrapping_sub(self as _, other as _); + // This is the same sequence that Clang emits for pointer subtraction. + // It can be neither `nsw` nor `nuw` because the input is treated as + // unsigned but then the output is treated as signed, so neither works. + let d = isize::wrapping_sub(self as _, origin as _); intrinsics::exact_div(d, pointee_size as _) } @@ -775,9 +781,13 @@ impl *const T { /// `mem::size_of::()` then the result of the division is rounded towards /// zero. /// + /// Though this method is safe for any two pointers, note that its result + /// will be mostly useless if the two pointers aren't into the same allocated + /// object, for example if they point to two different local variables. + /// /// # Panics /// - /// This function panics if `T` is a zero-sized typed. + /// This function panics if `T` is a zero-sized type. /// /// # Examples /// @@ -800,11 +810,11 @@ impl *const T { /// ``` #[unstable(feature = "ptr_wrapping_offset_from", issue = "41079")] #[inline] - pub fn wrapping_offset_from(self, other: *const T) -> isize where T: Sized { + pub fn wrapping_offset_from(self, origin: *const T) -> isize where T: Sized { let pointee_size = mem::size_of::(); assert!(0 < pointee_size && pointee_size <= isize::max_value() as usize); - let d = isize::wrapping_sub(self as _, other as _); + let d = isize::wrapping_sub(self as _, origin as _); d.wrapping_div(pointee_size as _) } @@ -1424,7 +1434,7 @@ impl *mut T { /// `mem::size_of::()` then the result of the division is rounded towards /// zero. /// - /// This function returns `None` if `T` is a zero-sized typed. + /// This function returns `None` if `T` is a zero-sized type. /// /// # Examples /// @@ -1474,7 +1484,7 @@ impl *mut T { /// * The distance between the pointers, **in bytes**, cannot overflow an `isize`. /// /// * The distance between the pointers, in bytes, must be an exact multiple - /// of the size of `T` and `T` must not be a Zero-Sized Type ("ZST"). + /// of the size of `T`. /// /// * The distance being in bounds cannot rely on "wrapping around" the address space. /// @@ -1495,6 +1505,10 @@ impl *mut T { /// difficult to satisfy. The only advantage of this method is that it /// enables more aggressive compiler optimizations. /// + /// # Panics + /// + /// This function panics if `T` is a Zero-Sized Type ("ZST"). + /// /// # Examples /// /// Basic usage: @@ -1514,8 +1528,8 @@ impl *mut T { /// ``` #[unstable(feature = "ptr_offset_from", issue = "41079")] #[inline] - pub unsafe fn offset_from(self, other: *const T) -> isize where T: Sized { - (self as *const T).offset_from(other) + pub unsafe fn offset_from(self, origin: *const T) -> isize where T: Sized { + (self as *const T).offset_from(origin) } /// Calculates the distance between two pointers. The returned value is in @@ -1525,9 +1539,13 @@ impl *mut T { /// `mem::size_of::()` then the result of the division is rounded towards /// zero. /// + /// Though this method is safe for any two pointers, note that its result + /// will be mostly useless if the two pointers aren't into the same allocated + /// object, for example if they point to two different local variables. + /// /// # Panics /// - /// This function panics if `T` is a zero-sized typed. + /// This function panics if `T` is a zero-sized type. /// /// # Examples /// @@ -1550,8 +1568,8 @@ impl *mut T { /// ``` #[unstable(feature = "ptr_wrapping_offset_from", issue = "41079")] #[inline] - pub fn wrapping_offset_from(self, other: *const T) -> isize where T: Sized { - (self as *const T).wrapping_offset_from(other) + pub fn wrapping_offset_from(self, origin: *const T) -> isize where T: Sized { + (self as *const T).wrapping_offset_from(origin) } /// Computes the byte offset that needs to be applied in order to From 62649524b94fb83e2ed472088ea1a0cd87079fd2 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sat, 24 Mar 2018 20:41:20 -0700 Subject: [PATCH 5/5] Fix doctest mutability copy-pasta --- src/libcore/ptr.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libcore/ptr.rs b/src/libcore/ptr.rs index 3fc8421d3b0..0723fa8a23d 100644 --- a/src/libcore/ptr.rs +++ b/src/libcore/ptr.rs @@ -1516,7 +1516,7 @@ impl *mut T { /// ``` /// #![feature(ptr_offset_from)] /// - /// let a = [0; 5]; + /// let mut a = [0; 5]; /// let ptr1: *mut i32 = &mut a[1]; /// let ptr2: *mut i32 = &mut a[3]; /// unsafe { @@ -1554,7 +1554,7 @@ impl *mut T { /// ``` /// #![feature(ptr_wrapping_offset_from)] /// - /// let a = [0; 5]; + /// let mut a = [0; 5]; /// let ptr1: *mut i32 = &mut a[1]; /// let ptr2: *mut i32 = &mut a[3]; /// assert_eq!(ptr2.wrapping_offset_from(ptr1), 2);