Implement ptr::{read,write}_unaligned via repr(packed)

library/core/src/ptr/mod.rs

@@ -1810,16 +1810,24 @@ pub const unsafe fn read<T>(src: *const T) -> T {
 #[track_caller]
 #[rustc_diagnostic_item = "ptr_read_unaligned"]
 pub const unsafe fn read_unaligned<T>(src: *const T) -> T {
-    let mut tmp = MaybeUninit::<T>::uninit();
+    // Always true thanks to the repr, but to demonstrate
+    const {
+        assert!(mem::offset_of!(Packed::<T>, 0) == 0);
+        assert!(size_of::<T>() == size_of::<Packed<T>>());
+    }
+
+    let src = src.cast::<Packed<T>>();
     // SAFETY: the caller must guarantee that `src` is valid for reads.
-    // `src` cannot overlap `tmp` because `tmp` was just allocated on
-    // the stack as a separate allocation.
+    // Reading it as `Packed<T>` instead of `T` reads those same bytes because
+    // it's the same size (thus zero offset), but with alignment 1 instead.
     //
-    // Also, since we just wrote a valid value into `tmp`, it is guaranteed
-    // to be properly initialized.
+    // Similarly, because it's the same bytes it's sound to transmute from the
+    // `Packed<T>` to `T`.  Transmute is a value-based (not a place-based)
+    // operation that doesn't care about alignment.
     unsafe {
-        copy_nonoverlapping(src as *const u8, tmp.as_mut_ptr() as *mut u8, size_of::<T>());
-        tmp.assume_init()
+        let packed = read(src);
+        // Can't just destructure because that's not allowed in const fn
+        mem::transmute_neo(packed)
     }
 }
 
@@ -2012,14 +2020,18 @@ pub const unsafe fn write<T>(dst: *mut T, src: T) {
 #[rustc_diagnostic_item = "ptr_write_unaligned"]
 #[track_caller]
 pub const unsafe fn write_unaligned<T>(dst: *mut T, src: T) {
-    // SAFETY: the caller must guarantee that `dst` is valid for writes.
-    // `dst` cannot overlap `src` because the caller has mutable access
-    // to `dst` while `src` is owned by this function.
-    unsafe {
-        copy_nonoverlapping((&raw const src) as *const u8, dst as *mut u8, size_of::<T>());
-        // We are calling the intrinsic directly to avoid function calls in the generated code.
-        intrinsics::forget(src);
+    // Always true thanks to the repr, but to demonstrate
+    const {
+        assert!(mem::offset_of!(Packed::<T>, 0) == 0);
+        assert!(size_of::<T>() == size_of::<Packed<T>>());
     }
+
+    let dst = dst.cast::<Packed<T>>();
+    let src = Packed(src);
+    // SAFETY: the caller must guarantee that `dst` is valid for writes.
+    // Writing it as `Packed<T>` instead of `T` writes those same bytes because
+    // it's the same size (thus zero offset), but with alignment 1 instead.
+    unsafe { write(dst, src) }
 }
 
 /// Performs a volatile read of the value from `src` without moving it.
@@ -2809,3 +2821,6 @@ pub macro addr_of($place:expr) {
 pub macro addr_of_mut($place:expr) {
     &raw mut $place
 }
+
+#[repr(C, packed)]
+struct Packed<T>(T);

tests/codegen-llvm/read_write_unaligned.rs

@@ -0,0 +1,71 @@
+//@ compile-flags: -Copt-level=3
+//@ only-64bit
+
+#![crate_type = "lib"]
+
+use std::num::NonZero;
+use std::ptr::NonNull;
+
+// CHECK-LABEL: nonnull ptr @read_unaligned_ptr(ptr{{.+}}%ptr)
+#[no_mangle]
+unsafe fn read_unaligned_ptr(ptr: *const NonNull<i16>) -> NonNull<i16> {
+    // CHECK: start:
+    // CHECK-NEXT: [[TEMP:%.+]] = load ptr, ptr %ptr, align 1
+    // CHECK-SAME: !nonnull
+    // CHECK-SAME: !noundef
+    // CHECK-NEXT: ret ptr [[TEMP]]
+    ptr.read_unaligned()
+}
+
+// CHECK-LABEL: range(i16 1, 0) i16 @read_unaligned_i16(ptr{{.+}}%ptr)
+#[no_mangle]
+unsafe fn read_unaligned_i16(ptr: *const NonZero<i16>) -> NonZero<i16> {
+    // CHECK: start:
+    // CHECK-NEXT: [[TEMP:%.+]] = load i16, ptr %ptr, align 1
+    // CHECK-NOT: !noundef
+    // CHECK-NOT: !range
+    // CHECK-NEXT: ret i16 [[TEMP]]
+    ptr.read_unaligned()
+}
+
+// CHECK-LABEL: void @typed_copy_unaligned_i32(ptr{{.+}}%src, ptr{{.+}}%dst)
+#[no_mangle]
+unsafe fn typed_copy_unaligned_i32(src: *const NonZero<i32>, dst: *mut NonZero<i32>) {
+    // CHECK: start:
+    // CHECK-NEXT: [[TEMP:%.+]] = load i32, ptr %src, align 1
+    // CHECK-NOT: !noundef
+    // CHECK-NOT: !range
+    // CHECK-NEXT: store i32 [[TEMP]], ptr %dst, align 1
+    // CHECK-NEXT: ret void
+    dst.write_unaligned(src.read_unaligned())
+}
+
+// CHECK-LABEL: void @write_unaligned_i64(ptr{{.+}}%ptr, i64{{.+}}%val)
+#[no_mangle]
+unsafe fn write_unaligned_i64(ptr: *mut NonZero<i64>, val: NonZero<i64>) {
+    // CHECK: start:
+    // CHECK-NEXT: store i64 %val, ptr %ptr, align 1
+    // CHECK-NEXT: ret void
+    ptr.write_unaligned(val)
+}
+
+#[repr(align(128))]
+struct HugeBuffer([u64; 1 << 10]);
+
+// CHECK-LABEL: void @read_unaligned_huge(ptr{{.+}}%_0, ptr{{.+}}%ptr)
+#[no_mangle]
+unsafe fn read_unaligned_huge(ptr: *const HugeBuffer) -> HugeBuffer {
+    // CHECK: start:
+    // CHECK-NEXT: call void @llvm.memcpy{{.+}} align 128 dereferenceable(8192) %_0, {{.+}} align 1 dereferenceable(8192) %ptr, i64 8192,
+    // CHECK-NEXT: ret void
+    ptr.read_unaligned()
+}
+
+// CHECK-LABEL: void @write_unaligned_huge(ptr{{.+}}%ptr, ptr{{.+}}%val)
+#[no_mangle]
+unsafe fn write_unaligned_huge(ptr: *mut HugeBuffer, val: HugeBuffer) {
+    // CHECK: start:
+    // CHECK-NEXT: call void @llvm.memcpy{{.+}} align 1 dereferenceable(8192) %ptr, {{.+}} align 128 dereferenceable(8192) %val, i64 8192,
+    // CHECK-NEXT: ret void
+    ptr.write_unaligned(val)
+}

tests/mir-opt/pre-codegen/unaligned.rs

@@ -0,0 +1,39 @@
+//@ compile-flags: -O -Zmir-opt-level=2
+//@ ignore-std-debug-assertions (there's one in `ptr::read`)
+
+#![crate_type = "lib"]
+
+// EMIT_MIR unaligned.unaligned_copy_manual.runtime-optimized.after.mir
+pub unsafe fn unaligned_copy_manual(src: *const u128, dst: *mut u128) {
+    #[repr(packed)]
+    struct Packed<T>(T);
+
+    // CHECK-LABEL: fn unaligned_copy_manual(_1: *const u128, _2: *mut u128) -> ()
+    // CHECK: [[SRCU:_.+]] = copy _1 as *const unaligned_copy_manual::Packed<u128> (PtrToPtr);
+    // CHECK: [[DSTU:_.+]] = copy _2 as *mut unaligned_copy_manual::Packed<u128> (PtrToPtr);
+    // CHECK: [[TEMP:_.+]] = copy ((*[[SRCU]]).0: u128);
+    // ((*[[DSTU]]).0: u128) = move [[TEMP]];
+    let src = src.cast::<Packed<u128>>();
+    let dst = dst.cast::<Packed<u128>>();
+    unsafe { (*dst).0 = (*src).0 };
+}
+
+// EMIT_MIR unaligned.unaligned_copy_generic.runtime-optimized.after.mir
+pub unsafe fn unaligned_copy_generic<T>(src: *const T, dst: *mut T) {
+    // CHECK-LABEL: fn unaligned_copy_generic(
+    // CHECK: debug src => _1;
+    // CHECK: debug dst => _2;
+    // CHECK: debug val => [[VAL:_.+]];
+    // CHECK: [[SRC_P:_.+]] = copy _1 as *const {{.+}}::Packed<T> (PtrToPtr);
+    // CHECK: [[PACKED1:_.+]] = copy (*[[SRC_P]]);
+    // CHECK: [[VAL]] = copy [[PACKED1]] as T (Transmute);
+    // CHECK: [[DST_P:_.+]] = copy _2 as *mut {{.+}}::Packed<T> (PtrToPtr);
+    // CHECK: [[PACKED2:_.+]] = {{.+}}::Packed::<T>(copy [[VAL]]);
+    // CHECK: (*[[DST_P]]) = copy [[PACKED2]];
+    // CHECK-NOT: copy_nonoverlapping
+    // CHECK-NOT: drop
+    unsafe {
+        let val = std::ptr::read_unaligned(src);
+        std::ptr::write_unaligned(dst, val);
+    }
+}

tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir

@@ -0,0 +1,57 @@
+// MIR for `unaligned_copy_generic` after runtime-optimized
+
+fn unaligned_copy_generic(_1: *const T, _2: *mut T) -> () {
+    debug src => _1;
+    debug dst => _2;
+    let mut _0: ();
+    let _5: T;
+    scope 1 {
+        debug val => _5;
+        scope 8 (inlined #[track_caller] write_unaligned::<T>) {
+            let _6: *mut std::ptr::Packed<T>;
+            scope 9 {
+                let _7: std::ptr::Packed<T>;
+                scope 10 {
+                    scope 12 (inlined #[track_caller] std::ptr::write::<std::ptr::Packed<T>>) {
+                    }
+                }
+            }
+            scope 11 (inlined std::ptr::mut_ptr::<impl *mut T>::cast::<std::ptr::Packed<T>>) {
+            }
+        }
+    }
+    scope 2 (inlined #[track_caller] read_unaligned::<T>) {
+        let _3: *const std::ptr::Packed<T>;
+        scope 3 {
+            let _4: std::ptr::Packed<T>;
+            scope 4 {
+                scope 7 (inlined transmute_neo::<std::ptr::Packed<T>, T>) {
+                }
+            }
+            scope 6 (inlined #[track_caller] std::ptr::read::<std::ptr::Packed<T>>) {
+            }
+        }
+        scope 5 (inlined std::ptr::const_ptr::<impl *const T>::cast::<std::ptr::Packed<T>>) {
+        }
+    }
+
+    bb0: {
+        StorageLive(_5);
+        StorageLive(_3);
+        _3 = copy _1 as *const std::ptr::Packed<T> (PtrToPtr);
+        StorageLive(_4);
+        _4 = copy (*_3);
+        _5 = copy _4 as T (Transmute);
+        StorageDead(_4);
+        StorageDead(_3);
+        StorageLive(_6);
+        _6 = copy _2 as *mut std::ptr::Packed<T> (PtrToPtr);
+        StorageLive(_7);
+        _7 = std::ptr::Packed::<T>(copy _5);
+        (*_6) = copy _7;
+        StorageDead(_7);
+        StorageDead(_6);
+        StorageDead(_5);
+        return;
+    }
+}

tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir

@@ -0,0 +1,34 @@
+// MIR for `unaligned_copy_manual` after runtime-optimized
+
+fn unaligned_copy_manual(_1: *const u128, _2: *mut u128) -> () {
+    debug src => _1;
+    debug dst => _2;
+    let mut _0: ();
+    let _3: *const unaligned_copy_manual::Packed<u128>;
+    let mut _5: u128;
+    scope 1 {
+        debug src => _3;
+        let _4: *mut unaligned_copy_manual::Packed<u128>;
+        scope 2 {
+            debug dst => _4;
+        }
+        scope 4 (inlined std::ptr::mut_ptr::<impl *mut u128>::cast::<Packed<u128>>) {
+        }
+    }
+    scope 3 (inlined std::ptr::const_ptr::<impl *const u128>::cast::<Packed<u128>>) {
+    }
+
+    bb0: {
+        StorageLive(_3);
+        _3 = copy _1 as *const unaligned_copy_manual::Packed<u128> (PtrToPtr);
+        StorageLive(_4);
+        _4 = copy _2 as *mut unaligned_copy_manual::Packed<u128> (PtrToPtr);
+        StorageLive(_5);
+        _5 = copy ((*_3).0: u128);
+        ((*_4).0: u128) = move _5;
+        StorageDead(_5);
+        StorageDead(_4);
+        StorageDead(_3);
+        return;
+    }
+}