diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 593011edecf27..b4d47205f2fea 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -1810,16 +1810,24 @@ pub const unsafe fn read(src: *const T) -> T { #[track_caller] #[rustc_diagnostic_item = "ptr_read_unaligned"] pub const unsafe fn read_unaligned(src: *const T) -> T { - let mut tmp = MaybeUninit::::uninit(); + // Always true thanks to the repr, but to demonstrate + const { + assert!(mem::offset_of!(Packed::, 0) == 0); + assert!(size_of::() == size_of::>()); + } + + let src = src.cast::>(); // SAFETY: the caller must guarantee that `src` is valid for reads. - // `src` cannot overlap `tmp` because `tmp` was just allocated on - // the stack as a separate allocation. + // Reading it as `Packed` instead of `T` reads those same bytes because + // it's the same size (thus zero offset), but with alignment 1 instead. // - // Also, since we just wrote a valid value into `tmp`, it is guaranteed - // to be properly initialized. + // Similarly, because it's the same bytes it's sound to transmute from the + // `Packed` to `T`. Transmute is a value-based (not a place-based) + // operation that doesn't care about alignment. unsafe { - copy_nonoverlapping(src as *const u8, tmp.as_mut_ptr() as *mut u8, size_of::()); - tmp.assume_init() + let packed = read(src); + // Can't just destructure because that's not allowed in const fn + mem::transmute_neo(packed) } } @@ -2012,14 +2020,18 @@ pub const unsafe fn write(dst: *mut T, src: T) { #[rustc_diagnostic_item = "ptr_write_unaligned"] #[track_caller] pub const unsafe fn write_unaligned(dst: *mut T, src: T) { - // SAFETY: the caller must guarantee that `dst` is valid for writes. - // `dst` cannot overlap `src` because the caller has mutable access - // to `dst` while `src` is owned by this function. - unsafe { - copy_nonoverlapping((&raw const src) as *const u8, dst as *mut u8, size_of::()); - // We are calling the intrinsic directly to avoid function calls in the generated code. - intrinsics::forget(src); + // Always true thanks to the repr, but to demonstrate + const { + assert!(mem::offset_of!(Packed::, 0) == 0); + assert!(size_of::() == size_of::>()); } + + let dst = dst.cast::>(); + let src = Packed(src); + // SAFETY: the caller must guarantee that `dst` is valid for writes. + // Writing it as `Packed` instead of `T` writes those same bytes because + // it's the same size (thus zero offset), but with alignment 1 instead. + unsafe { write(dst, src) } } /// Performs a volatile read of the value from `src` without moving it. @@ -2809,3 +2821,6 @@ pub macro addr_of($place:expr) { pub macro addr_of_mut($place:expr) { &raw mut $place } + +#[repr(C, packed)] +struct Packed(T); diff --git a/tests/codegen-llvm/read_write_unaligned.rs b/tests/codegen-llvm/read_write_unaligned.rs new file mode 100644 index 0000000000000..beba4400dc54a --- /dev/null +++ b/tests/codegen-llvm/read_write_unaligned.rs @@ -0,0 +1,71 @@ +//@ compile-flags: -Copt-level=3 +//@ only-64bit + +#![crate_type = "lib"] + +use std::num::NonZero; +use std::ptr::NonNull; + +// CHECK-LABEL: nonnull ptr @read_unaligned_ptr(ptr{{.+}}%ptr) +#[no_mangle] +unsafe fn read_unaligned_ptr(ptr: *const NonNull) -> NonNull { + // CHECK: start: + // CHECK-NEXT: [[TEMP:%.+]] = load ptr, ptr %ptr, align 1 + // CHECK-SAME: !nonnull + // CHECK-SAME: !noundef + // CHECK-NEXT: ret ptr [[TEMP]] + ptr.read_unaligned() +} + +// CHECK-LABEL: range(i16 1, 0) i16 @read_unaligned_i16(ptr{{.+}}%ptr) +#[no_mangle] +unsafe fn read_unaligned_i16(ptr: *const NonZero) -> NonZero { + // CHECK: start: + // CHECK-NEXT: [[TEMP:%.+]] = load i16, ptr %ptr, align 1 + // CHECK-NOT: !noundef + // CHECK-NOT: !range + // CHECK-NEXT: ret i16 [[TEMP]] + ptr.read_unaligned() +} + +// CHECK-LABEL: void @typed_copy_unaligned_i32(ptr{{.+}}%src, ptr{{.+}}%dst) +#[no_mangle] +unsafe fn typed_copy_unaligned_i32(src: *const NonZero, dst: *mut NonZero) { + // CHECK: start: + // CHECK-NEXT: [[TEMP:%.+]] = load i32, ptr %src, align 1 + // CHECK-NOT: !noundef + // CHECK-NOT: !range + // CHECK-NEXT: store i32 [[TEMP]], ptr %dst, align 1 + // CHECK-NEXT: ret void + dst.write_unaligned(src.read_unaligned()) +} + +// CHECK-LABEL: void @write_unaligned_i64(ptr{{.+}}%ptr, i64{{.+}}%val) +#[no_mangle] +unsafe fn write_unaligned_i64(ptr: *mut NonZero, val: NonZero) { + // CHECK: start: + // CHECK-NEXT: store i64 %val, ptr %ptr, align 1 + // CHECK-NEXT: ret void + ptr.write_unaligned(val) +} + +#[repr(align(128))] +struct HugeBuffer([u64; 1 << 10]); + +// CHECK-LABEL: void @read_unaligned_huge(ptr{{.+}}%_0, ptr{{.+}}%ptr) +#[no_mangle] +unsafe fn read_unaligned_huge(ptr: *const HugeBuffer) -> HugeBuffer { + // CHECK: start: + // CHECK-NEXT: call void @llvm.memcpy{{.+}} align 128 dereferenceable(8192) %_0, {{.+}} align 1 dereferenceable(8192) %ptr, i64 8192, + // CHECK-NEXT: ret void + ptr.read_unaligned() +} + +// CHECK-LABEL: void @write_unaligned_huge(ptr{{.+}}%ptr, ptr{{.+}}%val) +#[no_mangle] +unsafe fn write_unaligned_huge(ptr: *mut HugeBuffer, val: HugeBuffer) { + // CHECK: start: + // CHECK-NEXT: call void @llvm.memcpy{{.+}} align 1 dereferenceable(8192) %ptr, {{.+}} align 128 dereferenceable(8192) %val, i64 8192, + // CHECK-NEXT: ret void + ptr.write_unaligned(val) +} diff --git a/tests/mir-opt/pre-codegen/unaligned.rs b/tests/mir-opt/pre-codegen/unaligned.rs new file mode 100644 index 0000000000000..4aeb10c261e21 --- /dev/null +++ b/tests/mir-opt/pre-codegen/unaligned.rs @@ -0,0 +1,39 @@ +//@ compile-flags: -O -Zmir-opt-level=2 +//@ ignore-std-debug-assertions (there's one in `ptr::read`) + +#![crate_type = "lib"] + +// EMIT_MIR unaligned.unaligned_copy_manual.runtime-optimized.after.mir +pub unsafe fn unaligned_copy_manual(src: *const u128, dst: *mut u128) { + #[repr(packed)] + struct Packed(T); + + // CHECK-LABEL: fn unaligned_copy_manual(_1: *const u128, _2: *mut u128) -> () + // CHECK: [[SRCU:_.+]] = copy _1 as *const unaligned_copy_manual::Packed (PtrToPtr); + // CHECK: [[DSTU:_.+]] = copy _2 as *mut unaligned_copy_manual::Packed (PtrToPtr); + // CHECK: [[TEMP:_.+]] = copy ((*[[SRCU]]).0: u128); + // ((*[[DSTU]]).0: u128) = move [[TEMP]]; + let src = src.cast::>(); + let dst = dst.cast::>(); + unsafe { (*dst).0 = (*src).0 }; +} + +// EMIT_MIR unaligned.unaligned_copy_generic.runtime-optimized.after.mir +pub unsafe fn unaligned_copy_generic(src: *const T, dst: *mut T) { + // CHECK-LABEL: fn unaligned_copy_generic( + // CHECK: debug src => _1; + // CHECK: debug dst => _2; + // CHECK: debug val => [[VAL:_.+]]; + // CHECK: [[SRC_P:_.+]] = copy _1 as *const {{.+}}::Packed (PtrToPtr); + // CHECK: [[PACKED1:_.+]] = copy (*[[SRC_P]]); + // CHECK: [[VAL]] = copy [[PACKED1]] as T (Transmute); + // CHECK: [[DST_P:_.+]] = copy _2 as *mut {{.+}}::Packed (PtrToPtr); + // CHECK: [[PACKED2:_.+]] = {{.+}}::Packed::(copy [[VAL]]); + // CHECK: (*[[DST_P]]) = copy [[PACKED2]]; + // CHECK-NOT: copy_nonoverlapping + // CHECK-NOT: drop + unsafe { + let val = std::ptr::read_unaligned(src); + std::ptr::write_unaligned(dst, val); + } +} diff --git a/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir new file mode 100644 index 0000000000000..4d0cfa6a4dfa3 --- /dev/null +++ b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir @@ -0,0 +1,57 @@ +// MIR for `unaligned_copy_generic` after runtime-optimized + +fn unaligned_copy_generic(_1: *const T, _2: *mut T) -> () { + debug src => _1; + debug dst => _2; + let mut _0: (); + let _5: T; + scope 1 { + debug val => _5; + scope 8 (inlined #[track_caller] write_unaligned::) { + let _6: *mut std::ptr::Packed; + scope 9 { + let _7: std::ptr::Packed; + scope 10 { + scope 12 (inlined #[track_caller] std::ptr::write::>) { + } + } + } + scope 11 (inlined std::ptr::mut_ptr::::cast::>) { + } + } + } + scope 2 (inlined #[track_caller] read_unaligned::) { + let _3: *const std::ptr::Packed; + scope 3 { + let _4: std::ptr::Packed; + scope 4 { + scope 7 (inlined transmute_neo::, T>) { + } + } + scope 6 (inlined #[track_caller] std::ptr::read::>) { + } + } + scope 5 (inlined std::ptr::const_ptr::::cast::>) { + } + } + + bb0: { + StorageLive(_5); + StorageLive(_3); + _3 = copy _1 as *const std::ptr::Packed (PtrToPtr); + StorageLive(_4); + _4 = copy (*_3); + _5 = copy _4 as T (Transmute); + StorageDead(_4); + StorageDead(_3); + StorageLive(_6); + _6 = copy _2 as *mut std::ptr::Packed (PtrToPtr); + StorageLive(_7); + _7 = std::ptr::Packed::(copy _5); + (*_6) = copy _7; + StorageDead(_7); + StorageDead(_6); + StorageDead(_5); + return; + } +} diff --git a/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir new file mode 100644 index 0000000000000..450df935a0fb3 --- /dev/null +++ b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir @@ -0,0 +1,34 @@ +// MIR for `unaligned_copy_manual` after runtime-optimized + +fn unaligned_copy_manual(_1: *const u128, _2: *mut u128) -> () { + debug src => _1; + debug dst => _2; + let mut _0: (); + let _3: *const unaligned_copy_manual::Packed; + let mut _5: u128; + scope 1 { + debug src => _3; + let _4: *mut unaligned_copy_manual::Packed; + scope 2 { + debug dst => _4; + } + scope 4 (inlined std::ptr::mut_ptr::::cast::>) { + } + } + scope 3 (inlined std::ptr::const_ptr::::cast::>) { + } + + bb0: { + StorageLive(_3); + _3 = copy _1 as *const unaligned_copy_manual::Packed (PtrToPtr); + StorageLive(_4); + _4 = copy _2 as *mut unaligned_copy_manual::Packed (PtrToPtr); + StorageLive(_5); + _5 = copy ((*_3).0: u128); + ((*_4).0: u128) = move _5; + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + return; + } +}