Add a layout_of_val intrinsic for when you want both size_of_val+align_of_val

compiler/rustc_codegen_llvm/src/lib.rs

@@ -319,8 +319,12 @@ impl CodegenBackend for LlvmCodegenBackend {
     }
 
     fn replaced_intrinsics(&self) -> Vec<Symbol> {
-        let mut will_not_use_fallback =
-            vec![sym::unchecked_funnel_shl, sym::unchecked_funnel_shr, sym::carrying_mul_add];
+        let mut will_not_use_fallback = vec![
+            sym::unchecked_funnel_shl,
+            sym::unchecked_funnel_shr,
+            sym::carrying_mul_add,
+            sym::layout_of_val,
+        ];
 
         if llvm_util::get_version() >= (22, 0, 0) {
             will_not_use_fallback.push(sym::carryless_mul);

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -1,4 +1,4 @@
-use rustc_abi::{Align, WrappingRange};
+use rustc_abi::{Align, FIRST_VARIANT, FieldIdx, WrappingRange};
 use rustc_middle::mir::SourceInfo;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::{bug, span_bug};
@@ -7,7 +7,7 @@ use rustc_span::sym;
 use rustc_target::spec::Arch;
 
 use super::FunctionCx;
-use super::operand::OperandRef;
+use super::operand::{OperandRef, OperandRefBuilder};
 use super::place::PlaceRef;
 use crate::common::{AtomicRmwBinOp, SynchronizationScope};
 use crate::errors::InvalidMonomorphization;
@@ -150,17 +150,39 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
             sym::va_start => bx.va_start(args[0].immediate()),
             sym::va_end => bx.va_end(args[0].immediate()),
-            sym::size_of_val => {
+            sym::size_of_val | sym::align_of_val | sym::layout_of_val => {
                 let tp_ty = fn_args.type_at(0);
                 let (_, meta) = args[0].val.pointer_parts();
-                let (llsize, _) = size_of_val::size_and_align_of_dst(bx, tp_ty, meta);
-                llsize
-            }
-            sym::align_of_val => {
-                let tp_ty = fn_args.type_at(0);
-                let (_, meta) = args[0].val.pointer_parts();
-                let (_, llalign) = size_of_val::size_and_align_of_dst(bx, tp_ty, meta);
-                llalign
+                let (llsize, llalign) = size_of_val::size_and_align_of_dst(bx, tp_ty, meta);
+                match name {
+                    sym::size_of_val => llsize,
+                    sym::align_of_val => llalign,
+                    sym::layout_of_val => {
+                        // The builder insulates us from in-memory order, but double-check declared order
+                        debug_assert!({
+                            let layout_adt = result.layout.ty.ty_adt_def().unwrap();
+                            let layout_fields = layout_adt.variant(FIRST_VARIANT).fields.as_slice();
+                            if let [size, align] = &layout_fields.raw
+                                && size.name == sym::size
+                                && align.name == sym::align
+                            {
+                                true
+                            } else {
+                                false
+                            }
+                        });
+
+                        let mut builder = OperandRefBuilder::<'_, Bx::Value>::new(result.layout);
+                        builder.insert_imm(FieldIdx::from_u32(0), llsize);
+                        builder.insert_imm(FieldIdx::from_u32(1), llalign);
+                        let val = builder.build(bx.cx()).val;
+                        // the match can only return a single `Bx::Value`,
+                        // so we need to do the store and return.
+                        val.store(bx, result);
+                        return Ok(());
+                    }
+                    _ => bug!(),
+                }
             }
             sym::vtable_size | sym::vtable_align => {
                 let vtable = args[0].immediate();

compiler/rustc_codegen_ssa/src/size_of_val.rs

@@ -159,7 +159,12 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             // Furthermore, `align >= unsized_align`, and therefore we only need to do:
             // let full_size = (unsized_offset_unadjusted + unsized_size).align_to(full_align);
 
-            let full_size = bx.add(unsized_offset_unadjusted, unsized_size);
+            // This is the size *before* rounding up, which cannot exceed the size *after*
+            // rounding up, which itself cannot exceed `isize::MAX`. Thus the addition
+            // itself cannot overflow `isize::MAX`, let alone `usize::MAX`.
+            // (The range attribute from loading the size from the vtable is enough to prove
+            // `nuw`, but not `nsw`, which we only know from Rust's layout rules.)
+            let full_size = bx.unchecked_suadd(unsized_offset_unadjusted, unsized_size);
 
             // Issue #27023: must add any necessary padding to `size`
             // (to make it a multiple of `align`) before returning it.

compiler/rustc_hir/src/lang_items.rs

@@ -158,6 +158,7 @@ language_item_table! {
     Unsize,                  sym::unsize,              unsize_trait,               Target::Trait,          GenericRequirement::Minimum(1);
     AlignOf,                 sym::mem_align_const,     align_const,                Target::AssocConst,     GenericRequirement::Exact(0);
     SizeOf,                  sym::mem_size_const,      size_const,                 Target::AssocConst,     GenericRequirement::Exact(0);
+    LayoutOf,                sym::mem_layout_const,    layout_const,               Target::AssocConst,     GenericRequirement::Exact(0);
     OffsetOf,                sym::offset_of,           offset_of,                  Target::Fn,             GenericRequirement::Exact(1);
     /// Trait injected by `#[derive(PartialEq)]`, (i.e. "Partial EQ").
     StructuralPeq,           sym::structural_peq,      structural_peq_trait,       Target::Trait,          GenericRequirement::None;

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -294,6 +294,9 @@ pub(crate) fn check_intrinsic_type(
         sym::size_of_val | sym::align_of_val => {
             (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.types.usize)
         }
+        sym::layout_of_val => {
+            (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.ty_alloc_layout(span))
+        }
         sym::offset_of => (1, 0, vec![tcx.types.u32, tcx.types.u32], tcx.types.usize),
         sym::field_offset => (1, 0, vec![], tcx.types.usize),
         sym::rustc_peek => (1, 0, vec![param(0)], param(0)),

compiler/rustc_middle/src/ty/context.rs

@@ -1038,6 +1038,12 @@ impl<'tcx> TyCtxt<'tcx> {
         self.type_of(ordering_enum).no_bound_vars().unwrap()
     }
 
+    /// Gets a `Ty` representing the [`LangItem::AllocLayout`]
+    pub fn ty_alloc_layout(self, span: Span) -> Ty<'tcx> {
+        let layout_did = self.require_lang_item(hir::LangItem::AllocLayout, span);
+        self.type_of(layout_did).no_bound_vars().unwrap()
+    }
+
     /// Obtain the given diagnostic item's `DefId`. Use `is_diagnostic_item` if you just want to
     /// compare against another `DefId`, since `is_diagnostic_item` is cheaper.
     pub fn get_diagnostic_item(self, name: Symbol) -> Option<DefId> {

compiler/rustc_mir_transform/src/instsimplify.rs

@@ -271,6 +271,8 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> {
                 LangItem::SizeOf
             } else if self.tcx.is_intrinsic(fn_def_id, sym::align_of_val) {
                 LangItem::AlignOf
+            } else if self.tcx.is_intrinsic(fn_def_id, sym::layout_of_val) {
+                LangItem::LayoutOf
             } else {
                 return;
             };

compiler/rustc_span/src/symbol.rs

@@ -1141,6 +1141,7 @@ symbols! {
         large_assignments,
         last,
         lateout,
+        layout_of_val,
         lazy_normalization_consts,
         lazy_type_alias,
         le,
@@ -1244,6 +1245,7 @@ symbols! {
         mem_discriminant,
         mem_drop,
         mem_forget,
+        mem_layout_const,
         mem_size_const,
         mem_swap,
         mem_uninitialized,

library/core/src/alloc/layout.rs

@@ -5,7 +5,7 @@
 // Your performance intuition is useless. Run perf.
 
 use crate::error::Error;
-use crate::intrinsics::{unchecked_add, unchecked_mul, unchecked_sub};
+use crate::intrinsics::{self, unchecked_add, unchecked_mul, unchecked_sub};
 use crate::mem::{Alignment, SizedTypeProperties};
 use crate::ptr::NonNull;
 use crate::{assert_unsafe_precondition, fmt, mem};
@@ -26,6 +26,9 @@ use crate::{assert_unsafe_precondition, fmt, mem};
 /// requirements, or use the more lenient `Allocator` interface.)
 #[stable(feature = "alloc_layout", since = "1.28.0")]
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+// BEWARE! The implementation of the `layout_of_val` intrinsic is coupled to the
+// declared order of these fields.  As a reminder, you'll also get a (debug-only)
+// ICE if you change their names, though you can easily update that expectation.
 #[lang = "alloc_layout"]
 pub struct Layout {
     // size of the requested block of memory, measured in bytes.
@@ -210,14 +213,23 @@ impl Layout {
     /// Produces layout describing a record that could be used to
     /// allocate backing structure for `T` (which could be a trait
     /// or other unsized type like a slice).
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::alloc::Layout;
+    ///
+    /// let array = [1_u8, 2, 3];
+    /// assert_eq!(Layout::for_value::<[u8]>(&array), Layout::from_size_align(3, 1).unwrap());
+    /// ```
     #[stable(feature = "alloc_layout", since = "1.28.0")]
     #[rustc_const_stable(feature = "const_alloc_layout", since = "1.85.0")]
     #[must_use]
     #[inline]
     pub const fn for_value<T: ?Sized>(t: &T) -> Self {
-        let (size, alignment) = (size_of_val(t), Alignment::of_val(t));
-        // SAFETY: see rationale in `new` for why this is using the unsafe variant
-        unsafe { Layout::from_size_alignment_unchecked(size, alignment) }
+        // SAFETY: val is a reference, so if it's to a DST it has valid metadata.
+        // (And if `T` is sized there's no requirements on the pointer.)
+        unsafe { Layout::for_value_raw(t) }
     }
 
     /// Produces layout describing a record that could be used to
@@ -247,14 +259,36 @@ impl Layout {
     ///
     /// [trait object]: ../../book/ch17-02-trait-objects.html
     /// [extern type]: ../../unstable-book/language-features/extern-types.html
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(layout_for_ptr)]
+    ///
+    /// use std::alloc::Layout;
+    /// use std::ptr;
+    ///
+    /// let arbitrary = ptr::without_provenance::<[u16; 3]>(123456);
+    /// assert_eq!(
+    ///     // SAFETY: for a sized pointee, the function is always sound.
+    ///     unsafe { Layout::for_value_raw(arbitrary) },
+    ///     Layout::from_size_align(6, 2).unwrap(),
+    /// );
+    ///
+    /// let slice = ptr::slice_from_raw_parts(arbitrary, 789);
+    /// assert_eq!(
+    ///     // SAFETY: with a slice pointee, this is sound because the length
+    ///     // is short enough that size in bytes doesn't overflow isize::MAX.
+    ///     unsafe { Layout::for_value_raw(slice) },
+    ///     Layout::from_size_align(6 * 789, 2).unwrap(),
+    /// );
+    /// ```
     #[unstable(feature = "layout_for_ptr", issue = "69835")]
     #[must_use]
     #[inline]
-    pub const unsafe fn for_value_raw<T: ?Sized>(t: *const T) -> Self {
+    pub const unsafe fn for_value_raw<T: ?Sized>(ptr: *const T) -> Self {
         // SAFETY: we pass along the prerequisites of these functions to the caller
-        let (size, alignment) = unsafe { (mem::size_of_val_raw(t), Alignment::of_val_raw(t)) };
-        // SAFETY: see rationale in `new` for why this is using the unsafe variant
-        unsafe { Layout::from_size_alignment_unchecked(size, alignment) }
+        unsafe { intrinsics::layout_of_val(ptr) }
     }
 
     /// Creates a `NonNull` that is dangling, but well-aligned for this Layout.

library/core/src/intrinsics/mod.rs

@@ -53,6 +53,7 @@
     issue = "none"
 )]
 
+use crate::alloc::Layout;
 use crate::ffi::{VaArgSafe, VaList};
 use crate::marker::{ConstParamTy, DiscriminantKind, PointeeSized, Tuple};
 use crate::{mem, ptr};
@@ -2887,6 +2888,26 @@ pub const fn type_id_vtable(
     )
 }
 
+/// The size and alignment of the referenced value in bytes.
+///
+/// The stabilized version of this intrinsic is [`Layout::for_value_raw`].
+///
+/// # Safety
+///
+/// See [`Layout::for_value_raw`] for safety conditions.
+#[rustc_nounwind]
+#[unstable(feature = "core_intrinsics", issue = "none")]
+#[rustc_intrinsic]
+// This adds no semantics or UB atop just calling `size_of_val`+`align_of_val`.
+#[miri::intrinsic_fallback_is_spec]
+pub const unsafe fn layout_of_val<T: ?Sized>(ptr: *const T) -> Layout {
+    // SAFETY: we pass along the prerequisites of these functions to the caller
+    let (size, align) = unsafe { (size_of_val(ptr), align_of_val(ptr)) };
+    // SAFETY: The size and alignment of a valid allocation (or type)
+    // always meet the requirements of `Layout`.
+    unsafe { Layout::from_size_align_unchecked(size, align) }
+}
+
 /// Compute the type information of a concrete type.
 /// It can only be called at compile time, the backends do
 /// not implement it.

library/core/src/mem/mod.rs

@@ -1421,6 +1421,7 @@ pub trait SizedTypeProperties: Sized {
 
     #[doc(hidden)]
     #[unstable(feature = "sized_type_properties", issue = "none")]
+    #[lang = "mem_layout_const"]
     const LAYOUT: Layout = {
         // SAFETY: if the type is instantiated, rustc already ensures that its
         // layout is valid. Use the unchecked constructor to avoid inserting a

tests/codegen-llvm/intrinsics/layout_of_val.rs

@@ -0,0 +1,76 @@
+//@ compile-flags: -Copt-level=3 -C no-prepopulate-passes -Z inline-mir
+//@ only-64bit (so I don't need to worry about usize)
+//@ needs-deterministic-layouts
+
+// Note that the layout algorithm currently puts the align before the size,
+// because the *type* for the size doesn't have a niche.  This test may need
+// to be updated if the in-memory field order of `Layout` ever changes.
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::alloc::Layout;
+use std::intrinsics::layout_of_val;
+
+// CHECK-LABEL: @thin_metadata(
+#[no_mangle]
+pub unsafe fn thin_metadata(ptr: *const [u32; 2]) -> Layout {
+    // CHECK-NOT: alloca
+    // CHECK: ret { i64, i64 } { i64 4, i64 8 }
+    layout_of_val(ptr)
+}
+
+// CHECK-LABEL: @slice_metadata(ptr noundef %ptr.0, i64 noundef %ptr.1)
+#[no_mangle]
+pub unsafe fn slice_metadata(ptr: *const [u32]) -> Layout {
+    // CHECK: [[LAYOUT:%.+]] = alloca [16 x i8], align 8
+    // CHECK-NOT: load
+    // CHECK-NOT: store
+    // CHECK: [[BYTES:%.+]] = mul nuw nsw i64 %ptr.1, 4
+    // CHECK-NEXT: store i64 4, ptr [[LAYOUT]], align 8
+    // CHECK-NEXT: [[SIZEP:%.+]] = getelementptr inbounds i8, ptr [[LAYOUT]], i64 8
+    // CHECK-NEXT: store i64 [[BYTES]], ptr [[SIZEP]], align 8
+    // CHECK-NOT: store
+    layout_of_val(ptr)
+}
+
+pub struct WithTail<T: ?Sized>([u32; 3], T);
+
+// CHECK-LABEL: @dst_metadata
+// CHECK-SAME: (ptr noundef %ptr.0, ptr{{.+}}%ptr.1)
+#[no_mangle]
+pub unsafe fn dst_metadata(ptr: *const WithTail<dyn std::fmt::Debug>) -> Layout {
+    // CHECK: [[LAYOUT:%.+]] = alloca [16 x i8], align 8
+    // CHECK-NOT: load
+    // CHECK-NOT: store
+    // CHECK: [[DST_SIZEP:%.+]] = getelementptr inbounds i8, ptr %ptr.1, i64 8
+    // CHECK-NEXT: [[DST_SIZE:%.+]] = load i64, ptr [[DST_SIZEP]], align 8,
+    // CHECK-SAME: !range [[SIZE_RANGE:.+]], !invariant.load
+    // CHECK-NEXT: [[DST_ALIGNP:%.+]] = getelementptr inbounds i8, ptr %ptr.1, i64 16
+    // CHECK-NEXT: [[DST_ALIGN:%.+]] = load i64, ptr [[DST_ALIGNP]], align 8,
+    // CHECK-SAME: !range [[ALIGN_RANGE:!.+]], !invariant.load
+
+    // CHECK-NEXT: [[STRUCT_MORE:%.+]] = icmp ugt i64 4, [[DST_ALIGN]]
+    // CHECK-NEXT: [[ALIGN:%.+]] = select i1 [[STRUCT_MORE]], i64 4, i64 [[DST_ALIGN]]
+
+    // CHECK-NEXT: [[MINSIZE:%.+]] = add nuw nsw i64 12, [[DST_SIZE]]
+    // CHECK-NEXT: [[ALIGN_M1:%.+]] = sub i64 [[ALIGN]], 1
+    // CHECK-NEXT: [[MAXSIZE:%.+]] = add i64 [[MINSIZE]], [[ALIGN_M1]]
+    // CHECK-NEXT: [[ALIGN_NEG:%.+]] = sub i64 0, [[ALIGN]]
+    // CHECK-NEXT: [[SIZE:%.+]] = and i64 [[MAXSIZE]], [[ALIGN_NEG]]
+
+    // CHECK-NEXT: store i64 [[ALIGN]], ptr [[LAYOUT]], align 8
+    // CHECK-NEXT: [[LAYOUT_SIZEP:%.+]] = getelementptr inbounds i8, ptr [[LAYOUT]], i64 8
+    // CHECK-NEXT: store i64 [[SIZE]], ptr [[LAYOUT_SIZEP]], align 8
+
+    // CHECK-NOT: store
+    // CHECK: load i64, {{.+}} !range [[ALIGNMENT_RANGE:!.+]],
+    // CHECK-NOT: store
+    layout_of_val(ptr)
+}
+
+// CHECK-LABEL: declare
+
+// CHECK: [[ALIGNMENT_RANGE]] = !{i64 1, i64 -[[#0x7FFFFFFFFFFFFFFF]]}
+// CHECK: [[SIZE_RANGE]] = !{i64 0, i64 -[[#0x8000000000000000]]}
+// CHECK: [[ALIGN_RANGE]] = !{i64 1, i64 [[#0x20000001]]}

tests/mir-opt/instsimplify/align_or_size_of_sized_val.layout_of_val_sized.InstSimplify-after-simplifycfg.diff

@@ -0,0 +1,22 @@
+- // MIR for `layout_of_val_sized` before InstSimplify-after-simplifycfg
++ // MIR for `layout_of_val_sized` after InstSimplify-after-simplifycfg
+
+  fn layout_of_val_sized(_1: &T) -> Layout {
+      debug val => _1;
+      let mut _0: std::alloc::Layout;
+      let mut _2: *const T;
+
+      bb0: {
+          StorageLive(_2);
+          _2 = &raw const (*_1);
+-         _0 = layout_of_val::<T>(move _2) -> [return: bb1, unwind unreachable];
++         _0 = const <T as std::mem::SizedTypeProperties>::LAYOUT;
++         goto -> bb1;
+      }
+
+      bb1: {
+          StorageDead(_2);
+          return;
+      }
+  }
+

tests/mir-opt/instsimplify/align_or_size_of_sized_val.rs

@@ -4,6 +4,8 @@
 #![crate_type = "lib"]
 #![feature(core_intrinsics)]
 
+use std::alloc::Layout;
+
 // EMIT_MIR align_or_size_of_sized_val.align_of_val_sized.InstSimplify-after-simplifycfg.diff
 pub fn align_of_val_sized<T>(val: &T) -> usize {
     // CHECK-LABEL: fn align_of_val_sized
@@ -17,3 +19,10 @@ pub fn size_of_val_sized<T>(val: &T) -> usize {
     // CHECK: _0 = const <T as std::mem::SizedTypeProperties>::SIZE;
     unsafe { core::intrinsics::size_of_val(val) }
 }
+
+// EMIT_MIR align_or_size_of_sized_val.layout_of_val_sized.InstSimplify-after-simplifycfg.diff
+pub fn layout_of_val_sized<T>(val: &T) -> Layout {
+    // CHECK-LABEL: fn layout_of_val_sized
+    // CHECK: _0 = const <T as std::mem::SizedTypeProperties>::LAYOUT;
+    unsafe { core::intrinsics::layout_of_val(val) }
+}