Enum Layout Optimization: Variant fields reordering around niche.

compiler/rustc_abi/src/lib.rs

@@ -94,7 +94,6 @@ bitflags! {
         /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details.
         const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5;
         const IS_SCALABLE        = 1 << 6;
-         // Any of these flags being set prevent field reordering optimisation.
         const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
                                  | ReprFlags::IS_SCALABLE.bits()

compiler/rustc_ast/src/ast.rs

@@ -4359,7 +4359,13 @@ mod size_asserts {
     static_assert_size!(GenericBound, 88);
     static_assert_size!(Generics, 40);
     static_assert_size!(Impl, 80);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Item, 144);
+    #[cfg(bootstrap)]
     static_assert_size!(Item, 152);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(ItemKind, 80);
+    #[cfg(bootstrap)]
     static_assert_size!(ItemKind, 88);
     static_assert_size!(LitKind, 24);
     static_assert_size!(Local, 96);

tests/ui/print_type_sizes/padding.stdout

@@ -1,21 +1,15 @@
-print-type-size type: `E1`: 12 bytes, alignment: 4 bytes
-print-type-size     discriminant: 1 bytes
-print-type-size     variant `B`: 11 bytes
-print-type-size         padding: 3 bytes
-print-type-size         field `.0`: 8 bytes, alignment: 4 bytes
-print-type-size     variant `A`: 7 bytes
+print-type-size type: `E1`: 8 bytes, alignment: 4 bytes
+print-type-size     variant `B`: 8 bytes
+print-type-size         field `.0`: 8 bytes
+print-type-size     variant `A`: 5 bytes
+print-type-size         field `.0`: 4 bytes
 print-type-size         field `.1`: 1 bytes
-print-type-size         padding: 2 bytes
-print-type-size         field `.0`: 4 bytes, alignment: 4 bytes
-print-type-size type: `E2`: 12 bytes, alignment: 4 bytes
-print-type-size     discriminant: 1 bytes
-print-type-size     variant `B`: 11 bytes
-print-type-size         padding: 3 bytes
-print-type-size         field `.0`: 8 bytes, alignment: 4 bytes
-print-type-size     variant `A`: 7 bytes
+print-type-size type: `E2`: 8 bytes, alignment: 4 bytes
+print-type-size     variant `B`: 8 bytes
+print-type-size         field `.0`: 8 bytes
+print-type-size     variant `A`: 5 bytes
+print-type-size         field `.1`: 4 bytes
 print-type-size         field `.0`: 1 bytes
-print-type-size         padding: 2 bytes
-print-type-size         field `.1`: 4 bytes, alignment: 4 bytes
 print-type-size type: `S`: 8 bytes, alignment: 4 bytes
 print-type-size     field `.g`: 4 bytes
 print-type-size     field `.a`: 1 bytes

tests/ui/stats/input-stats.rs

@@ -14,6 +14,7 @@
 // bump occurs, stage1 and stage2 will give different outputs for this test.
 // Add an `ignore-stage1` comment marker to work around that problem during
 // that time.
+//@ ignore-stage1
 
 
 // The aim here is to include at least one of every different type of top-level

tests/ui/stats/input-stats.stderr

@@ -2,14 +2,14 @@ ast-stats ================================================================
 ast-stats POST EXPANSION AST STATS: input_stats
 ast-stats Name                Accumulated Size         Count     Item Size
 ast-stats ----------------------------------------------------------------
-ast-stats Item                   1_672 (NN.N%)            11           152
-ast-stats - Enum                     152 (NN.N%)             1
-ast-stats - ExternCrate              152 (NN.N%)             1
-ast-stats - ForeignMod               152 (NN.N%)             1
-ast-stats - Impl                     152 (NN.N%)             1
-ast-stats - Trait                    152 (NN.N%)             1
-ast-stats - Fn                       304 (NN.N%)             2
-ast-stats - Use                      608 (NN.N%)             4
+ast-stats Item                   1_584 (NN.N%)            11           144
+ast-stats - Enum                     144 (NN.N%)             1
+ast-stats - ExternCrate              144 (NN.N%)             1
+ast-stats - ForeignMod               144 (NN.N%)             1
+ast-stats - Impl                     144 (NN.N%)             1
+ast-stats - Trait                    144 (NN.N%)             1
+ast-stats - Fn                       288 (NN.N%)             2
+ast-stats - Use                      576 (NN.N%)             4
 ast-stats Ty                       896 (NN.N%)            14            64
 ast-stats - Ptr                       64 (NN.N%)             1
 ast-stats - Ref                       64 (NN.N%)             1
@@ -57,7 +57,7 @@ ast-stats GenericArgs               40 (NN.N%)             1            40
 ast-stats - AngleBracketed            40 (NN.N%)             1
 ast-stats Crate                     40 (NN.N%)             1            40
 ast-stats ----------------------------------------------------------------
-ast-stats Total                  7_624                   127
+ast-stats Total                  7_536                   127
 ast-stats ================================================================
 hir-stats ================================================================
 hir-stats HIR STATS: input_stats

tests/ui/structs-enums/enum-niche-fill-around-tag.rs

@@ -0,0 +1,126 @@
+//@ run-pass
+
+#![allow(dead_code)]
+#![feature(offset_of_enum)]
+
+use std::mem::{align_of, offset_of};
+use std::mem::size_of;
+use std::num::NonZero;
+
+enum Inner {
+    A(u8, u32),
+    B(u32),
+}
+
+enum Outer {
+    A(Inner),
+    B(u32, u8),
+}
+
+struct WithPadding {
+    a: bool,
+    b: bool,
+    c: u32,
+}
+
+enum RepackPadding {
+    A(u32, u8),
+    B(WithPadding),
+}
+
+struct RepackedBase {
+    int32: u32,
+    boolean: bool,
+}
+
+enum RepackedAroundNiche {
+    A(RepackedBase),
+    B(u16, u32, u8),
+}
+
+enum NestedRepackedAroundNiche {
+    A(RepackedAroundNiche),
+    B(u16, u32, u8),
+}
+
+struct RepackedPair {
+    field2: u16,
+    field3: u8,
+}
+
+enum RepackedNestedAggregate {
+    A(u16, RepackedPair),
+    B(NestedRepackedAroundNiche),
+}
+
+struct FittingNichePayload {
+    word: u32,
+    flag: bool,
+}
+
+// The first and last max-sized niche candidates cannot encode all variants.
+// The middle candidate can, so the enum layout has to keep looking.
+enum ChooseFittingNiche {
+    SmallNicheFront(u16, u32, NonZero<u8>),
+    WideNiche(FittingNichePayload),
+    SmallNicheBack(u16, u32, NonZero<u8>),
+    V0,
+    V1,
+    V2,
+}
+
+fn main() {
+    assert_eq!(size_of::<Inner>(), 8);
+
+    assert_eq!(size_of::<Outer>(), 8);
+    assert_eq!(size_of::<RepackPadding>(), 8);
+
+    assert_eq!(size_of::<RepackedBase>(), 8);
+    assert_eq!(align_of::<RepackedBase>(), 4);
+    assert_eq!(offset_of!(RepackedBase, int32), 0);
+    assert_eq!(offset_of!(RepackedBase, boolean), 4);
+
+    assert_eq!(size_of::<RepackedAroundNiche>(), 8);
+    assert_eq!(align_of::<RepackedAroundNiche>(), 4);
+    assert_eq!(size_of::<Option<RepackedAroundNiche>>(), 8);
+    assert_eq!(offset_of!(RepackedAroundNiche, A.0), 0);
+    assert_eq!(offset_of!(RepackedAroundNiche, B.1), 0);
+    assert_eq!(offset_of!(RepackedAroundNiche, B.2), 5);
+    assert_eq!(offset_of!(RepackedAroundNiche, B.0), 6);
+
+    assert_eq!(size_of::<NestedRepackedAroundNiche>(), 8);
+    assert_eq!(align_of::<NestedRepackedAroundNiche>(), 4);
+    assert_eq!(size_of::<Option<NestedRepackedAroundNiche>>(), 8);
+    assert_eq!(offset_of!(NestedRepackedAroundNiche, A.0), 0);
+    assert_eq!(offset_of!(NestedRepackedAroundNiche, B.1), 0);
+    assert_eq!(offset_of!(NestedRepackedAroundNiche, B.2), 5);
+    assert_eq!(offset_of!(NestedRepackedAroundNiche, B.0), 6);
+
+    assert_eq!(size_of::<RepackedPair>(), 4);
+    assert_eq!(align_of::<RepackedPair>(), 2);
+    assert_eq!(offset_of!(RepackedPair, field2), 0);
+    assert_eq!(offset_of!(RepackedPair, field3), 2);
+
+    assert_eq!(size_of::<RepackedNestedAggregate>(), 8);
+    assert_eq!(align_of::<RepackedNestedAggregate>(), 4);
+    assert_eq!(size_of::<Option<RepackedNestedAggregate>>(), 8);
+    assert_eq!(offset_of!(RepackedNestedAggregate, B.0), 0);
+    assert_eq!(offset_of!(RepackedNestedAggregate, A.1), 0);
+    assert_eq!(offset_of!(RepackedNestedAggregate, A.0), 6);
+
+    assert_eq!(size_of::<FittingNichePayload>(), 8);
+    assert_eq!(align_of::<FittingNichePayload>(), 4);
+    assert_eq!(offset_of!(FittingNichePayload, word), 0);
+    assert_eq!(offset_of!(FittingNichePayload, flag), 4);
+
+    assert_eq!(size_of::<ChooseFittingNiche>(), 8);
+    assert_eq!(align_of::<ChooseFittingNiche>(), 4);
+    assert_eq!(size_of::<Option<ChooseFittingNiche>>(), 8);
+    assert_eq!(offset_of!(ChooseFittingNiche, WideNiche.0), 0);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.1), 0);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.2), 5);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheFront.0), 6);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.1), 0);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.2), 5);
+    assert_eq!(offset_of!(ChooseFittingNiche, SmallNicheBack.0), 6);
+}