From 418c2788bd5e583d05e9d85266e0b3f44adeb7bb Mon Sep 17 00:00:00 2001 From: Michael Benfield Date: Thu, 6 Oct 2022 19:46:27 +0000 Subject: [PATCH] Use a Flag to enable more niche optimizations. Also: Try to fit other variants by moving fields around the niche. Keep multiple niches, not just the largest one. Look for multiple largest variants. Introduce repr(flag). Fixes #101567 --- Cargo.lock | 1 + compiler/rustc_ast/src/ast.rs | 6 +- compiler/rustc_attr/src/builtin.rs | 2 + .../src/abi/comments.rs | 10 +- .../src/discriminant.rs | 5 +- .../src/debuginfo/metadata/enums/mod.rs | 50 +- compiler/rustc_codegen_ssa/src/mir/place.rs | 111 +-- .../rustc_const_eval/src/interpret/operand.rs | 46 +- .../rustc_const_eval/src/interpret/place.rs | 11 +- .../src/interpret/validity.rs | 16 +- compiler/rustc_hir/src/hir.rs | 12 +- compiler/rustc_middle/src/ty/layout.rs | 24 +- compiler/rustc_middle/src/ty/mod.rs | 11 +- compiler/rustc_passes/src/check_attr.rs | 6 + compiler/rustc_span/src/symbol.rs | 1 + compiler/rustc_target/src/abi/mod.rs | 134 +++- compiler/rustc_ty_utils/Cargo.toml | 1 + compiler/rustc_ty_utils/src/layout.rs | 644 ++++++++++++++---- src/test/ui/layout/debug.stderr | 52 +- src/test/ui/layout/hexagon-enum.stderr | 105 +-- ...-scalarpair-payload-might-be-uninit.stderr | 139 ++-- .../issue-96185-overaligned-enum.stderr | 46 +- src/test/ui/layout/thumb-enum.stderr | 105 +-- .../layout/zero-sized-array-enum-niche.stderr | 155 +++-- src/test/ui/print_type_sizes/padding.stdout | 32 +- src/test/ui/stats/hir-stats.stderr | 152 ++--- src/test/ui/structs-enums/type-sizes.rs | 55 +- 27 files changed, 1301 insertions(+), 631 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5336477e0ed5..7ce14b31bf253 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4249,6 +4249,7 @@ dependencies = [ "rustc_target", "rustc_trait_selection", "rustc_type_ir", + "smallvec", "tracing", ] diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 0efde1e7b2124..28254adcfdc89 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -3065,6 +3065,8 @@ mod size_asserts { static_assert_size!(PathSegment, 24); static_assert_size!(Stmt, 32); static_assert_size!(StmtKind, 16); - static_assert_size!(Ty, 96); - static_assert_size!(TyKind, 72); + #[cfg(not(bootstrap))] + static_assert_size!(Ty, 88); + #[cfg(not(bootstrap))] + static_assert_size!(TyKind, 64); } diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs index 753f62dd589d0..3c3f382836dbf 100644 --- a/compiler/rustc_attr/src/builtin.rs +++ b/compiler/rustc_attr/src/builtin.rs @@ -944,6 +944,7 @@ pub enum ReprAttr { ReprSimd, ReprTransparent, ReprAlign(u32), + ReprFlag, } #[derive(Eq, PartialEq, Debug, Copy, Clone)] @@ -998,6 +999,7 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec { recognised = true; None } + sym::flag => Some(ReprFlag), name => int_type_of_word(name).map(ReprInt), }; diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs index 7f4619b5c940b..d8fddeeaccbf5 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs @@ -82,14 +82,8 @@ pub(super) fn add_local_place_comments<'tcx>( return; } let TyAndLayout { ty, layout } = place.layout(); - let rustc_target::abi::LayoutS { - size, - align, - abi: _, - variants: _, - fields: _, - largest_niche: _, - } = layout.0.0; + let rustc_target::abi::LayoutS { size, align, abi: _, variants: _, fields: _, niches: _ } = + layout.0.0; let (kind, extra) = match *place.inner() { CPlaceInner::Var(place_local, var) => { diff --git a/compiler/rustc_codegen_cranelift/src/discriminant.rs b/compiler/rustc_codegen_cranelift/src/discriminant.rs index 97b395bcd0518..4bbcb195f0901 100644 --- a/compiler/rustc_codegen_cranelift/src/discriminant.rs +++ b/compiler/rustc_codegen_cranelift/src/discriminant.rs @@ -42,7 +42,8 @@ pub(crate) fn codegen_set_discriminant<'tcx>( Variants::Multiple { tag: _, tag_field, - tag_encoding: TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start }, + tag_encoding: + TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start, .. }, variants: _, } => { if variant_index != untagged_variant { @@ -113,7 +114,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>( let res = CValue::by_val(val, dest_layout); dest.write_cvalue(fx, res); } - TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => { + TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start, .. } => { // Rebase from niche values to discriminants, and check // whether the result is in range for the niche variants. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 14044d0f99b98..487b793d357f0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -430,32 +430,34 @@ fn compute_discriminant_value<'ll, 'tcx>( enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val, ), &Variants::Multiple { - tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, untagged_variant }, - tag, + //tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, untagged_variant, .. }, + //tag, .. } => { - if variant_index == untagged_variant { - let valid_range = enum_type_and_layout - .for_variant(cx, variant_index) - .largest_niche - .as_ref() - .unwrap() - .valid_range; - - let min = valid_range.start.min(valid_range.end); - let min = tag.size(cx).truncate(min); - - let max = valid_range.start.max(valid_range.end); - let max = tag.size(cx).truncate(max); - - DiscrResult::Range(min, max) - } else { - let value = (variant_index.as_u32() as u128) - .wrapping_sub(niche_variants.start().as_u32() as u128) - .wrapping_add(niche_start); - let value = tag.size(cx).truncate(value); - DiscrResult::Value(value) - } + // YYY + DiscrResult::Range(0, 1) + //if variant_index == untagged_variant { + // let valid_range = enum_type_and_layout + // .for_variant(cx, variant_index) + // .largest_niche + // .as_ref() + // .unwrap() + // .valid_range; + + // let min = valid_range.start.min(valid_range.end); + // let min = tag.size(cx).truncate(min); + + // let max = valid_range.start.max(valid_range.end); + // let max = tag.size(cx).truncate(max); + + // DiscrResult::Range(min, max) + //} else { + // let value = (variant_index.as_u32() as u128) + // .wrapping_sub(niche_variants.start().as_u32() as u128) + // .wrapping_add(niche_start); + // let value = tag.size(cx).truncate(value); + // DiscrResult::Value(value) + //} } } } diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index 9c18df5643f1c..ea554805ccd2f 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -1,7 +1,7 @@ use super::operand::OperandValue; use super::{FunctionCx, LocalRef}; -use crate::common::IntPredicate; +use crate::common::{IntPredicate, TypeKind}; use crate::glue; use crate::traits::*; @@ -227,13 +227,13 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { } }; - // Read the tag/niche-encoded discriminant from memory. - let tag = self.project_field(bx, tag_field); - let tag = bx.load_operand(tag); + let tag_place = self.project_field(bx, tag_field); // Decode the discriminant (specifically if it's niche-encoded). match *tag_encoding { TagEncoding::Direct => { + // Read the tag from memory. + let tag = bx.load_operand(tag_place); let signed = match tag_scalar.primitive() { // We use `i1` for bytes that are always `0` or `1`, // e.g., `#[repr(i8)] enum E { A, B }`, but we can't @@ -244,11 +244,30 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { }; bx.intcast(tag.immediate(), cast_to, signed) } - TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => { - // Rebase from niche values to discriminants, and check - // whether the result is in range for the niche variants. - let niche_llty = bx.cx().immediate_backend_type(tag.layout); - let tag = tag.immediate(); + TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start, ref flag, } => { + let read = |bx: &mut Bx, place: Self| -> (V, ::Type) { + let ty = bx.cx().immediate_backend_type(place.layout); + let op = bx.load_operand(place); + let val = op.immediate(); + if bx.cx().type_kind(ty) == TypeKind::Pointer { + let new_ty = bx.cx().type_isize(); + let new_val = bx.ptrtoint(val, new_ty); + (new_val, new_ty) + } else { + (val, ty) + } + }; + + let (tag, niche_llty) = read(bx, tag_place); + + let (untagged_in_niche, flag_eq_magic_value_opt) = if let Some(flag) = flag { + let flag_place = self.project_field(bx, flag.field); + let (flag_imm, flag_llty) = read(bx, flag_place); + let magic_value = bx.cx().const_uint_big(flag_llty, flag.magic_value); + (flag.untagged_in_niche, Some(bx.icmp(IntPredicate::IntEQ, flag_imm, magic_value))) + } else { + (true, None) + }; // We first compute the "relative discriminant" (wrt `niche_variants`), // that is, if `n = niche_variants.end() - niche_variants.start()`, @@ -259,23 +278,8 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { // and check that it is in the range `niche_variants`, because // that might not fit in the same type, on top of needing an extra // comparison (see also the comment on `let niche_discr`). - let relative_discr = if niche_start == 0 { - // Avoid subtracting `0`, which wouldn't work for pointers. - // FIXME(eddyb) check the actual primitive type here. - tag - } else { - bx.sub(tag, bx.cx().const_uint_big(niche_llty, niche_start)) - }; + let relative_discr = bx.sub(tag, bx.cx().const_uint_big(niche_llty, niche_start)); let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32(); - let is_niche = if relative_max == 0 { - // Avoid calling `const_uint`, which wouldn't work for pointers. - // Also use canonical == 0 instead of non-canonical u<= 0. - // FIXME(eddyb) check the actual primitive type here. - bx.icmp(IntPredicate::IntEQ, relative_discr, bx.cx().const_null(niche_llty)) - } else { - let relative_max = bx.cx().const_uint(niche_llty, relative_max as u64); - bx.icmp(IntPredicate::IntULE, relative_discr, relative_max) - }; // NOTE(eddyb) this addition needs to be performed on the final // type, in case the niche itself can't represent all variant @@ -285,7 +289,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { // In other words, `niche_variants.end - niche_variants.start` // is representable in the niche, but `niche_variants.end` // might not be, in extreme cases. - let niche_discr = { + let potential_niche_discr = { let relative_discr = if relative_max == 0 { // HACK(eddyb) since we have only one niche, we know which // one it is, and we can avoid having a dynamic value here. @@ -299,11 +303,29 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { ) }; - bx.select( - is_niche, - niche_discr, - bx.cx().const_uint(cast_to, untagged_variant.as_u32() as u64), - ) + let untagged_discr = bx.cx().const_uint(cast_to, untagged_variant.as_u32() as u64); + + let niche_discr = if untagged_in_niche { + let relative_max_const = bx.cx().const_uint(niche_llty, relative_max as u64); + let is_niche = bx.icmp(IntPredicate::IntULE, relative_discr, relative_max_const); + bx.select( + is_niche, + potential_niche_discr, + untagged_discr, + ) + } else { + potential_niche_discr + }; + + if let Some(flag_eq_magic_value) = flag_eq_magic_value_opt { + bx.select( + flag_eq_magic_value, + niche_discr, + untagged_discr, + ) + } else { + niche_discr + } } } } @@ -337,23 +359,30 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { } Variants::Multiple { tag_encoding: - TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start }, + TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start, ref flag, }, tag_field, .. } => { + let store = |bx: &mut Bx, value: u128, place: Self| { + let ty = bx.cx().immediate_backend_type(place.layout); + let val = if bx.cx().type_kind(ty) == TypeKind::Pointer { + let ty_isize = bx.cx().type_isize(); + let llvalue = bx.cx().const_uint_big(ty_isize, value); + bx.inttoptr(llvalue, ty) + } else { + bx.cx().const_uint_big(ty, value) + }; + OperandValue::Immediate(val).store(bx, place); + }; if variant_index != untagged_variant { + if let Some(flag) = flag { + let place = self.project_field(bx, flag.field); + store(bx, flag.magic_value, place); + } let niche = self.project_field(bx, tag_field); - let niche_llty = bx.cx().immediate_backend_type(niche.layout); let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); let niche_value = (niche_value as u128).wrapping_add(niche_start); - // FIXME(eddyb): check the actual primitive type here. - let niche_llval = if niche_value == 0 { - // HACK(eddyb): using `c_null` as it works on all types. - bx.cx().const_null(niche_llty) - } else { - bx.cx().const_uint_big(niche_llty, niche_value) - }; - OperandValue::Immediate(niche_llval).store(bx, niche); + store(bx, niche_value, niche); } } } diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 510adde62962b..8168c50de5a2c 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -722,27 +722,55 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Return the cast value, and the index. (discr_val, index.0) } - TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => { + TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start, ref flag, } => { + let is_magic_val = if let Some(flag) = flag { + let flag_val = self.read_immediate(&self.operand_field(op, flag.field)?)?; + let flag_val = flag_val.to_scalar(); + match flag_val.try_to_int() { + Err(dbg_val) => { + // So this is a pointer then, and casting to an int + // failed. Can only happen during CTFE. If the magic + // value is 0 and the scalar is not null, we know + // the pointer cannot be the magic value. Anything + // else we conservatively reject. + let ptr_definitely_not_magic_value = + flag.magic_value == 0 && !self.scalar_may_be_null(flag_val)?; + if !ptr_definitely_not_magic_value { + throw_ub!(InvalidTag(dbg_val)) + } + false + } + Ok(flag_bits) => { + let flag_layout = + self.layout_of(flag.scalar.primitive().to_int_ty(*self.tcx))?; + let flag_bits = flag_bits.assert_bits(flag_layout.size); + flag_bits == flag.magic_value + } + } + } else { + true + }; let tag_val = tag_val.to_scalar(); // Compute the variant this niche value/"tag" corresponds to. With niche layout, // discriminant (encoded in niche/tag) and variant index are the same. let variants_start = niche_variants.start().as_u32(); let variants_end = niche_variants.end().as_u32(); - let variant = match tag_val.try_to_int() { - Err(dbg_val) => { + let variant = match (is_magic_val, tag_val.try_to_int()) { + (false, _) => untagged_variant, + (true, Err(dbg_val)) => { // So this is a pointer then, and casting to an int failed. // Can only happen during CTFE. // The niche must be just 0, and the ptr not null, then we know this is // okay. Everything else, we conservatively reject. - let ptr_valid = niche_start == 0 + let ptr_definitely_not_in_niche_variants = niche_start == 0 && variants_start == variants_end && !self.scalar_may_be_null(tag_val)?; - if !ptr_valid { + if !ptr_definitely_not_in_niche_variants { throw_ub!(InvalidTag(dbg_val)) } untagged_variant } - Ok(tag_bits) => { + (true, Ok(tag_bits)) => { let tag_bits = tag_bits.assert_bits(tag_layout.size); // We need to use machine arithmetic to get the relative variant idx: // variant_index_relative = tag_val - niche_start_val @@ -791,6 +819,8 @@ mod size_asserts { // These are in alphabetical order, which is easy to maintain. static_assert_size!(Immediate, 48); static_assert_size!(ImmTy<'_>, 64); - static_assert_size!(Operand, 56); - static_assert_size!(OpTy<'_>, 80); + #[cfg(not(bootstrap))] + static_assert_size!(Operand, 48); + #[cfg(not(bootstrap))] + static_assert_size!(OpTy<'_>, 72); } diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index eeeb7d6d3e5cc..74777acc80390 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -823,15 +823,22 @@ where } abi::Variants::Multiple { tag_encoding: - TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start }, + TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start, ref flag, }, tag: tag_layout, tag_field, .. } => { - // No need to validate that the discriminant here because the + // No need to validate the discriminant here because the // `TyAndLayout::for_variant()` call earlier already checks the variant is valid. if variant_index != untagged_variant { + if let Some(flag) = flag { + let flag_layout = self.layout_of(flag.scalar.primitive().to_int_ty(*self.tcx))?; + let val = ImmTy::from_uint(flag.magic_value, flag_layout); + let flag_dest = self.place_field(dest, flag.field)?; + self.write_immediate(*val, &flag_dest)?; + } + let variants_start = niche_variants.start().as_u32(); let variant_index_relative = variant_index .as_u32() diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index d4146c2424101..893187c37d8e5 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -16,7 +16,7 @@ use rustc_middle::ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_span::symbol::{sym, Symbol}; use rustc_span::DUMMY_SP; -use rustc_target::abi::{Abi, Scalar as ScalarAbi, Size, VariantIdx, Variants, WrappingRange}; +use rustc_target::abi::{Abi, Scalar as ScalarAbi, Size, TagEncoding, VariantIdx, Variants, WrappingRange}; use std::hash::Hash; @@ -220,8 +220,18 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' fn aggregate_field_path_elem(&mut self, layout: TyAndLayout<'tcx>, field: usize) -> PathElem { // First, check if we are projecting to a variant. match layout.variants { - Variants::Multiple { tag_field, .. } => { - if tag_field == field { + Variants::Multiple { tag_field, ref tag_encoding, .. } => { + let flag_field = if let TagEncoding::Niche { + flag: Some(f), + .. + } = tag_encoding { + Some(f.field) + } else { + None + }; + // For now, let's just report both tag and flag fields as being + // part of the tag. + if tag_field == field || flag_field == Some(field) { return match layout.ty.kind() { ty::Adt(def, ..) if def.is_enum() => PathElem::EnumTag, ty::Generator(..) => PathElem::GeneratorTag, diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index 922ce738dbb15..405d03b325431 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -3520,8 +3520,10 @@ mod size_asserts { static_assert_size!(Expr<'_>, 64); static_assert_size!(ExprKind<'_>, 48); static_assert_size!(FnDecl<'_>, 40); - static_assert_size!(ForeignItem<'_>, 72); - static_assert_size!(ForeignItemKind<'_>, 40); + #[cfg(not(bootstrap))] + static_assert_size!(ForeignItem<'_>, 64); + #[cfg(not(bootstrap))] + static_assert_size!(ForeignItemKind<'_>, 32); static_assert_size!(GenericArg<'_>, 24); static_assert_size!(GenericBound<'_>, 48); static_assert_size!(Generics<'_>, 56); @@ -3540,8 +3542,10 @@ mod size_asserts { static_assert_size!(Res, 12); static_assert_size!(Stmt<'_>, 32); static_assert_size!(StmtKind<'_>, 16); - static_assert_size!(TraitItem<'_>, 88); - static_assert_size!(TraitItemKind<'_>, 48); + #[cfg(not(bootstrap))] + static_assert_size!(TraitItem<'_>, 80); + #[cfg(not(bootstrap))] + static_assert_size!(TraitItemKind<'_>, 40); static_assert_size!(Ty<'_>, 48); static_assert_size!(TyKind<'_>, 32); } diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index 5f8729a8ddf3c..0fcb5a6c0b1f4 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -585,7 +585,7 @@ where None => FieldsShape::Arbitrary { offsets: vec![], memory_index: vec![] }, }, abi: Abi::Uninhabited, - largest_niche: None, + niches: Vec::new(), align: tcx.data_layout.i8_align, size: Size::ZERO, }) @@ -722,9 +722,25 @@ where } // Discriminant field for enums (where applicable). - Variants::Multiple { tag, .. } => { - assert_eq!(i, 0); - return TyMaybeWithLayout::TyAndLayout(tag_layout(tag)); + Variants::Multiple { tag, ref tag_encoding, .. } => { + if i == 0 { + if let TagEncoding::Niche { flag: Some(_), .. } = tag_encoding { + // There's a flag, which means the niche may not correspond to + // any field of any variant. + let prim = tag.primitive(); + let size = prim.size(cx); + let range = WrappingRange::full(size); + TyMaybeWithLayout::TyAndLayout(tag_layout( + Scalar::Initialized { value: prim, valid_range: range }, + )) + } else { + TyMaybeWithLayout::TyAndLayout(tag_layout(tag)) + } + } else { + assert_eq!(i, 1); + let TagEncoding::Niche { flag: Some(Flag {scalar, .. }), .. } = tag_encoding else { bug!("request field 1 for an enum with no flag") }; + TyMaybeWithLayout::TyAndLayout(tag_layout(*scalar)) + } } } } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 753d7bffe84c2..36b32a11aaa37 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1942,11 +1942,12 @@ bitflags! { const IS_C = 1 << 0; const IS_SIMD = 1 << 1; const IS_TRANSPARENT = 1 << 2; + const USE_FLAG = 1 << 3; // Internal only for now. If true, don't reorder fields. - const IS_LINEAR = 1 << 3; + const IS_LINEAR = 1 << 4; // If true, the type's layout can be randomized using // the seed stored in `ReprOptions.layout_seed` - const RANDOMIZE_LAYOUT = 1 << 4; + const RANDOMIZE_LAYOUT = 1 << 5; // Any of these flags being set prevent field reordering optimisation. const IS_UNOPTIMISABLE = ReprFlags::IS_C.bits | ReprFlags::IS_SIMD.bits @@ -2004,6 +2005,7 @@ impl ReprOptions { } attr::ReprTransparent => ReprFlags::IS_TRANSPARENT, attr::ReprSimd => ReprFlags::IS_SIMD, + attr::ReprFlag => ReprFlags::USE_FLAG, attr::ReprInt(i) => { size = Some(i); ReprFlags::empty() @@ -2050,6 +2052,11 @@ impl ReprOptions { self.flags.contains(ReprFlags::IS_TRANSPARENT) } + #[inline] + pub fn use_flag(&self) -> bool { + self.flags.contains(ReprFlags::USE_FLAG) + } + #[inline] pub fn linear(&self) -> bool { self.flags.contains(ReprFlags::IS_LINEAR) diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 87433538512b9..bd65bd88e4375 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -1651,6 +1651,12 @@ impl CheckAttrVisitor<'_> { _ => ("a", "struct, enum, or union"), } } + sym::flag => { + match target { + Target::Enum => continue, + _ => ("an", "enum"), + } + } sym::i8 | sym::u8 | sym::i16 diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 8cb7d147d0253..7d3f9b291b949 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -723,6 +723,7 @@ symbols! { file_macro, fill, finish, + flag, flags, float, float_to_int_unchecked, diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs index 7171ca7bf8950..df7d8d1f9b289 100644 --- a/compiler/rustc_target/src/abi/mod.rs +++ b/compiler/rustc_target/src/abi/mod.rs @@ -924,7 +924,7 @@ impl Scalar { } /// Describes how the fields of a type are located in memory. -#[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] +#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] pub enum FieldsShape { /// Scalar primitives and `!`, which never have fields. Primitive, @@ -1114,7 +1114,7 @@ rustc_index::newtype_index! { } } -#[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] +#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] pub enum Variants<'a> { /// Single enum variants, structs/tuples, unions, and all non-ADTs. Single { index: VariantIdx }, @@ -1133,7 +1133,15 @@ pub enum Variants<'a> { }, } -#[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] +#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub struct Flag { + pub magic_value: u128, + pub scalar: Scalar, + pub field: usize, + pub untagged_in_niche: bool, +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] pub enum TagEncoding { /// The tag directly stores the discriminant, but possibly with a smaller layout /// (so converting the tag to the discriminant can require sign extension). @@ -1153,25 +1161,63 @@ pub enum TagEncoding { untagged_variant: VariantIdx, niche_variants: RangeInclusive, niche_start: u128, + flag: Option, }, } #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)] -pub struct Niche { +pub struct NicheData { pub offset: Size, pub value: Primitive, pub valid_range: WrappingRange, } +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +pub struct Niche { + pub data: NicheData, + pub flag: Option, +} + impl Niche { pub fn from_scalar(cx: &C, offset: Size, scalar: Scalar) -> Option { let Scalar::Initialized { value, valid_range } = scalar else { return None }; - let niche = Niche { offset, value, valid_range }; + let data = NicheData { offset, value, valid_range }; + let niche = Niche { data, flag: None }; if niche.available(cx) > 0 { Some(niche) } else { None } } + pub fn from_flag_and_scalar( + cx: &C, + flag_offset: Size, + flag_scalar: Scalar, + niche_offset: Size, + niche_scalar: Scalar, + ) -> Option { + let niche_data = |offset, scalar| { + let Scalar::Initialized {value, valid_range } = scalar else { return None }; + Some(NicheData { + offset, + value, + valid_range, + }) + }; + + let niche = Niche { + data: niche_data(niche_offset, niche_scalar)?, + flag: Some( + niche_data(flag_offset, flag_scalar)?, + ) + }; + + if niche.available(cx) > 0 || niche.available_with_new_flag(cx) > 0 { + Some(niche) + } else { + None + } + } + pub fn available(&self, cx: &C) -> u128 { - let Self { value, valid_range: v, .. } = *self; + let NicheData { value, valid_range: v, .. } = self.data; let size = value.size(cx); assert!(size.bits() <= 128); let max_value = size.unsigned_int_max(); @@ -1181,10 +1227,65 @@ impl Niche { niche.end.wrapping_sub(niche.start) & max_value } + pub fn available_with_new_flag(&self, cx: &C) -> u128 { + self.flag.map_or(0, |flag_data| { + let valid_range = flag_data.valid_range; + let size = flag_data.value.size(cx); + assert!(size.bits() <= 128); + let max_value = size.unsigned_int_max(); + if valid_range.end.wrapping_sub(valid_range.start) < max_value { + let niche_size = self.data.value.size(cx); + assert!(niche_size.bits() <= 128); + let max = size.unsigned_int_max(); + if max < u128::MAX { + max + 1 + } else { + max + } + } else { + 0 + } + }) + } + + pub fn reserve_with_new_flag(&self, cx: &C, count: u128) -> Option<(u128, Scalar, Scalar)> { + assert!(count > 0); + + self.flag.map_or(None, |flag_data| { + let size = flag_data.value.size(cx); + assert!(size.bits() <= 128); + let max_value = size.unsigned_int_max(); + let valid_range = flag_data.valid_range; + if valid_range.end.wrapping_sub(valid_range.start) >= max_value { + return None; + } + + let new_magic_value = valid_range.end.wrapping_add(1) & max_value; + let niche_size = self.data.value.size(cx); + let niche_max_value = niche_size.unsigned_int_max(); + if count - 1 > niche_max_value { + return None; + } + + let flag_scalar = Scalar::Initialized { + value: flag_data.value, + valid_range: valid_range.with_end(new_magic_value), + }; + let niche_scalar = Scalar::Initialized { + value: self.data.value, + // YYY + //valid_range: WrappingRange::full(self.data.value.size(cx)), + valid_range: WrappingRange { start: 0, end: count - 1 }, + }; + + Some((new_magic_value, flag_scalar, niche_scalar)) + }) + } + pub fn reserve(&self, cx: &C, count: u128) -> Option<(u128, Scalar)> { assert!(count > 0); - let Self { value, valid_range: v, .. } = *self; + let NicheData { value, valid_range: v, .. } = self.data; let size = value.size(cx); assert!(size.bits() <= 128); let max_value = size.unsigned_int_max(); @@ -1239,7 +1340,7 @@ impl Niche { } } -#[derive(PartialEq, Eq, Hash, HashStable_Generic)] +#[derive(Clone, PartialEq, Eq, Hash, HashStable_Generic)] pub struct LayoutS<'a> { /// Says where the fields are located within the layout. pub fields: FieldsShape, @@ -1262,9 +1363,8 @@ pub struct LayoutS<'a> { /// have to be taken into account to find all fields of this layout. pub abi: Abi, - /// The leaf scalar with the largest number of invalid values - /// (i.e. outside of its `valid_range`), if it exists. - pub largest_niche: Option, + /// Leaf scalars with invalid values. + pub niches: Vec, pub align: AbiAndPrefAlign, pub size: Size, @@ -1272,14 +1372,14 @@ pub struct LayoutS<'a> { impl<'a> LayoutS<'a> { pub fn scalar(cx: &C, scalar: Scalar) -> Self { - let largest_niche = Niche::from_scalar(cx, Size::ZERO, scalar); + let niches = Niche::from_scalar(cx, Size::ZERO, scalar).into_iter().collect(); let size = scalar.size(cx); let align = scalar.align(cx); LayoutS { variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Primitive, abi: Abi::Scalar(scalar), - largest_niche, + niches, size, align, } @@ -1291,13 +1391,13 @@ impl<'a> fmt::Debug for LayoutS<'a> { // This is how `Layout` used to print before it become // `Interned`. We print it like this to avoid having to update // expected output in a lot of tests. - let LayoutS { size, align, abi, fields, largest_niche, variants } = self; + let LayoutS { size, align, abi, fields, niches, variants } = self; f.debug_struct("Layout") .field("size", size) .field("align", align) .field("abi", abi) .field("fields", fields) - .field("largest_niche", largest_niche) + .field("niches", niches) .field("variants", variants) .finish() } @@ -1327,8 +1427,8 @@ impl<'a> Layout<'a> { self.0.0.abi } - pub fn largest_niche(self) -> Option { - self.0.0.largest_niche + pub fn niches(self) -> impl Iterator { + self.0.0.niches.iter() } pub fn align(self) -> AbiAndPrefAlign { diff --git a/compiler/rustc_ty_utils/Cargo.toml b/compiler/rustc_ty_utils/Cargo.toml index 5e4ba47306192..afce8a318d0a6 100644 --- a/compiler/rustc_ty_utils/Cargo.toml +++ b/compiler/rustc_ty_utils/Cargo.toml @@ -19,3 +19,4 @@ rustc_target = { path = "../rustc_target" } rustc_trait_selection = { path = "../rustc_trait_selection" } rustc_type_ir = { path = "../rustc_type_ir" } rustc_index = { path = "../rustc_index" } +smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 345911f4309ae..c1c18c5b35066 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -13,7 +13,7 @@ use rustc_span::symbol::Symbol; use rustc_span::DUMMY_SP; use rustc_target::abi::*; -use std::cmp::{self, Ordering}; +use std::cmp; use std::iter; use std::num::NonZeroUsize; use std::ops::Bound; @@ -21,6 +21,8 @@ use std::ops::Bound; use rand::{seq::SliceRandom, SeedableRng}; use rand_xoshiro::Xoshiro128StarStar; +use smallvec::SmallVec; + use crate::layout_sanity_check::sanity_check_layout; pub fn provide(providers: &mut ty::query::Providers) { @@ -96,12 +98,10 @@ fn scalar_pair<'tcx>(cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, a: Scalar, b: Scalar) -> let b_offset = a.size(dl).align_to(b_align.abi); let size = (b_offset + b.size(dl)).align_to(align.abi); - // HACK(nox): We iter on `b` and then `a` because `max_by_key` - // returns the last maximum. - let largest_niche = Niche::from_scalar(dl, b_offset, b) + let niches: Vec = Niche::from_scalar(dl, Size::ZERO, a) .into_iter() - .chain(Niche::from_scalar(dl, Size::ZERO, a)) - .max_by_key(|niche| niche.available(dl)); + .chain(Niche::from_scalar(dl, b_offset, b).into_iter()) + .collect(); LayoutS { variants: Variants::Single { index: VariantIdx::new(0) }, @@ -110,7 +110,7 @@ fn scalar_pair<'tcx>(cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, a: Scalar, b: Scalar) -> memory_index: vec![0, 1], }, abi: Abi::ScalarPair(a, b), - largest_niche, + niches, align, size, } @@ -187,8 +187,7 @@ fn univariant_uninterned<'tcx>( let mut sized = true; let mut offsets = vec![Size::ZERO; fields.len()]; let mut offset = Size::ZERO; - let mut largest_niche = None; - let mut largest_niche_available = 0; + let mut niches = Vec::new(); if let StructKind::Prefixed(prefix_size, prefix_align) = kind { let prefix_align = @@ -226,16 +225,25 @@ fn univariant_uninterned<'tcx>( debug!("univariant offset: {:?} field: {:#?}", offset, field); offsets[i as usize] = offset; - if let Some(mut niche) = field.largest_niche { - let available = niche.available(dl); - if available > largest_niche_available { - largest_niche_available = available; - niche.offset += offset; - largest_niche = Some(niche); + niches.extend(field.niches.iter().cloned().map(|mut niche| { + niche.data.offset += offset; + if let Some(ref mut data) = niche.flag.as_mut() { + data.offset += offset; } - } + niche + })); offset = offset.checked_add(field.size, dl).ok_or(LayoutError::SizeOverflow(ty))?; + + // Niche with most available values first. + niches.sort_by_key(|niche| { + (niche.flag.is_some(), !niche.available(dl), niche.data.offset.bytes()) + }); + + // We may accumulate many niches as types nest. For performance we don't + // keep all of them around. (I haven't done any real benchmarking of + // this yet, and 4 is arbitrary.) + niches.truncate(4); } if let Some(repr_align) = repr.align { @@ -328,12 +336,324 @@ fn univariant_uninterned<'tcx>( variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Arbitrary { offsets, memory_index }, abi, - largest_niche, + niches, align, size, }) } +#[derive(Clone, Copy)] +struct UsedFlag { + magic_value: u128, + scalar: Scalar, + offset: Size, + untagged_in_niche: bool, +} + +#[derive(Clone, Copy)] +struct UsedNiche { + start: u128, + scalar: Scalar, + offset: Size, + flag: Option, +} + +fn fit_layouts_around( + variant_layouts: &mut IndexVec>, + variants: &IndexVec>>, + largest_variant_index: VariantIdx, + niche1: (Size, Size), // (offset, size) + niche2: (Size, Size), + size: Size, +) -> bool { + enum PastNicheStatus { + PastNone, + Past1, + PastBoth, + } + use PastNicheStatus::*; + + variant_layouts.iter_enumerated_mut().all(|(i, layout)| { + if i == largest_variant_index { + return true; + } + + layout.niches = Vec::new(); + + if layout.size <= niche1.0 { + // This variant will fit before the niches. + return true; + } + + // Put as many fields as possible before the niche, + // then try to fill in the rest after. + let fields = &variants[i]; + let ordered_indices: SmallVec<[usize; 8]> = + layout.fields.index_by_increasing_offset().collect(); + let FieldsShape::Arbitrary { ref mut offsets, .. } = layout.fields else { bug!() }; + let mut next = Size::ZERO; + let mut past_niche = PastNone; + for j in ordered_indices.iter().cloned() { + let offset = &mut offsets[j]; + let field_size = fields[j].layout.size(); + let field_align = fields[j].layout.align().abi; + + match past_niche { + PastBoth => *offset = next.align_to(field_align), + Past1 => { + *offset = next.align_to(field_align); + if *offset + field_size > niche2.0 { + *offset = (niche2.0 + niche2.1).align_to(field_align); + past_niche = PastBoth; + } + } + PastNone => { + if *offset + field_size > niche1.0 { + *offset = (niche1.0 + niche1.1).align_to(field_align); + past_niche = Past1; + if *offset + field_size > niche2.0 { + *offset = (niche2.0 + niche2.1).align_to(field_align); + past_niche = PastBoth; + } + } + } + } + next = *offset + field_size; + if next > size { + return false; + } + } + layout.size = next.align_to(layout.align.abi); + + // It can't be a Scalar or ScalarPair. + if !layout.abi.is_uninhabited() { + layout.abi = Abi::Aggregate { sized: true }; + } + + true + }) +} + +struct NicheFinder<'a, 'tcx> { + dl: &'a TargetDataLayout, + variant_layouts: &'a mut IndexVec>, + variants: &'a IndexVec>>, + largest_variant_index: VariantIdx, + size: Size, + align: Align, + count: u128, + backup_variant_layouts: &'a IndexVec>, + niches: &'a [Niche], +} + +impl<'a, 'tcx> NicheFinder<'a, 'tcx> { + fn find_niche_noflag(&mut self) -> Option { + for niche in self.niches.iter() { + let used_niche = match niche.reserve(self.dl, self.count) { + Some((start, scalar)) => UsedNiche { + start, + scalar, + offset: niche.data.offset, + flag: niche.flag.map(|f| UsedFlag { + magic_value: f.valid_range.end, + scalar: Scalar::Initialized { value: f.value, valid_range: f.valid_range }, + offset: f.offset, + untagged_in_niche: true, + }), + }, + None => match niche.reserve_with_new_flag(self.dl, self.count) { + Some((magic_value, flag_scalar, niche_scalar)) => UsedNiche { + start: 0, + scalar: niche_scalar, + offset: niche.data.offset, + flag: Some(UsedFlag { + magic_value, + scalar: flag_scalar, + offset: niche.flag.unwrap().offset, + untagged_in_niche: false, + }), + }, + None => continue, + }, + }; + + let niche_size = used_niche.scalar.size(self.dl); + let (offset1, size1, offset2, size2) = match used_niche.flag { + None => (used_niche.offset, niche_size, used_niche.offset, niche_size), + Some(f) => { + let flag_size = f.scalar.size(self.dl); + if used_niche.offset < f.offset { + (used_niche.offset, niche_size, f.offset, flag_size) + } else { + (f.offset, flag_size, used_niche.offset, niche_size) + } + } + }; + + if fit_layouts_around( + self.variant_layouts, + self.variants, + self.largest_variant_index, + (offset1, size1), + (offset2, size2), + self.size, + ) { + return Some(used_niche); + } + + self.variant_layouts.clone_from(self.backup_variant_layouts); + } + None + } + + fn try_flag( + &mut self, + flag_data: &NicheData, + niche_primitive: Primitive, + niche_offset: Size, + ) -> Option { + if !niche_primitive.is_int() && !niche_primitive.is_ptr() { + return None; + } + + let flag_primitive: Primitive = flag_data.value; + let flag_offset: Size = flag_data.offset; + let flag_valid_range: WrappingRange = flag_data.valid_range; + let niche_size = niche_primitive.size(self.dl); + let niche_max = niche_size.unsigned_int_max(); + + if niche_max < self.count - 1 { + return None; + } + + let flag_size = flag_primitive.size(self.dl); + let flag_max = flag_size.unsigned_int_max(); + let magic_value = flag_valid_range.end.wrapping_add(1) & flag_max; + + let (offset1, size1, offset2, size2) = if flag_offset < niche_offset { + (flag_offset, flag_size, niche_offset, niche_size) + } else { + (niche_offset, niche_size, flag_offset, flag_size) + }; + + if fit_layouts_around( + self.variant_layouts, + self.variants, + self.largest_variant_index, + (offset1, size1), + (offset2, size2), + self.size, + ) { + Some(UsedNiche { + start: 0, + scalar: Scalar::Initialized { + value: niche_primitive, + valid_range: WrappingRange { start: 0, end: self.count - 1 }, + }, + offset: niche_offset, + flag: Some(UsedFlag { + magic_value, + scalar: Scalar::Initialized { + value: flag_primitive, + valid_range: flag_valid_range.with_end(magic_value), + }, + offset: flag_offset, + untagged_in_niche: false, + }), + }) + } else { + self.variant_layouts.clone_from(self.backup_variant_layouts); + None + } + } + + fn try_offset(&mut self, flag_data: &NicheData, offset: Size) -> Option { + let u16_prim = Primitive::Int(Integer::I16, false); + let u16_size = u16_prim.size(self.dl); + let u16_align = u16_prim.align(self.dl).abi; + let u8_prim = Primitive::Int(Integer::I8, false); + let u8_size = u8_prim.size(self.dl); + + let flag_offset = flag_data.offset; + let flag_size = flag_data.value.size(self.dl); + + // Try a u16. + if offset.is_aligned(u16_align) + && self.align >= u16_align + && offset + u16_size <= self.size + && (flag_offset + flag_size <= offset || offset + u16_size <= flag_offset) + { + let opt_niche = self.try_flag(&flag_data, u16_prim, offset); + if opt_niche.is_some() { + return opt_niche; + } + } + + // Try a u8. + if offset + u8_size <= self.size + && (flag_offset + flag_size <= offset || offset + u8_size <= flag_offset) + { + self.try_flag(&flag_data, u8_prim, offset) + } else { + None + } + } + + fn find_niche_flag(&mut self) -> Option { + for flag_niche in self.niches.iter() { + if flag_niche.flag.is_some() { + continue; + } + + // If the untagged variant uses the ScalarPair ABI, try to preserve + // that. + if let Abi::ScalarPair(first, second) = + self.variant_layouts[self.largest_variant_index].abi + { + let (other_offset, other_scalar) = if flag_niche.data.offset == Size::ZERO { + (first.size(self.dl).align_to(second.align(self.dl).abi), second) + } else { + (Size::ZERO, first) + }; + + let opt_niche = + self.try_flag(&flag_niche.data, other_scalar.primitive(), other_offset); + if opt_niche.is_some() { + return opt_niche; + } + } + + // Otherwise, we don't even need to pay attention to the fields of + // the variant; just avoid the flag. There seems to be little point + // in looking for anything larger than a u16 niche, since + // we will be able to use all its values. + + let flag_size = flag_niche.data.value.size(self.dl); + let u16_prim = Primitive::Int(Integer::I16, false); + let u16_align = u16_prim.align(self.dl).abi; + let potential_niche_offsets = [ + flag_niche.data.offset + flag_size, + (flag_niche.data.offset + flag_size).align_to(u16_align), + Size::from_bytes(self.size.bytes() - 1), + Size::from_bytes(self.size.bytes().max(2) - 2), + Size::ZERO, + ]; + + for (i, potential_niche_offset) in potential_niche_offsets.iter().cloned().enumerate() { + if potential_niche_offsets[..i].iter().any(|&off| off == potential_niche_offset) { + continue; + } + let opt_niche = self.try_offset(&flag_niche.data, potential_niche_offset); + if opt_niche.is_some() { + return opt_niche; + } + } + } + + None + } +} + fn layout_of_uncached<'tcx>( cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, ty: Ty<'tcx>, @@ -386,7 +706,7 @@ fn layout_of_uncached<'tcx>( variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Primitive, abi: Abi::Uninhabited, - largest_niche: None, + niches: Vec::new(), align: dl.i8_align, size: Size::ZERO, }), @@ -449,13 +769,14 @@ fn layout_of_uncached<'tcx>( Abi::Aggregate { sized: true } }; - let largest_niche = if count != 0 { element.largest_niche } else { None }; + // YYY - this is an arbitrary choice + let niches = if count != 0 { element.niches.clone() } else { Vec::new() }; tcx.intern_layout(LayoutS { variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Array { stride: element.size, count }, abi, - largest_niche, + niches, align: element.align, size, }) @@ -466,7 +787,7 @@ fn layout_of_uncached<'tcx>( variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Array { stride: element.size, count: 0 }, abi: Abi::Aggregate { sized: false }, - largest_niche: None, + niches: Vec::new(), align: element.align, size: Size::ZERO, }) @@ -475,7 +796,7 @@ fn layout_of_uncached<'tcx>( variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Array { stride: Size::from_bytes(1), count: 0 }, abi: Abi::Aggregate { sized: false }, - largest_niche: None, + niches: Vec::new(), align: dl.i8_align, size: Size::ZERO, }), @@ -627,7 +948,7 @@ fn layout_of_uncached<'tcx>( variants: Variants::Single { index: VariantIdx::new(0) }, fields, abi: Abi::Vector { element: e_abi, count: e_len }, - largest_niche: e_ly.largest_niche, + niches: e_ly.niches.clone(), size, align, }) @@ -707,7 +1028,7 @@ fn layout_of_uncached<'tcx>( NonZeroUsize::new(variants[index].len()).ok_or(LayoutError::Unknown(ty))?, ), abi, - largest_niche: None, + niches: Vec::new(), align, size: size.align_to(align.abi), })); @@ -781,7 +1102,7 @@ fn layout_of_uncached<'tcx>( Abi::Vector { element, count: _ } => hide_niches(element), Abi::Aggregate { sized: _ } => {} } - st.largest_niche = None; + st.niches = Vec::new(); return Ok(tcx.intern_layout(st)); } @@ -812,19 +1133,9 @@ fn layout_of_uncached<'tcx>( valid_range.end = end; } - // Update `largest_niche` if we have introduced a larger niche. - let niche = Niche::from_scalar(dl, Size::ZERO, *scalar); - if let Some(niche) = niche { - match st.largest_niche { - Some(largest_niche) => { - // Replace the existing niche even if they're equal, - // because this one is at a lower offset. - if largest_niche.available(dl) <= niche.available(dl) { - st.largest_niche = Some(niche); - } - } - None => st.largest_niche = Some(niche), - } + if st.niches.is_empty() { + st.niches = + Niche::from_scalar(dl, Size::ZERO, *scalar).into_iter().collect(); } } _ => assert!( @@ -872,6 +1183,7 @@ fn layout_of_uncached<'tcx>( } let mut align = dl.aggregate_align; + let mut size = Size::ZERO; let mut variant_layouts = variants .iter_enumerated() .map(|(j, v)| { @@ -885,96 +1197,101 @@ fn layout_of_uncached<'tcx>( st.variants = Variants::Single { index: j }; align = align.max(st.align); + size = size.max(st.size); Ok(st) }) .collect::, _>>()?; - let largest_variant_index = match variant_layouts + if size == Size::ZERO { + return Ok(None); + } + + // Find the indices of the variants of largest size. It + // might seem at first glance that we only want to find a + // unique largest variant, but actually niche optimization + // can still happen if multiple variants are the largest + // size, because some of that size may be padding and the + // fields can move. + let largest_variant_indices: SmallVec<[VariantIdx; 8]> = variant_layouts .iter_enumerated() - .max_by_key(|(_i, layout)| layout.size.bytes()) + .filter(|(_i, layout)| layout.size == size) .map(|(i, _layout)| i) - { - None => return Ok(None), - Some(i) => i, - }; + .collect(); - let all_indices = VariantIdx::new(0)..=VariantIdx::new(variants.len() - 1); - let needs_disc = |index: VariantIdx| { - index != largest_variant_index && !absent(&variants[index]) - }; - let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap() - ..=all_indices.rev().find(|v| needs_disc(*v)).unwrap(); + size = size.align_to(align.abi); + let mut largest_variant_index_found = None; + let mut used_niche_opt_found = None; + let mut niche_variants_found = None; - let count = niche_variants.size_hint().1.unwrap() as u128; - - // Find the field with the largest niche - let (field_index, niche, (niche_start, niche_scalar)) = match variants - [largest_variant_index] - .iter() - .enumerate() - .filter_map(|(j, field)| Some((j, field.largest_niche?))) - .max_by_key(|(_, niche)| niche.available(dl)) - .and_then(|(j, niche)| Some((j, niche, niche.reserve(cx, count)?))) - { - None => return Ok(None), - Some(x) => x, - }; - - let niche_offset = niche.offset - + variant_layouts[largest_variant_index].fields.offset(field_index); - let niche_size = niche.value.size(dl); - let size = variant_layouts[largest_variant_index].size.align_to(align.abi); - - let all_variants_fit = - variant_layouts.iter_enumerated_mut().all(|(i, layout)| { - if i == largest_variant_index { - return true; - } - - layout.largest_niche = None; - - if layout.size <= niche_offset { - // This variant will fit before the niche. - return true; - } - - // Determine if it'll fit after the niche. - let this_align = layout.align.abi; - let this_offset = (niche_offset + niche_size).align_to(this_align); + for largest_variant_index in largest_variant_indices.iter().cloned() { + let all_indices = VariantIdx::new(0)..=VariantIdx::new(variants.len() - 1); + let needs_disc = |index: VariantIdx| { + index != largest_variant_index && !absent(&variants[index]) + }; + let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap() + ..=all_indices.rev().find(|v| needs_disc(*v)).unwrap(); - if this_offset + layout.size > size { - return false; - } + let count = niche_variants.size_hint().1.unwrap() as u128; - // It'll fit, but we need to make some adjustments. - match layout.fields { - FieldsShape::Arbitrary { ref mut offsets, .. } => { - for (j, offset) in offsets.iter_mut().enumerate() { - if !variants[i][j].is_zst() { - *offset += this_offset; - } - } - } - _ => { - panic!("Layout of fields should be Arbitrary for variants") - } - } + let niches = + std::mem::take(&mut variant_layouts[largest_variant_index].niches); + if niches.is_empty() { + continue; + } - // It can't be a Scalar or ScalarPair because the offset isn't 0. - if !layout.abi.is_uninhabited() { - layout.abi = Abi::Aggregate { sized: true }; - } - layout.size += this_offset; + let backup_variant_layouts = variant_layouts.clone(); + let mut niche_finder = NicheFinder { + dl, + variant_layouts: &mut variant_layouts, + variants: &variants, + largest_variant_index, + size, + align: align.abi, + count, + backup_variant_layouts: &backup_variant_layouts, + niches: &niches, + }; - true - }); + let used_niche_opt = if def.repr().use_flag() { + niche_finder + .find_niche_flag() + .or_else(|| niche_finder.find_niche_noflag()) + } else { + niche_finder + .find_niche_noflag() + .or_else(|| niche_finder.find_niche_flag()) + }; - if !all_variants_fit { - return Ok(None); + if used_niche_opt.is_some() { + used_niche_opt_found = used_niche_opt; + largest_variant_index_found = Some(largest_variant_index); + niche_variants_found = Some(niche_variants); + break; + } } - let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar); + let Some(used_niche) = used_niche_opt_found else { return Ok(None) }; + let Some(largest_variant_index) = largest_variant_index_found else { return Ok(None) }; + let Some(niche_variants) = niche_variants_found else { return Ok(None) }; + + let niche_offset = used_niche.offset; + let niche_scalar = used_niche.scalar; + let niche_start = used_niche.start; + + let niches = if let Some(used_flag) = used_niche.flag { + Niche::from_flag_and_scalar( + dl, + used_flag.offset, + used_flag.scalar, + niche_offset, + niche_scalar, + ) + } else { + Niche::from_scalar(dl, niche_offset, niche_scalar) + } + .into_iter() + .collect(); let others_zst = variant_layouts .iter_enumerated() @@ -982,20 +1299,48 @@ fn layout_of_uncached<'tcx>( let same_size = size == variant_layouts[largest_variant_index].size; let same_align = align == variant_layouts[largest_variant_index].align; + // If we used a flag, the niche may not be a field of the + // largest variant, which matters if we're going to use the + // ScalarPair ABI. + let niche_is_a_field = |first: Scalar, second: Scalar| -> bool { + if used_niche.flag.is_none() { + return true; + } + + let other_primitive = if used_niche.offset == Size::ZERO { + first.primitive() + } else { + let offset = first.size(dl).align_to(second.align(dl).abi); + if offset != used_niche.offset { + return false; + } + second.primitive() + }; + + other_primitive == used_niche.scalar.primitive() + }; + let abi = if variant_layouts.iter().all(|v| v.abi.is_uninhabited()) { Abi::Uninhabited } else if same_size && same_align && others_zst { match variant_layouts[largest_variant_index].abi { // When the total alignment and size match, we can use the // same ABI as the scalar variant with the reserved niche. - Abi::Scalar(_) => Abi::Scalar(niche_scalar), - Abi::ScalarPair(first, second) => { - // Only the niche is guaranteed to be initialised, - // so use union layouts for the other primitive. - if niche_offset == Size::ZERO { - Abi::ScalarPair(niche_scalar, second.to_union()) + Abi::Scalar(_) if used_niche.flag.is_none() => { + Abi::Scalar(niche_scalar) + } + Abi::ScalarPair(first, second) if niche_is_a_field(first, second) => { + let (offset, scalar) = match used_niche.flag { + None => (niche_offset, niche_scalar), + Some(data) => (data.offset, data.scalar), + }; + // Only the niche (or the flag, if present) + // is guaranteed to be initialized, so use + // union layout for the other primitive. + if offset == Size::ZERO { + Abi::ScalarPair(scalar, second.to_union()) } else { - Abi::ScalarPair(first.to_union(), niche_scalar) + Abi::ScalarPair(first.to_union(), scalar) } } _ => Abi::Aggregate { sized: true }, @@ -1011,16 +1356,33 @@ fn layout_of_uncached<'tcx>( untagged_variant: largest_variant_index, niche_variants, niche_start, + flag: used_niche.flag.map(|f| Flag { + magic_value: f.magic_value, + scalar: f.scalar, + field: 1, + untagged_in_niche: f.untagged_in_niche, + }), }, tag_field: 0, variants: IndexVec::new(), }, - fields: FieldsShape::Arbitrary { - offsets: vec![niche_offset], - memory_index: vec![0], + fields: if let Some(f) = used_niche.flag { + FieldsShape::Arbitrary { + offsets: vec![niche_offset, f.offset], + memory_index: if niche_offset < f.offset { + vec![0, 1] + } else { + vec![1, 0] + }, + } + } else { + FieldsShape::Arbitrary { + offsets: vec![niche_offset], + memory_index: vec![0], + } }, abi, - largest_niche, + niches, size, align, }; @@ -1277,7 +1639,7 @@ fn layout_of_uncached<'tcx>( } } - let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag); + let niches = Niche::from_scalar(dl, Size::ZERO, tag).into_iter().collect(); let tagged_layout = LayoutS { variants: Variants::Multiple { @@ -1287,7 +1649,7 @@ fn layout_of_uncached<'tcx>( variants: IndexVec::new(), }, fields: FieldsShape::Arbitrary { offsets: vec![Size::ZERO], memory_index: vec![0] }, - largest_niche, + niches, abi, align, size, @@ -1295,25 +1657,17 @@ fn layout_of_uncached<'tcx>( let tagged_layout = TmpLayout { layout: tagged_layout, variants: layout_variants }; - let mut best_layout = match (tagged_layout, niche_filling_layout) { - (tl, Some(nl)) => { - // Pick the smaller layout; otherwise, - // pick the layout with the larger niche; otherwise, - // pick tagged as it has simpler codegen. - use Ordering::*; - let niche_size = |tmp_l: &TmpLayout<'_>| { - tmp_l.layout.largest_niche.map_or(0, |n| n.available(dl)) - }; - match ( - tl.layout.size.cmp(&nl.layout.size), - niche_size(&tl).cmp(&niche_size(&nl)), - ) { - (Greater, _) => nl, - (Equal, Less) => nl, - _ => tl, - } - } - (tl, None) => tl, + let mut best_layout = if let Some(nl) = niche_filling_layout { + // Pick the smaller layout; otherwise pick the layout with the + // largest niche; otherwise pick tagged as it has simpler + // codegen. + let key = |tmp_l: &TmpLayout<'_>| { + let n_size = tmp_l.layout.niches.first().map_or(0, |niche| niche.available(dl)); + (tmp_l.layout.size, !n_size) + }; + cmp::min_by_key(tagged_layout, nl, key) + } else { + tagged_layout }; // Now we can intern the variant layouts and store them in the enum layout. @@ -1662,7 +2016,7 @@ fn generator_layout<'tcx>( }, fields: outer_fields, abi, - largest_niche: prefix.largest_niche, + niches: prefix.niches.clone(), size, align, }); diff --git a/src/test/ui/layout/debug.stderr b/src/test/ui/layout/debug.stderr index c5e1c41d1304c..fcff4ffe709f1 100644 --- a/src/test/ui/layout/debug.stderr +++ b/src/test/ui/layout/debug.stderr @@ -15,16 +15,19 @@ error: layout_of(E) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I32, - false, - ), - valid_range: 0..=0, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I32, + false, + ), + valid_range: 0..=0, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -49,7 +52,7 @@ error: layout_of(E) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -73,7 +76,7 @@ error: layout_of(E) = Layout { 2, ], }, - largest_niche: None, + niches: [], variants: Single { index: 1, }, @@ -120,7 +123,7 @@ error: layout_of(S) = Layout { 2, ], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -142,7 +145,7 @@ error: layout_of(U) = Layout { fields: Union( 2, ), - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -182,16 +185,19 @@ error: layout_of(std::result::Result) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I32, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I32, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -233,7 +239,7 @@ error: layout_of(std::result::Result) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -268,7 +274,7 @@ error: layout_of(std::result::Result) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 1, }, @@ -297,7 +303,7 @@ error: layout_of(i32) = Layout { }, ), fields: Primitive, - largest_niche: None, + niches: [], variants: Single { index: 0, }, diff --git a/src/test/ui/layout/hexagon-enum.stderr b/src/test/ui/layout/hexagon-enum.stderr index d850dd69c96e3..5d5c12a7f5624 100644 --- a/src/test/ui/layout/hexagon-enum.stderr +++ b/src/test/ui/layout/hexagon-enum.stderr @@ -21,16 +21,19 @@ error: layout_of(A) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=0, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=0, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -55,7 +58,7 @@ error: layout_of(A) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -91,16 +94,19 @@ error: layout_of(B) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 255..=255, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 255..=255, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -125,7 +131,7 @@ error: layout_of(B) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -161,16 +167,19 @@ error: layout_of(C) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I16, - false, - ), - valid_range: 256..=256, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I16, + false, + ), + valid_range: 256..=256, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -195,7 +204,7 @@ error: layout_of(C) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -231,16 +240,19 @@ error: layout_of(P) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I32, - false, - ), - valid_range: 268435456..=268435456, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I32, + false, + ), + valid_range: 268435456..=268435456, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -265,7 +277,7 @@ error: layout_of(P) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -301,16 +313,19 @@ error: layout_of(T) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I32, - true, - ), - valid_range: 2164260864..=2164260864, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I32, + true, + ), + valid_range: 2164260864..=2164260864, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -335,7 +350,7 @@ error: layout_of(T) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, diff --git a/src/test/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr b/src/test/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr index bfabe2d12f7ff..dee6fbad967f7 100644 --- a/src/test/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr +++ b/src/test/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr @@ -27,16 +27,19 @@ error: layout_of(MissingPayloadField) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -77,7 +80,7 @@ error: layout_of(MissingPayloadField) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -95,7 +98,7 @@ error: layout_of(MissingPayloadField) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 1, }, @@ -138,16 +141,19 @@ error: layout_of(CommonPayloadField) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -189,7 +195,7 @@ error: layout_of(CommonPayloadField) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -224,7 +230,7 @@ error: layout_of(CommonPayloadField) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 1, }, @@ -266,16 +272,19 @@ error: layout_of(CommonPayloadFieldIsMaybeUninit) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -316,7 +325,7 @@ error: layout_of(CommonPayloadFieldIsMaybeUninit) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -350,7 +359,7 @@ error: layout_of(CommonPayloadFieldIsMaybeUninit) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 1, }, @@ -392,16 +401,19 @@ error: layout_of(NicheFirst) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=4, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=4, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -414,6 +426,7 @@ error: layout_of(NicheFirst) = Layout { untagged_variant: 0, niche_variants: 1..=2, niche_start: 3, + flag: None, }, tag_field: 0, variants: [ @@ -449,16 +462,7 @@ error: layout_of(NicheFirst) = Layout { 1, ], }, - largest_niche: Some( - Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=2, - }, - ), + niches: [], variants: Single { index: 0, }, @@ -476,7 +480,7 @@ error: layout_of(NicheFirst) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 1, }, @@ -494,7 +498,7 @@ error: layout_of(NicheFirst) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 2, }, @@ -536,16 +540,19 @@ error: layout_of(NicheSecond) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(1 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=4, + data: NicheData { + offset: Size(1 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=4, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -558,6 +565,7 @@ error: layout_of(NicheSecond) = Layout { untagged_variant: 0, niche_variants: 1..=2, niche_start: 3, + flag: None, }, tag_field: 0, variants: [ @@ -593,16 +601,7 @@ error: layout_of(NicheSecond) = Layout { 1, ], }, - largest_niche: Some( - Niche { - offset: Size(1 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=2, - }, - ), + niches: [], variants: Single { index: 0, }, @@ -620,7 +619,7 @@ error: layout_of(NicheSecond) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 1, }, @@ -638,7 +637,7 @@ error: layout_of(NicheSecond) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 2, }, diff --git a/src/test/ui/layout/issue-96185-overaligned-enum.stderr b/src/test/ui/layout/issue-96185-overaligned-enum.stderr index de6177c8dfc4d..cdf7dd5987865 100644 --- a/src/test/ui/layout/issue-96185-overaligned-enum.stderr +++ b/src/test/ui/layout/issue-96185-overaligned-enum.stderr @@ -15,16 +15,19 @@ error: layout_of(Aligned1) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -49,7 +52,7 @@ error: layout_of(Aligned1) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -67,7 +70,7 @@ error: layout_of(Aligned1) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 1, }, @@ -103,16 +106,19 @@ error: layout_of(Aligned2) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -137,7 +143,7 @@ error: layout_of(Aligned2) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -155,7 +161,7 @@ error: layout_of(Aligned2) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 1, }, diff --git a/src/test/ui/layout/thumb-enum.stderr b/src/test/ui/layout/thumb-enum.stderr index 227bd950b6666..569a4eb1dd4aa 100644 --- a/src/test/ui/layout/thumb-enum.stderr +++ b/src/test/ui/layout/thumb-enum.stderr @@ -21,16 +21,19 @@ error: layout_of(A) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=0, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=0, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -55,7 +58,7 @@ error: layout_of(A) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -91,16 +94,19 @@ error: layout_of(B) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 255..=255, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 255..=255, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -125,7 +131,7 @@ error: layout_of(B) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -161,16 +167,19 @@ error: layout_of(C) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I16, - false, - ), - valid_range: 256..=256, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I16, + false, + ), + valid_range: 256..=256, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -195,7 +204,7 @@ error: layout_of(C) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -231,16 +240,19 @@ error: layout_of(P) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I32, - false, - ), - valid_range: 268435456..=268435456, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I32, + false, + ), + valid_range: 268435456..=268435456, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -265,7 +277,7 @@ error: layout_of(P) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -301,16 +313,19 @@ error: layout_of(T) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I32, - true, - ), - valid_range: 2164260864..=2164260864, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I32, + true, + ), + valid_range: 2164260864..=2164260864, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -335,7 +350,7 @@ error: layout_of(T) = Layout { offsets: [], memory_index: [], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, diff --git a/src/test/ui/layout/zero-sized-array-enum-niche.stderr b/src/test/ui/layout/zero-sized-array-enum-niche.stderr index a3e82070e0f52..2e691db4b784e 100644 --- a/src/test/ui/layout/zero-sized-array-enum-niche.stderr +++ b/src/test/ui/layout/zero-sized-array-enum-niche.stderr @@ -15,16 +15,19 @@ error: layout_of(std::result::Result<[u32; 0], bool>) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -53,7 +56,7 @@ error: layout_of(std::result::Result<[u32; 0], bool>) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -75,16 +78,19 @@ error: layout_of(std::result::Result<[u32; 0], bool>) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(1 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(1 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Single { index: 1, }, @@ -114,16 +120,19 @@ error: layout_of(MultipleAlignments) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=2, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=2, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -152,7 +161,7 @@ error: layout_of(MultipleAlignments) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -174,7 +183,7 @@ error: layout_of(MultipleAlignments) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 1, }, @@ -196,16 +205,19 @@ error: layout_of(MultipleAlignments) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(1 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(1 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Single { index: 2, }, @@ -235,16 +247,19 @@ error: layout_of(std::result::Result<[u32; 0], Packed>) = 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I8, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I8, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -273,7 +288,7 @@ error: layout_of(std::result::Result<[u32; 0], Packed>) = 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -295,16 +310,19 @@ error: layout_of(std::result::Result<[u32; 0], Packed>) = 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(1 bytes), - value: Int( - I16, - false, - ), - valid_range: 1..=65535, + data: NicheData { + offset: Size(1 bytes), + value: Int( + I16, + false, + ), + valid_range: 1..=65535, + }, + flag: None, }, - ), + ], variants: Single { index: 1, }, @@ -334,16 +352,19 @@ error: layout_of(std::result::Result<[u32; 0], Packed>) = Layout { 0, ], }, - largest_niche: Some( + niches: [ Niche { - offset: Size(0 bytes), - value: Int( - I16, - false, - ), - valid_range: 0..=1, + data: NicheData { + offset: Size(0 bytes), + value: Int( + I16, + false, + ), + valid_range: 0..=1, + }, + flag: None, }, - ), + ], variants: Multiple { tag: Initialized { value: Int( @@ -356,6 +377,7 @@ error: layout_of(std::result::Result<[u32; 0], Packed>) = Layout { untagged_variant: 1, niche_variants: 0..=0, niche_start: 1, + flag: None, }, tag_field: 0, variants: [ @@ -376,7 +398,7 @@ error: layout_of(std::result::Result<[u32; 0], Packed>) = Layout { 0, ], }, - largest_niche: None, + niches: [], variants: Single { index: 0, }, @@ -398,16 +420,7 @@ error: layout_of(std::result::Result<[u32; 0], Packed>) = Layout { 0, ], }, - largest_niche: Some( - Niche { - offset: Size(0 bytes), - value: Int( - I16, - false, - ), - valid_range: 0..=0, - }, - ), + niches: [], variants: Single { index: 1, }, diff --git a/src/test/ui/print_type_sizes/padding.stdout b/src/test/ui/print_type_sizes/padding.stdout index 9afdf76245df7..b0a0ceb1ffcf6 100644 --- a/src/test/ui/print_type_sizes/padding.stdout +++ b/src/test/ui/print_type_sizes/padding.stdout @@ -1,21 +1,17 @@ -print-type-size type: `E1`: 12 bytes, alignment: 4 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `B`: 11 bytes -print-type-size padding: 3 bytes -print-type-size field `.0`: 8 bytes, alignment: 4 bytes -print-type-size variant `A`: 7 bytes -print-type-size field `.1`: 1 bytes -print-type-size padding: 2 bytes -print-type-size field `.0`: 4 bytes, alignment: 4 bytes -print-type-size type: `E2`: 12 bytes, alignment: 4 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `B`: 11 bytes -print-type-size padding: 3 bytes -print-type-size field `.0`: 8 bytes, alignment: 4 bytes -print-type-size variant `A`: 7 bytes -print-type-size field `.0`: 1 bytes -print-type-size padding: 2 bytes -print-type-size field `.1`: 4 bytes, alignment: 4 bytes +print-type-size type: `E1`: 8 bytes, alignment: 4 bytes +print-type-size variant `B`: 8 bytes +print-type-size field `.0`: 8 bytes +print-type-size variant `A`: 6 bytes +print-type-size field `.0`: 4 bytes +print-type-size padding: 1 bytes +print-type-size field `.1`: 1 bytes, alignment: 1 bytes +print-type-size type: `E2`: 8 bytes, alignment: 4 bytes +print-type-size variant `B`: 8 bytes +print-type-size field `.0`: 8 bytes +print-type-size variant `A`: 6 bytes +print-type-size field `.1`: 4 bytes +print-type-size padding: 1 bytes +print-type-size field `.0`: 1 bytes, alignment: 1 bytes print-type-size type: `S`: 8 bytes, alignment: 4 bytes print-type-size field `.g`: 4 bytes print-type-size field `.a`: 1 bytes diff --git a/src/test/ui/stats/hir-stats.stderr b/src/test/ui/stats/hir-stats.stderr index 350b140aada3c..cf281d5d933da 100644 --- a/src/test/ui/stats/hir-stats.stderr +++ b/src/test/ui/stats/hir-stats.stderr @@ -2,118 +2,118 @@ ast-stats-1 PRE EXPANSION AST STATS ast-stats-1 Name Accumulated Size Count Item Size ast-stats-1 ---------------------------------------------------------------- ast-stats-1 ExprField 48 ( 0.6%) 1 48 +ast-stats-1 GenericArgs 56 ( 0.7%) 1 56 +ast-stats-1 - AngleBracketed 56 ( 0.7%) 1 ast-stats-1 Crate 56 ( 0.7%) 1 56 ast-stats-1 Attribute 64 ( 0.8%) 2 32 ast-stats-1 - Normal 32 ( 0.4%) 1 ast-stats-1 - DocComment 32 ( 0.4%) 1 -ast-stats-1 GenericArgs 64 ( 0.8%) 1 64 -ast-stats-1 - AngleBracketed 64 ( 0.8%) 1 +ast-stats-1 WherePredicate 64 ( 0.8%) 1 64 +ast-stats-1 - BoundPredicate 64 ( 0.8%) 1 ast-stats-1 Local 72 ( 0.9%) 1 72 -ast-stats-1 WherePredicate 72 ( 0.9%) 1 72 -ast-stats-1 - BoundPredicate 72 ( 0.9%) 1 -ast-stats-1 Arm 96 ( 1.1%) 2 48 -ast-stats-1 ForeignItem 96 ( 1.1%) 1 96 -ast-stats-1 - Fn 96 ( 1.1%) 1 +ast-stats-1 Arm 96 ( 1.2%) 2 48 +ast-stats-1 ForeignItem 96 ( 1.2%) 1 96 +ast-stats-1 - Fn 96 ( 1.2%) 1 ast-stats-1 FieldDef 160 ( 1.9%) 2 80 ast-stats-1 Stmt 160 ( 1.9%) 5 32 ast-stats-1 - Local 32 ( 0.4%) 1 ast-stats-1 - MacCall 32 ( 0.4%) 1 -ast-stats-1 - Expr 96 ( 1.1%) 3 +ast-stats-1 - Expr 96 ( 1.2%) 3 ast-stats-1 Param 160 ( 1.9%) 4 40 ast-stats-1 FnDecl 200 ( 2.4%) 5 40 ast-stats-1 Variant 240 ( 2.9%) 2 120 -ast-stats-1 Block 288 ( 3.4%) 6 48 +ast-stats-1 Block 288 ( 3.5%) 6 48 ast-stats-1 GenericBound 352 ( 4.2%) 4 88 ast-stats-1 - Trait 352 ( 4.2%) 4 -ast-stats-1 AssocItem 416 ( 4.9%) 4 104 +ast-stats-1 AssocItem 416 ( 5.0%) 4 104 ast-stats-1 - TyAlias 208 ( 2.5%) 2 ast-stats-1 - Fn 208 ( 2.5%) 2 -ast-stats-1 GenericParam 480 ( 5.7%) 5 96 -ast-stats-1 PathSegment 720 ( 8.6%) 30 24 -ast-stats-1 Expr 832 ( 9.9%) 8 104 -ast-stats-1 - Path 104 ( 1.2%) 1 -ast-stats-1 - Match 104 ( 1.2%) 1 -ast-stats-1 - Struct 104 ( 1.2%) 1 +ast-stats-1 GenericParam 480 ( 5.8%) 5 96 +ast-stats-1 PathSegment 720 ( 8.7%) 30 24 +ast-stats-1 Expr 832 (10.0%) 8 104 +ast-stats-1 - Path 104 ( 1.3%) 1 +ast-stats-1 - Match 104 ( 1.3%) 1 +ast-stats-1 - Struct 104 ( 1.3%) 1 ast-stats-1 - Lit 208 ( 2.5%) 2 -ast-stats-1 - Block 312 ( 3.7%) 3 -ast-stats-1 Pat 840 (10.0%) 7 120 +ast-stats-1 - Block 312 ( 3.8%) 3 +ast-stats-1 Pat 840 (10.1%) 7 120 ast-stats-1 - Struct 120 ( 1.4%) 1 ast-stats-1 - Wild 120 ( 1.4%) 1 -ast-stats-1 - Ident 600 ( 7.1%) 5 -ast-stats-1 Ty 1_344 (16.0%) 14 96 -ast-stats-1 - Rptr 96 ( 1.1%) 1 -ast-stats-1 - Ptr 96 ( 1.1%) 1 -ast-stats-1 - ImplicitSelf 192 ( 2.3%) 2 -ast-stats-1 - Path 960 (11.4%) 10 -ast-stats-1 Item 1_656 (19.7%) 9 184 +ast-stats-1 - Ident 600 ( 7.2%) 5 +ast-stats-1 Ty 1_232 (14.9%) 14 88 +ast-stats-1 - Rptr 88 ( 1.1%) 1 +ast-stats-1 - Ptr 88 ( 1.1%) 1 +ast-stats-1 - ImplicitSelf 176 ( 2.1%) 2 +ast-stats-1 - Path 880 (10.6%) 10 +ast-stats-1 Item 1_656 (20.0%) 9 184 ast-stats-1 - Trait 184 ( 2.2%) 1 ast-stats-1 - Enum 184 ( 2.2%) 1 ast-stats-1 - ForeignMod 184 ( 2.2%) 1 ast-stats-1 - Impl 184 ( 2.2%) 1 ast-stats-1 - Fn 368 ( 4.4%) 2 -ast-stats-1 - Use 552 ( 6.6%) 3 +ast-stats-1 - Use 552 ( 6.7%) 3 ast-stats-1 ---------------------------------------------------------------- -ast-stats-1 Total 8_416 +ast-stats-1 Total 8_288 ast-stats-1 ast-stats-2 POST EXPANSION AST STATS ast-stats-2 Name Accumulated Size Count Item Size ast-stats-2 ---------------------------------------------------------------- ast-stats-2 ExprField 48 ( 0.5%) 1 48 +ast-stats-2 GenericArgs 56 ( 0.6%) 1 56 +ast-stats-2 - AngleBracketed 56 ( 0.6%) 1 ast-stats-2 Crate 56 ( 0.6%) 1 56 -ast-stats-2 GenericArgs 64 ( 0.7%) 1 64 -ast-stats-2 - AngleBracketed 64 ( 0.7%) 1 +ast-stats-2 WherePredicate 64 ( 0.7%) 1 64 +ast-stats-2 - BoundPredicate 64 ( 0.7%) 1 ast-stats-2 Local 72 ( 0.8%) 1 72 -ast-stats-2 WherePredicate 72 ( 0.8%) 1 72 -ast-stats-2 - BoundPredicate 72 ( 0.8%) 1 -ast-stats-2 Arm 96 ( 1.0%) 2 48 -ast-stats-2 ForeignItem 96 ( 1.0%) 1 96 -ast-stats-2 - Fn 96 ( 1.0%) 1 +ast-stats-2 Arm 96 ( 1.1%) 2 48 +ast-stats-2 ForeignItem 96 ( 1.1%) 1 96 +ast-stats-2 - Fn 96 ( 1.1%) 1 ast-stats-2 InlineAsm 120 ( 1.3%) 1 120 ast-stats-2 Attribute 128 ( 1.4%) 4 32 -ast-stats-2 - DocComment 32 ( 0.3%) 1 -ast-stats-2 - Normal 96 ( 1.0%) 3 -ast-stats-2 FieldDef 160 ( 1.7%) 2 80 -ast-stats-2 Stmt 160 ( 1.7%) 5 32 -ast-stats-2 - Local 32 ( 0.3%) 1 -ast-stats-2 - Semi 32 ( 0.3%) 1 -ast-stats-2 - Expr 96 ( 1.0%) 3 -ast-stats-2 Param 160 ( 1.7%) 4 40 +ast-stats-2 - DocComment 32 ( 0.4%) 1 +ast-stats-2 - Normal 96 ( 1.1%) 3 +ast-stats-2 FieldDef 160 ( 1.8%) 2 80 +ast-stats-2 Stmt 160 ( 1.8%) 5 32 +ast-stats-2 - Local 32 ( 0.4%) 1 +ast-stats-2 - Semi 32 ( 0.4%) 1 +ast-stats-2 - Expr 96 ( 1.1%) 3 +ast-stats-2 Param 160 ( 1.8%) 4 40 ast-stats-2 FnDecl 200 ( 2.2%) 5 40 -ast-stats-2 Variant 240 ( 2.6%) 2 120 -ast-stats-2 Block 288 ( 3.1%) 6 48 -ast-stats-2 GenericBound 352 ( 3.8%) 4 88 -ast-stats-2 - Trait 352 ( 3.8%) 4 -ast-stats-2 AssocItem 416 ( 4.5%) 4 104 +ast-stats-2 Variant 240 ( 2.7%) 2 120 +ast-stats-2 Block 288 ( 3.2%) 6 48 +ast-stats-2 GenericBound 352 ( 3.9%) 4 88 +ast-stats-2 - Trait 352 ( 3.9%) 4 +ast-stats-2 AssocItem 416 ( 4.6%) 4 104 ast-stats-2 - TyAlias 208 ( 2.3%) 2 ast-stats-2 - Fn 208 ( 2.3%) 2 -ast-stats-2 GenericParam 480 ( 5.2%) 5 96 -ast-stats-2 PathSegment 792 ( 8.7%) 33 24 -ast-stats-2 Pat 840 ( 9.2%) 7 120 +ast-stats-2 GenericParam 480 ( 5.3%) 5 96 +ast-stats-2 PathSegment 792 ( 8.8%) 33 24 +ast-stats-2 Pat 840 ( 9.3%) 7 120 ast-stats-2 - Struct 120 ( 1.3%) 1 ast-stats-2 - Wild 120 ( 1.3%) 1 -ast-stats-2 - Ident 600 ( 6.6%) 5 -ast-stats-2 Expr 936 (10.2%) 9 104 -ast-stats-2 - Path 104 ( 1.1%) 1 -ast-stats-2 - Match 104 ( 1.1%) 1 -ast-stats-2 - Struct 104 ( 1.1%) 1 -ast-stats-2 - InlineAsm 104 ( 1.1%) 1 +ast-stats-2 - Ident 600 ( 6.7%) 5 +ast-stats-2 Expr 936 (10.4%) 9 104 +ast-stats-2 - Path 104 ( 1.2%) 1 +ast-stats-2 - Match 104 ( 1.2%) 1 +ast-stats-2 - Struct 104 ( 1.2%) 1 +ast-stats-2 - InlineAsm 104 ( 1.2%) 1 ast-stats-2 - Lit 208 ( 2.3%) 2 -ast-stats-2 - Block 312 ( 3.4%) 3 -ast-stats-2 Ty 1_344 (14.7%) 14 96 -ast-stats-2 - Rptr 96 ( 1.0%) 1 -ast-stats-2 - Ptr 96 ( 1.0%) 1 -ast-stats-2 - ImplicitSelf 192 ( 2.1%) 2 -ast-stats-2 - Path 960 (10.5%) 10 -ast-stats-2 Item 2_024 (22.1%) 11 184 +ast-stats-2 - Block 312 ( 3.5%) 3 +ast-stats-2 Ty 1_232 (13.7%) 14 88 +ast-stats-2 - Rptr 88 ( 1.0%) 1 +ast-stats-2 - Ptr 88 ( 1.0%) 1 +ast-stats-2 - ImplicitSelf 176 ( 2.0%) 2 +ast-stats-2 - Path 880 ( 9.8%) 10 +ast-stats-2 Item 2_024 (22.4%) 11 184 ast-stats-2 - Trait 184 ( 2.0%) 1 ast-stats-2 - Enum 184 ( 2.0%) 1 ast-stats-2 - ExternCrate 184 ( 2.0%) 1 ast-stats-2 - ForeignMod 184 ( 2.0%) 1 ast-stats-2 - Impl 184 ( 2.0%) 1 -ast-stats-2 - Fn 368 ( 4.0%) 2 -ast-stats-2 - Use 736 ( 8.0%) 4 +ast-stats-2 - Fn 368 ( 4.1%) 2 +ast-stats-2 - Use 736 ( 8.2%) 4 ast-stats-2 ---------------------------------------------------------------- -ast-stats-2 Total 9_144 +ast-stats-2 Total 9_016 ast-stats-2 hir-stats HIR STATS hir-stats Name Accumulated Size Count Item Size @@ -139,31 +139,31 @@ hir-stats - Semi 32 ( 0.4%) 1 hir-stats - Expr 32 ( 0.4%) 1 hir-stats FnDecl 120 ( 1.3%) 3 40 hir-stats Attribute 128 ( 1.4%) 4 32 +hir-stats Variant 144 ( 1.6%) 2 72 hir-stats GenericArgs 144 ( 1.6%) 3 48 -hir-stats Variant 160 ( 1.8%) 2 80 hir-stats GenericBound 192 ( 2.1%) 4 48 hir-stats - Trait 192 ( 2.1%) 4 hir-stats WherePredicate 192 ( 2.1%) 3 64 hir-stats - BoundPredicate 192 ( 2.1%) 3 hir-stats Block 288 ( 3.2%) 6 48 -hir-stats Pat 360 ( 3.9%) 5 72 +hir-stats GenericParam 360 ( 4.0%) 5 72 +hir-stats Pat 360 ( 4.0%) 5 72 hir-stats - Wild 72 ( 0.8%) 1 hir-stats - Struct 72 ( 0.8%) 1 hir-stats - Binding 216 ( 2.4%) 3 -hir-stats GenericParam 400 ( 4.4%) 5 80 -hir-stats Generics 560 ( 6.1%) 10 56 +hir-stats Generics 560 ( 6.2%) 10 56 hir-stats Ty 720 ( 7.9%) 15 48 hir-stats - Ptr 48 ( 0.5%) 1 hir-stats - Rptr 48 ( 0.5%) 1 -hir-stats - Path 624 ( 6.8%) 13 -hir-stats Expr 768 ( 8.4%) 12 64 +hir-stats - Path 624 ( 6.9%) 13 +hir-stats Expr 768 ( 8.5%) 12 64 hir-stats - Path 64 ( 0.7%) 1 hir-stats - Struct 64 ( 0.7%) 1 hir-stats - Match 64 ( 0.7%) 1 hir-stats - InlineAsm 64 ( 0.7%) 1 hir-stats - Lit 128 ( 1.4%) 2 hir-stats - Block 384 ( 4.2%) 6 -hir-stats Item 960 (10.5%) 12 80 +hir-stats Item 960 (10.6%) 12 80 hir-stats - Trait 80 ( 0.9%) 1 hir-stats - Enum 80 ( 0.9%) 1 hir-stats - ExternCrate 80 ( 0.9%) 1 @@ -171,8 +171,8 @@ hir-stats - ForeignMod 80 ( 0.9%) 1 hir-stats - Impl 80 ( 0.9%) 1 hir-stats - Fn 160 ( 1.8%) 2 hir-stats - Use 400 ( 4.4%) 5 -hir-stats Path 1_280 (14.0%) 32 40 -hir-stats PathSegment 1_920 (21.0%) 40 48 +hir-stats Path 1_280 (14.1%) 32 40 +hir-stats PathSegment 1_920 (21.2%) 40 48 hir-stats ---------------------------------------------------------------- -hir-stats Total 9_128 +hir-stats Total 9_072 hir-stats diff --git a/src/test/ui/structs-enums/type-sizes.rs b/src/test/ui/structs-enums/type-sizes.rs index 7a23f13630a43..4219ea6848898 100644 --- a/src/test/ui/structs-enums/type-sizes.rs +++ b/src/test/ui/structs-enums/type-sizes.rs @@ -5,7 +5,7 @@ #![feature(never_type)] use std::mem::size_of; -use std::num::NonZeroU8; +use std::num::{NonZeroU64, NonZeroU8}; struct t {a: u8, b: i8} struct u {a: u8, b: i8, c: u8} @@ -168,6 +168,45 @@ pub enum EnumManyVariant { _F0, _F1, _F2, _F3, _F4, _F5, _F6, _F7, _F8, _F9, _FA, _FB, _FC, _FD, _FE, _FF, } +pub enum EnumContainingAnother { + A(X), + + // 0x100 niche variants. + _00, _01, _02, _03, _04, _05, _06, _07, _08, _09, _0A, _0B, _0C, _0D, _0E, _0F, + _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _1A, _1B, _1C, _1D, _1E, _1F, + _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _2A, _2B, _2C, _2D, _2E, _2F, + _30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _3A, _3B, _3C, _3D, _3E, _3F, + _40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _4A, _4B, _4C, _4D, _4E, _4F, + _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _5A, _5B, _5C, _5D, _5E, _5F, + _60, _61, _62, _63, _64, _65, _66, _67, _68, _69, _6A, _6B, _6C, _6D, _6E, _6F, + _70, _71, _72, _73, _74, _75, _76, _77, _78, _79, _7A, _7B, _7C, _7D, _7E, _7F, + _80, _81, _82, _83, _84, _85, _86, _87, _88, _89, _8A, _8B, _8C, _8D, _8E, _8F, + _90, _91, _92, _93, _94, _95, _96, _97, _98, _99, _9A, _9B, _9C, _9D, _9E, _9F, + _A0, _A1, _A2, _A3, _A4, _A5, _A6, _A7, _A8, _A9, _AA, _AB, _AC, _AD, _AE, _AF, + _B0, _B1, _B2, _B3, _B4, _B5, _B6, _B7, _B8, _B9, _BA, _BB, _BC, _BD, _BE, _BF, + _C0, _C1, _C2, _C3, _C4, _C5, _C6, _C7, _C8, _C9, _CA, _CB, _CC, _CD, _CE, _CF, + _D0, _D1, _D2, _D3, _D4, _D5, _D6, _D7, _D8, _D9, _DA, _DB, _DC, _DD, _DE, _DF, + _E0, _E1, _E2, _E3, _E4, _E5, _E6, _E7, _E8, _E9, _EA, _EB, _EC, _ED, _EE, _EF, + _F0, _F1, _F2, _F3, _F4, _F5, _F6, _F7, _F8, _F9, _FA, _FB, _FC, _FD, _FE, _FF, +} + +#[repr(flag)] +enum Provenance { + Concrete { + alloc_id: NonZeroU64, + sb: NonZeroU64, + }, + Wildcard, +} + +enum ProvenanceNoFlag { + Concrete { + alloc_id: NonZeroU64, + sb: NonZeroU64, + }, + Wildcard, +} + pub fn main() { assert_eq!(size_of::(), 1 as usize); assert_eq!(size_of::(), 4 as usize); @@ -249,4 +288,18 @@ pub fn main() { assert_eq!(size_of::>>(), 4); assert_eq!(size_of::>>(), 6); assert_eq!(size_of::>>(), 6); + + // Check for using a flag with a niche - a bool on its own can't + // accommodate 256 variants. + assert_eq!(size_of::>(), 2); + assert_eq!(size_of::>>(), 2); + assert_eq!(size_of::>>>(), 2); + + assert_eq!(size_of::(), 16); + assert_eq!(size_of::>(), 16); + assert_eq!(size_of::>>(), 16); + + assert_eq!(size_of::(), 16); + assert_eq!(size_of::>(), 24); + assert_eq!(size_of::>>(), 24); }