Skip to content

Commit 7846258

Browse files
zachs18zachs18
committed
Don't encode enum tag for uninhabited repr(Rust) enum variants
1 parent 2586fd0 commit 7846258

File tree

7 files changed

+99
-34
lines changed

7 files changed

+99
-34
lines changed

‎compiler/rustc_abi/src/layout.rs

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -844,11 +844,15 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
844844
let mut layout_variants = variants
845845
.iter_enumerated()
846846
.map(|(i, field_layouts)| {
847-
let mut st = self.univariant(
848-
field_layouts,
849-
repr,
850-
StructKind::Prefixed(min_ity.size(), prefix_align),
851-
)?;
847+
let uninhabited = field_layouts.iter().any(|f| f.is_uninhabited());
848+
// We don't need to encode the tag in uninhabited variants in repr(Rust) enums
849+
let struct_kind = if uninhabited && !repr.inhibit_enum_layout_opt() {
850+
StructKind::AlwaysSized
851+
} else {
852+
StructKind::Prefixed(min_ity.size(), prefix_align)
853+
};
854+
let mut st = self.univariant(field_layouts, repr, struct_kind)?;
855+
852856
st.variants = Variants::Single { index: i, variants: None };
853857
// Find the first field we can't move later
854858
// to make room for a larger discriminant.
@@ -918,6 +922,11 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
918922
let old_ity_size = min_ity.size();
919923
let new_ity_size = ity.size();
920924
for variant in &mut layout_variants {
925+
// Don't change field offsets of uninhabited variants in repr(Rust) enums,
926+
// they don't encode the tag and their fields may overlap with the tag.
927+
if variant.is_uninhabited() && !repr.inhibit_enum_layout_opt() {
928+
continue;
929+
}
921930
match variant.fields {
922931
FieldsShape::Arbitrary { ref mut offsets, .. } => {
923932
for i in offsets {
@@ -962,6 +971,12 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
962971
let FieldsShape::Arbitrary { ref offsets, .. } = layout_variant.fields else {
963972
panic!("encountered a non-arbitrary layout during enum layout");
964973
};
974+
// Don't look in uninhabited variants for repr(Rust) enums, they will never be
975+
// passed over an ABI so they don't matter for the purpose of determining
976+
// BackendRepr.
977+
if layout_variant.is_uninhabited() && !repr.inhibit_enum_layout_opt() {
978+
continue;
979+
}
965980
// We skip *all* ZST here and later check if we are good in terms of alignment.
966981
// This lets us handle some cases involving aligned ZST.
967982
let mut fields = iter::zip(field_layouts, offsets).filter(|p| !p.0.is_zst());
@@ -1078,6 +1093,43 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
10781093
.map(|v| v.randomization_seed)
10791094
.fold(repr.field_shuffle_seed, |acc, seed| acc.wrapping_add(seed));
10801095

1096+
// If all variants are uninhabited, the repr does not inhibit layout optimizations,
1097+
// and all fields are ZSTs, then the tagged layout will not have room for the tag.
1098+
// So in this case, we return an uninhabited layout that is big enough and aligned
1099+
// enough for all variant fields, but do not say it has any fields itself.
1100+
// Doing this only when the layout is too small to fit the tag gives better error
1101+
// messages during const-eval in some cases, "constructing invalid value at .<enum-tag>:
1102+
// encountered an uninhabited enum variant" instead of "constructing invalid value:
1103+
// encountered a value of uninhabited type".
1104+
// Note the we only reach this case when there is at least one non-1-aligned ZST field,
1105+
// since the all-1-ZST case is handled by the "present_variants" check in
1106+
// `layout_of_struct_or_enum`.
1107+
if uninhabited && size < tag.size(&self.cx) {
1108+
// The only way for the size to be less than the tag's size is for it to be zero,
1109+
// which can only occur when the repr does not inhibit layout optimization.
1110+
debug_assert!(
1111+
size == Size::ZERO,
1112+
"size was non-zero but less than tag size: 0 < {size:?} < {:?}",
1113+
tag.size(&self.cx)
1114+
);
1115+
debug_assert!(
1116+
!repr.inhibit_enum_layout_opt(),
1117+
"enum size was zero with layout optimizations disabled"
1118+
);
1119+
return Ok(LayoutData {
1120+
fields: FieldsShape::Arbitrary { offsets: [].into(), memory_index: [].into() },
1121+
variants: Variants::Empty { variants: Some(layout_variants) },
1122+
backend_repr: BackendRepr::Memory { sized: true },
1123+
largest_niche: None,
1124+
uninhabited: true,
1125+
align,
1126+
size,
1127+
max_repr_align,
1128+
unadjusted_abi_align,
1129+
randomization_seed: combined_seed,
1130+
});
1131+
}
1132+
10811133
let tagged_layout = LayoutData {
10821134
variants: Variants::Multiple {
10831135
tag,

‎tests/codegen-llvm/enum/enum-aggregate.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ fn make_uninhabited_err_indirectly(n: Never) -> Result<u32, Never> {
114114
fn make_fully_uninhabited_result(v: u32, n: Never) -> Result<(u32, Never), (Never, u32)> {
115115
// Actually reaching this would be UB, so we don't actually build a result.
116116

117-
// CHECK-LABEL: { i32, i32 } @make_fully_uninhabited_result(i32{{( signext)?}} %v)
117+
// CHECK-LABEL: i32 @make_fully_uninhabited_result(i32{{( signext)?}} %v)
118118
// CHECK-NEXT: start:
119119
// CHECK-NEXT: call void @llvm.trap()
120120
// CHECK-NEXT: call void @llvm.trap()

‎tests/ui/consts/const-eval/ub-enum.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,9 @@ const BAD_OPTION_CHAR: Option<(char, char)> = Some(('x', unsafe { mem::transmute
9595

9696
// All variants are uninhabited but also have data.
9797
// Use `0` as constant to make behavior endianness-independent.
98-
const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(0u64) };
98+
const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(0u32) };
9999
//~^ ERROR uninhabited enum variant
100-
const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u64) };
100+
const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u32) };
101101
//~^ ERROR uninhabited enum variant
102102

103103
// All variants have same-size data but only one inhabited.

‎tests/ui/consts/const-eval/ub-enum.stderr

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,13 +111,13 @@ LL | const BAD_OPTION_CHAR: Option<(char, char)> = Some(('x', unsafe { mem::tran
111111
error[E0080]: constructing invalid value at .<enum-tag>: encountered an uninhabited enum variant
112112
--> $DIR/ub-enum.rs:98:77
113113
|
114-
LL | const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(0u64) };
114+
LL | const BAD_UNINHABITED_WITH_DATA1: Result<(i32, Never), (i32, !)> = unsafe { mem::transmute(0u32) };
115115
| ^^^^^^^^^^^^^^^^^^^^ evaluation of `BAD_UNINHABITED_WITH_DATA1` failed here
116116

117117
error[E0080]: constructing invalid value at .<enum-tag>: encountered an uninhabited enum variant
118118
--> $DIR/ub-enum.rs:100:77
119119
|
120-
LL | const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u64) };
120+
LL | const BAD_UNINHABITED_WITH_DATA2: Result<(i32, !), (i32, Never)> = unsafe { mem::transmute(0u32) };
121121
| ^^^^^^^^^^^^^^^^^^^^ evaluation of `BAD_UNINHABITED_WITH_DATA2` failed here
122122

123123
error[E0080]: read discriminant of an uninhabited enum variant

‎tests/ui/layout/debug.stderr

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ LL | union EmptyUnion {}
55
| ^^^^^^^^^^^^^^^^^^^
66

77
error: layout_of(E) = Layout {
8-
size: Size(12 bytes),
8+
size: Size(8 bytes),
99
align: AbiAlign {
1010
abi: Align(4 bytes),
1111
},
@@ -65,23 +65,36 @@ error: layout_of(E) = Layout {
6565
randomization_seed: $SEED,
6666
},
6767
Layout {
68-
size: Size(12 bytes),
68+
size: Size(8 bytes),
6969
align: AbiAlign {
7070
abi: Align(4 bytes),
7171
},
72-
backend_repr: Memory {
73-
sized: true,
74-
},
72+
backend_repr: ScalarPair(
73+
Initialized {
74+
value: Int(
75+
I32,
76+
true,
77+
),
78+
valid_range: 0..=4294967295,
79+
},
80+
Initialized {
81+
value: Int(
82+
I32,
83+
true,
84+
),
85+
valid_range: 0..=4294967295,
86+
},
87+
),
7588
fields: Arbitrary {
7689
offsets: [
77-
Size(4 bytes),
78-
Size(4 bytes),
7990
Size(8 bytes),
91+
Size(0 bytes),
92+
Size(4 bytes),
8093
],
8194
memory_index: [
95+
2,
8296
0,
8397
1,
84-
2,
8598
],
8699
},
87100
largest_niche: None,
@@ -506,7 +519,7 @@ LL | union P3 { x: F32x4 }
506519
| ^^^^^^^^
507520

508521
error: layout_of(P4) = Layout {
509-
size: Size(12 bytes),
522+
size: Size(8 bytes),
510523
align: AbiAlign {
511524
abi: Align(1 bytes),
512525
},

‎tests/ui/layout/enum.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ enum UninhabitedVariantAlign { //~ERROR: abi: Align(2 bytes)
1212
}
1313

1414
#[rustc_layout(size)]
15-
enum UninhabitedVariantSpace { //~ERROR: size: Size(16 bytes)
15+
enum UninhabitedVariantSpace { //~ERROR: size: Size(15 bytes)
1616
A,
1717
B([u8; 15], !), // make sure there is space being reserved for this field.
1818
}
@@ -47,7 +47,7 @@ enum UninhabitedVariantUntagged { //~ERROR: size: Size(8 bytes)
4747
// the same size, but without a niche.
4848
#[rustc_layout(size, abi)]
4949
enum UninhabitedVariantUntaggedBigger { //~ERROR: size: Size(8 bytes)
50-
//~^ ERROR: abi: Memory
50+
//~^ ERROR: abi: ScalarPair
5151
A(i32),
5252
B([u8; 5], !),
5353
}
@@ -74,7 +74,7 @@ enum UninhabitedVariantLargeWithNiche {
7474
// This uses the tagged layout, but since all variants are uninhabited, none of them store the tag,
7575
// so we only need space for the fields, and the abi is Memory.
7676
#[rustc_layout(size, abi)]
77-
enum AllUninhabitedVariants { //~ERROR: size: Size(3 bytes)
77+
enum AllUninhabitedVariants { //~ERROR: size: Size(2 bytes)
7878
//~^ERROR: abi: Memory
7979
A(i8, bool, !),
8080
B(u8, u8, !),
@@ -103,15 +103,15 @@ enum TaggedI8 { //~ERROR: size: Size(4 bytes)
103103
// Tagged `(u16, i16)`
104104
#[rustc_layout(size, abi)]
105105
enum TaggedI16 { //~ERROR: size: Size(4 bytes)
106-
//~^ERROR: abi: Memory
106+
//~^ERROR: abi: ScalarPair
107107
A(i16),
108108
B(i8, i8, i8, AlignedNever)
109109
}
110110

111111
// This must not use tagged representation, since it's zero-sized.
112112
#[rustc_layout(size, abi)]
113-
enum AllUninhabitedVariantsAlignedZst { //~ERROR: size: Size(2 bytes)
114-
//~^ERROR: abi: Scalar
113+
enum AllUninhabitedVariantsAlignedZst { //~ERROR: size: Size(0 bytes)
114+
//~^ERROR: abi: Memory
115115
A(AlignedNever),
116116
B(AlignedNever),
117117
}

‎tests/ui/layout/enum.stderr

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ error: align: AbiAlign { abi: Align(2 bytes) }
44
LL | enum UninhabitedVariantAlign {
55
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
66

7-
error: size: Size(16 bytes)
7+
error: size: Size(15 bytes)
88
--> $DIR/enum.rs:15:1
99
|
1010
LL | enum UninhabitedVariantSpace {
@@ -46,7 +46,7 @@ error: size: Size(8 bytes)
4646
LL | enum UninhabitedVariantUntaggedBigger {
4747
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
4848

49-
error: abi: Memory { sized: true }
49+
error: abi: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=0 }, Initialized { value: Int(I32, true), valid_range: 0..=4294967295 })
5050
--> $DIR/enum.rs:49:1
5151
|
5252
LL | enum UninhabitedVariantUntaggedBigger {
@@ -65,7 +65,7 @@ LL | enum UninhabitedVariantWithNiche {
6565
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
6666

6767
error: layout_of(UninhabitedVariantLargeWithNiche) = Layout {
68-
size: Size(4 bytes),
68+
size: Size(3 bytes),
6969
align: AbiAlign {
7070
abi: Align(1 bytes),
7171
},
@@ -140,7 +140,7 @@ error: layout_of(UninhabitedVariantLargeWithNiche) = Layout {
140140
randomization_seed: 17394913183323368564,
141141
},
142142
Layout {
143-
size: Size(4 bytes),
143+
size: Size(3 bytes),
144144
align: AbiAlign {
145145
abi: Align(1 bytes),
146146
},
@@ -149,10 +149,10 @@ error: layout_of(UninhabitedVariantLargeWithNiche) = Layout {
149149
},
150150
fields: Arbitrary {
151151
offsets: [
152+
Size(0 bytes),
152153
Size(1 bytes),
153154
Size(2 bytes),
154155
Size(3 bytes),
155-
Size(4 bytes),
156156
],
157157
memory_index: [
158158
0,
@@ -182,7 +182,7 @@ error: layout_of(UninhabitedVariantLargeWithNiche) = Layout {
182182
LL | enum UninhabitedVariantLargeWithNiche {
183183
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
184184

185-
error: size: Size(3 bytes)
185+
error: size: Size(2 bytes)
186186
--> $DIR/enum.rs:77:1
187187
|
188188
LL | enum AllUninhabitedVariants {
@@ -224,19 +224,19 @@ error: size: Size(4 bytes)
224224
LL | enum TaggedI16 {
225225
| ^^^^^^^^^^^^^^
226226

227-
error: abi: Memory { sized: true }
227+
error: abi: ScalarPair(Initialized { value: Int(I16, false), valid_range: 0..=0 }, Initialized { value: Int(I16, true), valid_range: 0..=65535 })
228228
--> $DIR/enum.rs:105:1
229229
|
230230
LL | enum TaggedI16 {
231231
| ^^^^^^^^^^^^^^
232232

233-
error: size: Size(2 bytes)
233+
error: size: Size(0 bytes)
234234
--> $DIR/enum.rs:113:1
235235
|
236236
LL | enum AllUninhabitedVariantsAlignedZst {
237237
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
238238

239-
error: abi: Scalar(Initialized { value: Int(I16, false), valid_range: 0..=0 })
239+
error: abi: Memory { sized: true }
240240
--> $DIR/enum.rs:113:1
241241
|
242242
LL | enum AllUninhabitedVariantsAlignedZst {

0 commit comments

Comments
 (0)