diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 986c04e6b4813..09ad3254e5714 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -629,7 +629,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc for arg in args { match arg.val { - OperandValue::ZeroSized | OperandValue::Uninit => {} + OperandValue::ZeroSized => {} OperandValue::Immediate(_) => call_args.push(arg.immediate()), OperandValue::Pair(a, b) => { call_args.push(a); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4f80e5c6e81e1..edf943c81a755 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -962,7 +962,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { for arg in args { match arg.val { OperandValue::ZeroSized => {} - OperandValue::Uninit => {} OperandValue::Immediate(a) => llargs.push(a), OperandValue::Pair(a, b) => { llargs.push(a); @@ -1940,7 +1939,7 @@ fn get_args_from_tuple<'ll, 'tcx>( result } - OperandValue::ZeroSized | OperandValue::Uninit => vec![], + OperandValue::ZeroSized => vec![], } } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 339c0a968e9d3..d3af6eba33374 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -287,7 +287,7 @@ pub(crate) fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let (base, info) = match bx.load_operand(src).val { OperandValue::Pair(base, info) => unsize_ptr(bx, base, src_ty, dst_ty, Some(info)), OperandValue::Immediate(base) => unsize_ptr(bx, base, src_ty, dst_ty, None), - OperandValue::Ref(..) | OperandValue::ZeroSized | OperandValue::Uninit => bug!(), + OperandValue::Ref(..) | OperandValue::ZeroSized => bug!(), }; OperandValue::Pair(base, info).store(bx, dst); } diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 20d6c01f12f73..aab4259b6b41d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -23,7 +23,7 @@ use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode} use tracing::{debug, info}; use super::operand::OperandRef; -use super::operand::OperandValue::{self, Immediate, Pair, Ref, Uninit, ZeroSized}; +use super::operand::OperandValue::{self, Immediate, Pair, Ref, ZeroSized}; use super::place::{PlaceRef, PlaceValue}; use super::{CachedLlbb, FunctionCx, LocalRef}; use crate::base::{self, is_call_from_compiler_builtins_to_upstream_monomorphization}; @@ -581,10 +581,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { PassMode::Direct(_) | PassMode::Pair(..) => { let op = self.codegen_consume(bx, mir::Place::return_place().as_ref()); - match op.val { - Ref(place_val) => bx.load_from_place(bx.backend_type(op.layout), place_val), - Uninit => bx.cx().const_undef(bx.cx().immediate_backend_type(op.layout)), - _ => op.immediate_or_packed_pair(bx), + if let Ref(place_val) = op.val { + bx.load_from_place(bx.backend_type(op.layout), place_val) + } else { + op.immediate_or_packed_pair(bx) } } @@ -613,9 +613,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { place_val.llval } ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"), - OperandValue::Uninit => { - bug!("uninit return value shouldn't be in PassMode::Cast") - } }; if self.fn_abi.conv == CanonAbi::Arm(ArmCall::CCmseNonSecureEntry) { @@ -1964,7 +1961,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Force by-ref if we have to load through a cast pointer. let (mut llval, align, by_ref) = match op.val { - Immediate(_) | Pair(..) | Uninit => match arg.mode { + Immediate(_) | Pair(..) => match arg.mode { PassMode::Indirect { attrs, .. } => { // Indirect argument may have higher alignment requirements than the type's // alignment. This can happen, e.g. when passing types with <4 byte alignment @@ -1984,14 +1981,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { op.store_with_annotation(bx, scratch); (scratch.val.llval, scratch.val.align, true) } - PassMode::Direct(_) => { - if let Uninit = op.val { - let ibty = bx.cx().immediate_backend_type(arg.layout); - (bx.cx().const_undef(ibty), arg.layout.align.abi, false) - } else { - (op.immediate(), arg.layout.align.abi, false) - } - } + PassMode::Direct(_) => (op.immediate(), arg.layout.align.abi, false), PassMode::Ignore | PassMode::Pair(..) => unreachable!("handled above"), }, Ref(op_place_val) => match arg.mode { diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs index 68893c7ac8fb2..7d35d4b72bd1e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/constant.rs +++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs @@ -6,7 +6,7 @@ use rustc_middle::{bug, mir, span_bug}; use super::FunctionCx; use crate::diagnostics; -use crate::mir::operand::{OperandRef, OperandValue}; +use crate::mir::operand::OperandRef; use crate::traits::*; impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { @@ -17,10 +17,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ) -> OperandRef<'tcx, Bx::Value> { let val = self.eval_mir_constant(constant); let ty = self.monomorphize(constant.ty()); - if val.all_bytes_uninit(self.cx.tcx()) { - let layout = bx.layout_of(ty); - return OperandRef { val: OperandValue::Uninit, layout, move_annotation: None }; - } OperandRef::from_const(bx, val, ty) } diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index 7be2850aced90..c586b8080ef30 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -390,7 +390,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.set_var_name(a, &(name.clone() + ".0")); bx.set_var_name(b, &(name.clone() + ".1")); } - OperandValue::ZeroSized | OperandValue::Uninit => { + OperandValue::ZeroSized => { // These never have a value to talk about } }, diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 552c0b58a194e..1dbbc3fd28fb9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -86,12 +86,6 @@ pub enum OperandValue { /// `is_zst` on its `Layout` returns `true`. Note however that /// these values can still require alignment. ZeroSized, - /// A value for which all bytes are entirely uninitialized. - /// - /// Storing this value is a no-op; it propagates through field extraction. - /// Used to avoid emitting memcpys from uninit globals (which LLVM may - /// otherwise materialize as zero-fills) for `const ` operands. - Uninit, } impl OperandValue { @@ -101,7 +95,7 @@ impl OperandValue { match self { OperandValue::Immediate(llptr) => Some((llptr, None)), OperandValue::Pair(llptr, llextra) => Some((llptr, Some(llextra))), - OperandValue::Ref(_) | OperandValue::ZeroSized | OperandValue::Uninit => None, + OperandValue::Ref(_) | OperandValue::ZeroSized => None, } } @@ -129,7 +123,6 @@ impl OperandValue { #[must_use] pub(crate) fn is_expected_variant_for_type<'tcx>(&self, ty: TyAndLayout<'tcx>) -> bool { match (self, ty.backend_repr) { - (OperandValue::Uninit, _) => true, (OperandValue::ZeroSized, BackendRepr::Memory { .. }) => ty.is_zst(), (OperandValue::Ref(_), BackendRepr::Memory { .. }) => !ty.is_zst(), ( @@ -404,9 +397,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { ); } - let val = if let OperandValue::Uninit = self.val { - OperandValue::Uninit - } else if field.is_zst() { + let val = if field.is_zst() { OperandValue::ZeroSized } else if field.size == self.layout.size { assert_eq!(offset.bytes(), 0); @@ -505,7 +496,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { // Read the tag/niche-encoded discriminant from memory. let tag_op = match self.val { OperandValue::ZeroSized => bug!(), - OperandValue::Uninit => return bx.cx().const_poison(cast_to), OperandValue::Immediate(_) | OperandValue::Pair(_, _) => { self.extract_field(fx, bx, tag_field.as_usize()) } @@ -788,15 +778,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> { field: FieldIdx, field_operand: OperandRef<'tcx, V>, ) { - if matches!(field_operand.val, OperandValue::ZeroSized | OperandValue::Uninit) { + if let OperandValue::ZeroSized = field_operand.val { // A ZST never adds any state, so just ignore it. // This special-casing is worth it because of things like // `Result` where `Ok(never)` is legal to write, // but the type shows as FieldShape::Primitive so we can't // actually look at the layout for the field being set. - // - // Likewise, an uninit field does not contribute any value; - // the builder's unset slots will produce `const_undef` in `build()`. return; } @@ -1032,10 +1019,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue { // Avoid generating stores of zero-sized values, because the only way to have a // zero-sized value is through `undef`/`poison`, and the store itself is useless. } - OperandValue::Uninit => { - // Storing an entirely uninit value is a no-op: the destination is left - // uninitialized, which is valid since the value itself is uninit. - } OperandValue::Ref(val) => { assert!(dest.layout.is_sized(), "cannot directly store unsized values"); if val.llextra.is_some() { diff --git a/compiler/rustc_codegen_ssa/src/mir/retag.rs b/compiler/rustc_codegen_ssa/src/mir/retag.rs index c10302eff0d9b..680f0b44d70dd 100644 --- a/compiler/rustc_codegen_ssa/src/mir/retag.rs +++ b/compiler/rustc_codegen_ssa/src/mir/retag.rs @@ -298,9 +298,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { builder.update_imm(offset, fst); builder.update_imm(offset + Size::from_bytes(1), snd) } - OperandValue::Uninit => { - unreachable!("load_operand never produces Uninit") - } } } } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 4270073b29894..a5d78fc8a7f35 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -91,6 +91,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { true } + fn is_entirely_uninit_const(&self, operand: &mir::Operand<'tcx>) -> bool { + let mir::Operand::Constant(const_op) = operand else { return false }; + self.eval_mir_constant(const_op).all_bytes_uninit(self.cx.tcx()) + } + #[instrument(level = "trace", skip(self, bx))] pub(crate) fn codegen_rvalue( &mut self, @@ -100,6 +105,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ) { match *rvalue { mir::Rvalue::Use(ref operand, with_retag) => { + if self.is_entirely_uninit_const(operand) { + return; + } let cg_operand = self.codegen_operand(bx, operand); // Crucially, we do *not* use `OperandValue::Ref` for types with // `BackendRepr::Scalar | BackendRepr::ScalarPair`. This ensures we match the MIR @@ -170,9 +178,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValue::ZeroSized => { bug!("unsized coercion on a ZST rvalue"); } - OperandValue::Uninit => { - bug!("unsized coercion on an uninit rvalue"); - } } } @@ -191,12 +196,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return; } - let cg_elem = self.codegen_operand(bx, elem); - - if let OperandValue::Uninit = cg_elem.val { + // When the element is a const with all bytes uninit, emit a single memset that + // writes undef to the entire destination. + if self.is_entirely_uninit_const(elem) { + let size = bx.const_usize(dest.layout.size.bytes()); + bx.memset( + dest.val.llval, + bx.const_undef(bx.type_i8()), + size, + dest.val.align, + MemFlags::empty(), + ); return; } + let cg_elem = self.codegen_operand(bx, elem); + let try_init_all_same = |bx: &mut Bx, v| { let start = dest.val.llval; let size = bx.const_usize(dest.layout.size.bytes()); @@ -254,6 +269,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { assert_eq!(operands.len(), 1); } for (i, operand) in operands.iter_enumerated() { + // Do not generate stores for entirely uninit constant fields, for the same + // reason as in `Rvalue::Use` above. + if self.is_entirely_uninit_const(operand) { + continue; + } let op = self.codegen_operand(bx, operand); // Do not generate stores and GEPis for zero-sized fields. if !op.layout.is_zst() { @@ -356,7 +376,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let cx = bx.cx(); match (operand.val, operand.layout.backend_repr, cast.backend_repr) { _ if cast.is_zst() => OperandValue::ZeroSized, - (OperandValue::Uninit, _, _) => OperandValue::Uninit, (OperandValue::Ref(source_place_val), abi::BackendRepr::Memory { .. }, _) => { assert_eq!(source_place_val.llextra, None); // The existing alignment is part of `source_place_val`, diff --git a/tests/codegen-llvm/uninit-return-value.rs b/tests/codegen-llvm/uninit-return-value.rs deleted file mode 100644 index 3a4debb1d4f78..0000000000000 --- a/tests/codegen-llvm/uninit-return-value.rs +++ /dev/null @@ -1,13 +0,0 @@ -// Regression test for https://github.com/rust-lang/rust/issues/159815 - -#![crate_type = "lib"] - -use std::mem::MaybeUninit; - -// CHECK-LABEL: @f -#[no_mangle] -pub fn f() -> MaybeUninit<*const ()> { - // CHECK: start: - // CHECK-NEXT: ret ptr undef - const { MaybeUninit::uninit() } -}