codegen: skip stores for entirely-uninit constant aggregate fields, attempt #2

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -629,7 +629,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
 
         for arg in args {
             match arg.val {
-                OperandValue::ZeroSized | OperandValue::Uninit => {}
+                OperandValue::ZeroSized => {}
                 OperandValue::Immediate(_) => call_args.push(arg.immediate()),
                 OperandValue::Pair(a, b) => {
                     call_args.push(a);

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -962,7 +962,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         for arg in args {
             match arg.val {
                 OperandValue::ZeroSized => {}
-                OperandValue::Uninit => {}
                 OperandValue::Immediate(a) => llargs.push(a),
                 OperandValue::Pair(a, b) => {
                     llargs.push(a);
@@ -1940,7 +1939,7 @@ fn get_args_from_tuple<'ll, 'tcx>(
             result
         }
 
-        OperandValue::ZeroSized | OperandValue::Uninit => vec![],
+        OperandValue::ZeroSized => vec![],
     }
 }
 

compiler/rustc_codegen_ssa/src/base.rs

@@ -287,7 +287,7 @@ pub(crate) fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             let (base, info) = match bx.load_operand(src).val {
                 OperandValue::Pair(base, info) => unsize_ptr(bx, base, src_ty, dst_ty, Some(info)),
                 OperandValue::Immediate(base) => unsize_ptr(bx, base, src_ty, dst_ty, None),
-                OperandValue::Ref(..) | OperandValue::ZeroSized | OperandValue::Uninit => bug!(),
+                OperandValue::Ref(..) | OperandValue::ZeroSized => bug!(),
             };
             OperandValue::Pair(base, info).store(bx, dst);
         }

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -23,7 +23,7 @@ use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode}
 use tracing::{debug, info};
 
 use super::operand::OperandRef;
-use super::operand::OperandValue::{self, Immediate, Pair, Ref, Uninit, ZeroSized};
+use super::operand::OperandValue::{self, Immediate, Pair, Ref, ZeroSized};
 use super::place::{PlaceRef, PlaceValue};
 use super::{CachedLlbb, FunctionCx, LocalRef};
 use crate::base::{self, is_call_from_compiler_builtins_to_upstream_monomorphization};
@@ -581,10 +581,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
             PassMode::Direct(_) | PassMode::Pair(..) => {
                 let op = self.codegen_consume(bx, mir::Place::return_place().as_ref());
-                match op.val {
-                    Ref(place_val) => bx.load_from_place(bx.backend_type(op.layout), place_val),
-                    Uninit => bx.cx().const_undef(bx.cx().immediate_backend_type(op.layout)),
-                    _ => op.immediate_or_packed_pair(bx),
+                if let Ref(place_val) = op.val {
+                    bx.load_from_place(bx.backend_type(op.layout), place_val)
+                } else {
+                    op.immediate_or_packed_pair(bx)
                 }
             }
 
@@ -613,9 +613,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         place_val.llval
                     }
                     ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"),
-                    OperandValue::Uninit => {
-                        bug!("uninit return value shouldn't be in PassMode::Cast")
-                    }
                 };
 
                 if self.fn_abi.conv == CanonAbi::Arm(ArmCall::CCmseNonSecureEntry) {
@@ -1964,7 +1961,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         // Force by-ref if we have to load through a cast pointer.
         let (mut llval, align, by_ref) = match op.val {
-            Immediate(_) | Pair(..) | Uninit => match arg.mode {
+            Immediate(_) | Pair(..) => match arg.mode {
                 PassMode::Indirect { attrs, .. } => {
                     // Indirect argument may have higher alignment requirements than the type's
                     // alignment. This can happen, e.g. when passing types with <4 byte alignment
@@ -1984,14 +1981,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     op.store_with_annotation(bx, scratch);
                     (scratch.val.llval, scratch.val.align, true)
                 }
-                PassMode::Direct(_) => {
-                    if let Uninit = op.val {
-                        let ibty = bx.cx().immediate_backend_type(arg.layout);
-                        (bx.cx().const_undef(ibty), arg.layout.align.abi, false)
-                    } else {
-                        (op.immediate(), arg.layout.align.abi, false)
-                    }
-                }
+                PassMode::Direct(_) => (op.immediate(), arg.layout.align.abi, false),
                 PassMode::Ignore | PassMode::Pair(..) => unreachable!("handled above"),
             },
             Ref(op_place_val) => match arg.mode {

compiler/rustc_codegen_ssa/src/mir/constant.rs

@@ -6,7 +6,7 @@ use rustc_middle::{bug, mir, span_bug};
 
 use super::FunctionCx;
 use crate::diagnostics;
-use crate::mir::operand::{OperandRef, OperandValue};
+use crate::mir::operand::OperandRef;
 use crate::traits::*;
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
@@ -17,10 +17,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ) -> OperandRef<'tcx, Bx::Value> {
         let val = self.eval_mir_constant(constant);
         let ty = self.monomorphize(constant.ty());
-        if val.all_bytes_uninit(self.cx.tcx()) {
-            let layout = bx.layout_of(ty);
-            return OperandRef { val: OperandValue::Uninit, layout, move_annotation: None };
-        }
         OperandRef::from_const(bx, val, ty)
     }
 

compiler/rustc_codegen_ssa/src/mir/debuginfo.rs

@@ -390,7 +390,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         bx.set_var_name(a, &(name.clone() + ".0"));
                         bx.set_var_name(b, &(name.clone() + ".1"));
                     }
-                    OperandValue::ZeroSized | OperandValue::Uninit => {
+                    OperandValue::ZeroSized => {
                         // These never have a value to talk about
                     }
                 },

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -86,12 +86,6 @@ pub enum OperandValue<V> {
     /// `is_zst` on its `Layout` returns `true`. Note however that
     /// these values can still require alignment.
     ZeroSized,
-    /// A value for which all bytes are entirely uninitialized.
-    ///
-    /// Storing this value is a no-op; it propagates through field extraction.
-    /// Used to avoid emitting memcpys from uninit globals (which LLVM may
-    /// otherwise materialize as zero-fills) for `const <uninit>` operands.
-    Uninit,
 }
 
 impl<V: CodegenObject> OperandValue<V> {
@@ -101,7 +95,7 @@ impl<V: CodegenObject> OperandValue<V> {
         match self {
             OperandValue::Immediate(llptr) => Some((llptr, None)),
             OperandValue::Pair(llptr, llextra) => Some((llptr, Some(llextra))),
-            OperandValue::Ref(_) | OperandValue::ZeroSized | OperandValue::Uninit => None,
+            OperandValue::Ref(_) | OperandValue::ZeroSized => None,
         }
     }
 
@@ -129,7 +123,6 @@ impl<V: CodegenObject> OperandValue<V> {
     #[must_use]
     pub(crate) fn is_expected_variant_for_type<'tcx>(&self, ty: TyAndLayout<'tcx>) -> bool {
         match (self, ty.backend_repr) {
-            (OperandValue::Uninit, _) => true,
             (OperandValue::ZeroSized, BackendRepr::Memory { .. }) => ty.is_zst(),
             (OperandValue::Ref(_), BackendRepr::Memory { .. }) => !ty.is_zst(),
             (
@@ -404,9 +397,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
             );
         }
 
-        let val = if let OperandValue::Uninit = self.val {
-            OperandValue::Uninit
-        } else if field.is_zst() {
+        let val = if field.is_zst() {
             OperandValue::ZeroSized
         } else if field.size == self.layout.size {
             assert_eq!(offset.bytes(), 0);
@@ -505,7 +496,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         // Read the tag/niche-encoded discriminant from memory.
         let tag_op = match self.val {
             OperandValue::ZeroSized => bug!(),
-            OperandValue::Uninit => return bx.cx().const_poison(cast_to),
             OperandValue::Immediate(_) | OperandValue::Pair(_, _) => {
                 self.extract_field(fx, bx, tag_field.as_usize())
             }
@@ -788,15 +778,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandRefBuilder<'tcx, V> {
         field: FieldIdx,
         field_operand: OperandRef<'tcx, V>,
     ) {
-        if matches!(field_operand.val, OperandValue::ZeroSized | OperandValue::Uninit) {
+        if let OperandValue::ZeroSized = field_operand.val {
             // A ZST never adds any state, so just ignore it.
             // This special-casing is worth it because of things like
             // `Result<!, !>` where `Ok(never)` is legal to write,
             // but the type shows as FieldShape::Primitive so we can't
             // actually look at the layout for the field being set.
-            //
-            // Likewise, an uninit field does not contribute any value;
-            // the builder's unset slots will produce `const_undef` in `build()`.
             return;
         }
 
@@ -1032,10 +1019,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
                 // Avoid generating stores of zero-sized values, because the only way to have a
                 // zero-sized value is through `undef`/`poison`, and the store itself is useless.
             }
-            OperandValue::Uninit => {
-                // Storing an entirely uninit value is a no-op: the destination is left
-                // uninitialized, which is valid since the value itself is uninit.
-            }
             OperandValue::Ref(val) => {
                 assert!(dest.layout.is_sized(), "cannot directly store unsized values");
                 if val.llextra.is_some() {

compiler/rustc_codegen_ssa/src/mir/retag.rs

@@ -298,9 +298,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                                 builder.update_imm(offset, fst);
                                 builder.update_imm(offset + Size::from_bytes(1), snd)
                             }
-                            OperandValue::Uninit => {
-                                unreachable!("load_operand never produces Uninit")
-                            }
                         }
                     }
                 }

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -91,6 +91,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         true
     }
 
+    fn is_entirely_uninit_const(&self, operand: &mir::Operand<'tcx>) -> bool {
+        let mir::Operand::Constant(const_op) = operand else { return false };
+        self.eval_mir_constant(const_op).all_bytes_uninit(self.cx.tcx())
+    }
+
     #[instrument(level = "trace", skip(self, bx))]
     pub(crate) fn codegen_rvalue(
         &mut self,
@@ -100,6 +105,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ) {
         match *rvalue {
             mir::Rvalue::Use(ref operand, with_retag) => {
+                if self.is_entirely_uninit_const(operand) {
+                    return;
+                }
                 let cg_operand = self.codegen_operand(bx, operand);
                 // Crucially, we do *not* use `OperandValue::Ref` for types with
                 // `BackendRepr::Scalar | BackendRepr::ScalarPair`. This ensures we match the MIR
@@ -170,9 +178,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     OperandValue::ZeroSized => {
                         bug!("unsized coercion on a ZST rvalue");
                     }
-                    OperandValue::Uninit => {
-                        bug!("unsized coercion on an uninit rvalue");
-                    }
                 }
             }
 
@@ -191,12 +196,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     return;
                 }
 
-                let cg_elem = self.codegen_operand(bx, elem);
-
-                if let OperandValue::Uninit = cg_elem.val {
+                // When the element is a const with all bytes uninit, emit a single memset that
+                // writes undef to the entire destination.
+                if self.is_entirely_uninit_const(elem) {
+                    let size = bx.const_usize(dest.layout.size.bytes());
+                    bx.memset(
+                        dest.val.llval,
+                        bx.const_undef(bx.type_i8()),
+                        size,
+                        dest.val.align,
+                        MemFlags::empty(),
+                    );
                     return;
                 }
 
+                let cg_elem = self.codegen_operand(bx, elem);
+
                 let try_init_all_same = |bx: &mut Bx, v| {
                     let start = dest.val.llval;
                     let size = bx.const_usize(dest.layout.size.bytes());
@@ -254,6 +269,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     assert_eq!(operands.len(), 1);
                 }
                 for (i, operand) in operands.iter_enumerated() {
+                    // Do not generate stores for entirely uninit constant fields, for the same
+                    // reason as in `Rvalue::Use` above.
+                    if self.is_entirely_uninit_const(operand) {
+                        continue;
+                    }
                     let op = self.codegen_operand(bx, operand);
                     // Do not generate stores and GEPis for zero-sized fields.
                     if !op.layout.is_zst() {
@@ -356,7 +376,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let cx = bx.cx();
         match (operand.val, operand.layout.backend_repr, cast.backend_repr) {
             _ if cast.is_zst() => OperandValue::ZeroSized,
-            (OperandValue::Uninit, _, _) => OperandValue::Uninit,
             (OperandValue::Ref(source_place_val), abi::BackendRepr::Memory { .. }, _) => {
                 assert_eq!(source_place_val.llextra, None);
                 // The existing alignment is part of `source_place_val`,