Also skip downcasts if the size of the enum matches the transmute target type

Author: oli-obk

compiler/rustc_mir_transform/src/gvn.rs

@@ -1556,9 +1556,19 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
                 && let Value::Projection(field_value, ProjectionElem::Field(field_idx, ())) =
                     self.get(value)
             {
-                if let Value::Projection(_downcast_value, ProjectionElem::Downcast(_, _variant_idx)) =
+                if let Value::Projection(downcast_value, ProjectionElem::Downcast(_, _variant_idx)) =
                     self.get(field_value)
+                    && let downcast_ty = self.ty(downcast_value)
+                    && let Ok(downcast_layout) = self.ecx.layout_of(downcast_ty)
+                    && let Ok(projected_layout) = self.ecx.layout_of(self.ty(value))
+                    && downcast_layout.size == projected_layout.size
                 {
+                    from = downcast_ty;
+                    value = downcast_value;
+                    was_updated_this_iteration = true;
+                    if projected_layout.ty == to {
+                        return Some(value);
+                    }
                 } else if let Some((f_idx, field_ty)) =
                     self.value_is_all_in_one_field(self.ty(field_value), FIRST_VARIANT)
                 {

tests/mir-opt/const_prop/transmute.option_field.GVN.32bit.diff

@@ -23,7 +23,7 @@
           StorageLive(_4);
           _4 = copy _3;
 -         _0 = move _4 as *const () (Transmute);
-+         _0 = copy _3 as *const () (Transmute);
++         _0 = copy _1 as *const () (Transmute);
           StorageDead(_4);
 -         StorageDead(_3);
 +         nop;

tests/mir-opt/const_prop/transmute.option_field.GVN.64bit.diff

@@ -23,7 +23,7 @@
           StorageLive(_4);
           _4 = copy _3;
 -         _0 = move _4 as *const () (Transmute);
-+         _0 = copy _3 as *const () (Transmute);
++         _0 = copy _1 as *const () (Transmute);
           StorageDead(_4);
 -         StorageDead(_3);
 +         nop;

tests/mir-opt/const_prop/transmute.rs

@@ -88,7 +88,7 @@ pub unsafe fn unreachable_box() -> ! {
 pub unsafe fn option_field(x: Option<std::ptr::NonNull<()>>) -> *const () {
     // CHECK-LABEL: fn option_field(
     // CHECK: _3 = copy ((_1 as Some).0: std::ptr::NonNull<()>)
-    // CHECK: _0 = copy _3 as *const () (Transmute)
+    // CHECK: _0 = copy _1 as *const () (Transmute)
     if let Some(x) = x { unsafe { transmute(x) } } else { 0 as *const () }
 }
 

tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-abort.diff

@@ -69,7 +69,8 @@
           StorageLive(_3);
           StorageLive(_4);
           StorageLive(_5);
-          StorageLive(_6);
+-         StorageLive(_6);
++         nop;
           StorageLive(_7);
 -         _7 = copy _1;
 -         _6 = std::alloc::Global::alloc_impl_runtime(move _7, const false) -> [return: bb4, unwind unreachable];
@@ -93,11 +94,14 @@
       }
 
       bb6: {
-          _5 = move ((_6 as Ok).0: std::ptr::NonNull<[u8]>);
+-         _5 = move ((_6 as Ok).0: std::ptr::NonNull<[u8]>);
++         _5 = copy ((_6 as Ok).0: std::ptr::NonNull<[u8]>);
           StorageDead(_14);
           StorageDead(_10);
-          StorageDead(_6);
-          _4 = copy _5 as *mut [u8] (Transmute);
+-         StorageDead(_6);
+-         _4 = copy _5 as *mut [u8] (Transmute);
++         nop;
++         _4 = copy _6 as *mut [u8] (Transmute);
           StorageDead(_5);
           _3 = copy _4 as *mut u8 (PtrToPtr);
           StorageDead(_4);

tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-abort.diff

@@ -69,7 +69,8 @@
           StorageLive(_3);
           StorageLive(_4);
           StorageLive(_5);
-          StorageLive(_6);
+-         StorageLive(_6);
++         nop;
           StorageLive(_7);
 -         _7 = copy _1;
 -         _6 = std::alloc::Global::alloc_impl_runtime(move _7, const false) -> [return: bb4, unwind unreachable];
@@ -93,11 +94,14 @@
       }
 
       bb6: {
-          _5 = move ((_6 as Ok).0: std::ptr::NonNull<[u8]>);
+-         _5 = move ((_6 as Ok).0: std::ptr::NonNull<[u8]>);
++         _5 = copy ((_6 as Ok).0: std::ptr::NonNull<[u8]>);
           StorageDead(_14);
           StorageDead(_10);
-          StorageDead(_6);
-          _4 = copy _5 as *mut [u8] (Transmute);
+-         StorageDead(_6);
+-         _4 = copy _5 as *mut [u8] (Transmute);
++         nop;
++         _4 = copy _6 as *mut [u8] (Transmute);
           StorageDead(_5);
           _3 = copy _4 as *mut u8 (PtrToPtr);
           StorageDead(_4);