Use unions for uninhabitedness checking rather than mem::transmute

Author: varkor

src/librustc/ty/layout.rs

@@ -557,7 +557,7 @@ impl<'a, 'tcx> LayoutCx<'tcx, TyCtxt<'a, 'tcx, 'tcx>> {
                 let size = element.size.checked_mul(count, dl)
                     .ok_or(LayoutError::SizeOverflow(ty))?;
 
-                let abi = if size != Size::ZERO && ty.conservative_is_uninhabited(tcx) {
+                let abi = if count != 0 && ty.conservative_is_uninhabited(tcx) {
                     Abi::Uninhabited
                 } else {
                     Abi::Aggregate { sized: true }

src/test/ui/consts/const-eval/ub-uninhabit.rs

@@ -1,13 +1,3 @@
-// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
 #![feature(const_transmute)]
 #![allow(const_err)] // make sure we cannot allow away the errors tested here
 
@@ -16,14 +6,18 @@ use std::mem;
 #[derive(Copy, Clone)]
 enum Bar {}
 
-const BAD_BAD_BAD: Bar = unsafe { mem::transmute(()) };
-//~^ ERROR it is undefined behavior to use this value
+union TransmuteUnion<A: Clone + Copy, B: Clone + Copy> {
+    a: A,
+    b: B,
+}
+
+const BAD_BAD_BAD: Bar = unsafe { (TransmuteUnion::<(), Bar> { a: () }).b };
+//~^ ERROR this constant likely exhibits undefined behavior
 
 const BAD_BAD_REF: &Bar = unsafe { mem::transmute(1usize) };
 //~^ ERROR it is undefined behavior to use this value
 
-const BAD_BAD_ARRAY: [Bar; 1] = unsafe { mem::transmute(()) };
-//~^ ERROR it is undefined behavior to use this value
+const BAD_BAD_ARRAY: [Bar; 1] = unsafe { (TransmuteUnion::<(), [Bar; 1]> { a: () }).b };
+//~^ ERROR this constant likely exhibits undefined behavior
 
-fn main() {
-}
+fn main() {}

src/test/ui/consts/const-eval/ub-uninhabit.stderr

@@ -1,12 +1,10 @@
 error[E0080]: it is undefined behavior to use this value
   --> $DIR/ub-uninhabit.rs:19:1
    |
-LL | const BAD_BAD_BAD: Bar = unsafe { mem::transmute(()) };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^------------------^^^
-   |                                   |
-   |                                   entered unreachable code
+LL | const BAD_BAD_BAD: Bar = unsafe { (TransmuteUnion::<(), Bar> { a: () }).b };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a value of an uninhabited type
    |
-   = note: #[deny(const_err)] on by default
+   = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rust compiler repository if you believe it should not be considered undefined behavior
 
 error[E0080]: it is undefined behavior to use this value
   --> $DIR/ub-uninhabit.rs:22:1
@@ -19,10 +17,10 @@ LL | const BAD_BAD_REF: &Bar = unsafe { mem::transmute(1usize) };
 error[E0080]: it is undefined behavior to use this value
   --> $DIR/ub-uninhabit.rs:25:1
    |
-LL | const BAD_BAD_ARRAY: [Bar; 1] = unsafe { mem::transmute(()) };
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^------------------^^^
-   |                                          |
-   |                                          entered unreachable code
+LL | const BAD_BAD_ARRAY: [Bar; 1] = unsafe { (TransmuteUnion::<(), [Bar; 1]> { a: () }).b };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a value of an uninhabited type
+   |
+   = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rust compiler repository if you believe it should not be considered undefined behavior
 
 error: aborting due to 3 previous errors