[X86] Error on using h-registers with REX prefix in the assembler instead of leaving it to a fatal error in the encoder.

Using a fatal error is bad for user experience.

Reviewed By: pengfei

Differential Revision: https://reviews.llvm.org/D89837

Author: topperc

llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp

@@ -1210,8 +1210,6 @@ bool X86AsmParser::MatchRegisterByName(unsigned &RegNo, StringRef RegName,
     // FIXME: This should be done using Requires<Not64BitMode> and
     // Requires<In64BitMode> so "eiz" usage in 64-bit instructions can be also
     // checked.
-    // FIXME: Check AH, CH, DH, BH cannot be used in an instruction requiring a
-    // REX prefix.
     if (RegNo == X86::RIZ || RegNo == X86::RIP ||
         X86MCRegisterClasses[X86::GR64RegClassID].contains(RegNo) ||
         X86II::isX86_64NonExtLowByteReg(RegNo) ||
@@ -3619,6 +3617,33 @@ bool X86AsmParser::validateInstruction(MCInst &Inst, const OperandVector &Ops) {
   }
   }
 
+  const MCInstrDesc &MCID = MII.get(Inst.getOpcode());
+  // Check that we aren't mixing AH/BH/CH/DH with REX prefix. We only need to
+  // check this with the legacy encoding, VEX/EVEX/XOP don't use REX.
+  if ((MCID.TSFlags & X86II::EncodingMask) == 0) {
+    MCPhysReg HReg = X86::NoRegister;
+    bool UsesRex = MCID.TSFlags & X86II::REX_W;
+    unsigned NumOps = Inst.getNumOperands();
+    for (unsigned i = 0; i != NumOps; ++i) {
+      const MCOperand &MO = Inst.getOperand(i);
+      if (!MO.isReg())
+        continue;
+      unsigned Reg = MO.getReg();
+      if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
+        HReg = Reg;
+      if (X86II::isX86_64NonExtLowByteReg(Reg) ||
+          X86II::isX86_64ExtendedReg(Reg))
+        UsesRex = true;
+    }
+
+    if (UsesRex && HReg != X86::NoRegister) {
+      StringRef RegName = X86IntelInstPrinter::getRegisterName(HReg);
+      return Error(Ops[0]->getStartLoc(),
+                   "can't encode '" + RegName + "' in an instruction requiring "
+                   "REX prefix.");
+    }
+  }
+
   return false;
 }
 
@@ -3989,6 +4014,8 @@ bool X86AsmParser::MatchAndEmitATTInstruction(SMLoc IDLoc, unsigned &Opcode,
   unsigned NumSuccessfulMatches =
       std::count(std::begin(Match), std::end(Match), Match_Success);
   if (NumSuccessfulMatches == 1) {
+    if (!MatchingInlineAsm && validateInstruction(Inst, Operands))
+      return true;
     // Some instructions need post-processing to, for example, tweak which
     // encoding is selected. Loop on it while changes happen so the
     // individual transformations can chain off each other.

llvm/test/MC/X86/encoder-fail.s

@@ -1,3 +1,16 @@
-// RUN: not --crash llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2>&1 | FileCheck %s
-// CHECK: LLVM ERROR: Cannot encode high byte register in REX-prefixed instruction
- movzx %dh, %rsi
+// RUN: not llvm-mc -triple x86_64-unknown-unknown --show-encoding %s 2>&1 | FileCheck %s
+
+// CHECK: error: can't encode 'dh' in an instruction requiring REX prefix.
+movzx %dh, %rsi
+
+// CHECK: error: can't encode 'ah' in an instruction requiring REX prefix.
+movzx %ah, %r8d
+
+// CHECK: error: can't encode 'bh' in an instruction requiring REX prefix.
+add %bh, %sil
+
+// CHECK: error: can't encode 'ch' in an instruction requiring REX prefix.
+mov %ch, (%r8)
+
+// CHECK: error: can't encode 'dh' in an instruction requiring REX prefix.
+mov %dh, (%rax,%r8)