[Sema] Report warning when mix packoffset with nonpackoffset (microsoft#6553)

Check mix packoffset elements with nonpackoffset elements in a cbuffer
at Sema::ActOnFinishHLSLBuffer. Also check packoffset for aggregate
type.

Fixes microsoft#3724

Author: python3kgae

tools/clang/include/clang/Basic/DiagnosticSemaKinds.td

@@ -7544,6 +7544,9 @@ def err_hlsl_objectintemplateargument : Error<
   "%0 is an object and cannot be used as a type parameter">;
 def err_hlsl_packoffset_requires_cbuffer : Error<
   "packoffset is only allowed in a constant buffer">;
+def warn_hlsl_packoffset_mix : Warning<
+  "cannot mix packoffset elements with nonpackoffset elements in a cbuffer">;
+def err_hlsl_packoffset_overlap : Error<"packoffset overlap between %0, %1">;
 def err_hlsl_register_semantics_conflicting : Error<
   "conflicting register semantics">;
 def err_hlsl_register_or_offset_bind_not_valid: Error<

tools/clang/lib/Sema/SemaHLSL.cpp

@@ -11012,6 +11012,10 @@ void hlsl::DiagnosePackingOffset(clang::Sema *self, SourceLocation loc,
         self->Diag(loc, diag::err_hlsl_register_or_offset_bind_not_valid);
       }
     }
+    if (hlsl::IsMatrixType(self, type) || type->isArrayType() ||
+        type->isStructureType()) {
+      self->Diag(loc, diag::err_hlsl_register_or_offset_bind_not_valid);
+    }
   }
 }
 
@@ -14019,8 +14023,66 @@ Decl *Sema::ActOnStartHLSLBuffer(
 void Sema::ActOnFinishHLSLBuffer(Decl *Dcl, SourceLocation RBrace) {
   DXASSERT_NOMSG(Dcl != nullptr);
   DXASSERT(Dcl == HLSLBuffers.back(), "otherwise push/pop is incorrect");
-  dyn_cast<HLSLBufferDecl>(Dcl)->setRBraceLoc(RBrace);
+  auto *BufDecl = cast<HLSLBufferDecl>(Dcl);
+  BufDecl->setRBraceLoc(RBrace);
   HLSLBuffers.pop_back();
+
+  // Validate packoffset.
+  llvm::SmallVector<std::pair<VarDecl *, unsigned>, 4> PackOffsetVec;
+  bool HasPackOffset = false;
+  bool HasNonPackOffset = false;
+  for (auto *Field : BufDecl->decls()) {
+    VarDecl *Var = dyn_cast<VarDecl>(Field);
+    if (!Var)
+      continue;
+
+    unsigned Offset = UINT_MAX;
+
+    for (const hlsl::UnusualAnnotation *it : Var->getUnusualAnnotations()) {
+      if (it->getKind() == hlsl::UnusualAnnotation::UA_ConstantPacking) {
+        const hlsl::ConstantPacking *packOffset =
+            cast<hlsl::ConstantPacking>(it);
+        unsigned CBufferOffset = packOffset->Subcomponent << 2;
+        CBufferOffset += packOffset->ComponentOffset;
+        // Change to bits.
+        Offset = CBufferOffset << 5;
+        HasPackOffset = true;
+      }
+    }
+    PackOffsetVec.emplace_back(Var, Offset);
+    if (Offset == UINT_MAX) {
+      HasNonPackOffset = true;
+    }
+  }
+
+  if (HasPackOffset && HasNonPackOffset) {
+    Diag(BufDecl->getLocation(), diag::warn_hlsl_packoffset_mix);
+  } else if (HasPackOffset) {
+    // Make sure no overlap in packoffset.
+    llvm::SmallDenseMap<VarDecl *, std::pair<unsigned, unsigned>>
+        PackOffsetRanges;
+    for (auto &Pair : PackOffsetVec) {
+      VarDecl *Var = Pair.first;
+      unsigned Size = Context.getTypeSize(Var->getType());
+      unsigned Begin = Pair.second;
+      unsigned End = Begin + Size;
+      for (auto &Range : PackOffsetRanges) {
+        VarDecl *OtherVar = Range.first;
+        unsigned OtherBegin = Range.second.first;
+        unsigned OtherEnd = Range.second.second;
+        if (Begin < OtherEnd && OtherBegin < Begin) {
+          Diag(Var->getLocation(), diag::err_hlsl_packoffset_overlap)
+              << Var << OtherVar;
+          break;
+        } else if (OtherBegin < End && Begin < OtherBegin) {
+          Diag(Var->getLocation(), diag::err_hlsl_packoffset_overlap)
+              << Var << OtherVar;
+          break;
+        }
+      }
+      PackOffsetRanges[Var] = std::make_pair(Begin, End);
+    }
+  }
   PopDeclContext();
 }
 

tools/clang/test/CodeGenSPIRV/vk.layout.cbuffer.packoffset.error.hlsl

@@ -1,7 +1,7 @@
 // RUN: not %dxc -T vs_6_0 -E main -fcgl  %s -spirv  2>&1 | FileCheck %s
 
 cbuffer MyCBuffer {
-    float4 data1;
+    float4 data1 : packoffset(c0);
     float4 data2 : packoffset(c2);
     float  data3 : packoffset(c0);    // error: overlap
     float  data4 : packoffset(c10.z);

tools/clang/test/CodeGenSPIRV/vk.layout.cbuffer.packoffset.hlsl

@@ -14,11 +14,11 @@ struct S {
 };
 
 cbuffer MyCBuffer {                    // Offset
-    float4   data1;                    //                  0
+    float4   data1 : packoffset(c0);   //                  0
     float4   data2 : packoffset(c2);   // 2 * 16         = 32
     float    data3 : packoffset(c3.y); // 3 * 16 + 1 * 4 = 52
     float    data4 : packoffset(c3.z); // 3 * 16 + 2 * 4 = 56
-    float    data5;                    //                  60
+    float    data5 : packoffset(c3.w); //                  60
     float4   data6 : packoffset(c100); // 100 * 16       = 1600
     float2x3 data7 : packoffset(c110); // 110 * 16       = 1760
     S        data8 : packoffset(c150); // 150 * 16       = 2400

tools/clang/test/SemaHLSL/attributes/packoffset-invalid.hlsl

@@ -0,0 +1,57 @@
+// RUN: %dxc -E main -T lib_6_6 %s -verify
+
+// expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
+cbuffer Mix
+{
+    float4 M1 : packoffset(c0);
+    float M2;
+    float M3 : packoffset(c1.y);
+}
+
+// expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
+cbuffer Mix2
+{
+    float4 M4;
+    float M5 : packoffset(c1.y);
+    float M6 ;
+}
+
+// expected-error@+1{{packoffset is only allowed in a constant buffer}}
+float4 g : packoffset(c0);
+
+cbuffer IllegalOffset
+{
+    // expected-error@+1{{register type is unsupported - available types are 'b', 'c', 'i', 's', 't', 'u'}}
+    float4 i1 : packoffset(t2);
+    // expected-error@+1{{packoffset component should indicate offset with one of x, y, z, w, r, g, b, or a}}
+    float i2 : packoffset(c1.m);
+}
+
+cbuffer Overlap
+{
+    float4 o1 : packoffset(c0);
+    // expected-error@+1{{packoffset overlap between 'o2', 'o1'}}
+    float2 o2 : packoffset(c0.z);
+}
+
+cbuffer CrossReg
+{
+    // expected-error@+1{{register or offset bind not valid}}
+    float4 c1 : packoffset(c0.y);
+    // expected-error@+1{{register or offset bind not valid}}
+    float2 c2 : packoffset(c1.w);
+}
+
+struct ST {
+  float s;
+};
+
+cbuffer Aggregate
+{
+    // expected-error@+1{{register or offset bind not valid}}
+    ST A1 : packoffset(c0.y);
+    // expected-error@+1{{register or offset bind not valid}}
+    float A2[2] : packoffset(c1.w);
+    // expected-error@+1{{register or offset bind not valid}}
+    float2x1 m2 : packoffset(c12.z);
+}

tools/clang/test/SemaHLSL/packreg.hlsl

@@ -575,7 +575,7 @@ cbuffer MyBuffer2
 Texture2D<float4> Texture : register(t0);
 Texture2D<float4> Texture_ : register(t0);
 sampler Sampler : register(s0);
-
+// expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
 cbuffer Parameters : register(b0)
 {
   float4   DiffuseColor   : packoffset(c0) : register(c0);

tools/clang/test/SemaHLSL/semantics.hlsl

@@ -6,7 +6,7 @@ float g_foo1 : foo;
 float g_foo2 : foo : fubar;
 float g_foo3 : foo : fubar : register(c3);
 float g_foo4 : register(c4) : foo : fubar;
-
+// expected-warning@+1{{cannot mix packoffset elements with nonpackoffset elements in a cbuffer}}
 cbuffer CBInit : register(b2)
 {
     float g2_foo1 : foo;                                    /* fxc-error {{X3530: cannot mix packoffset elements with nonpackoffset elements in a cbuffer}} */