[SM6.10][LinAlg] Impl MatrixRef, CreateMatrix builtins

docs/DXIL.rst

@@ -2426,6 +2426,7 @@ ID  Name                                                  Description
 309 VectorReduceAnd                                       Bitwise AND reduction of the vector returning a scalar
 310 VectorReduceOr                                        Bitwise OR reduction of the vector returning a scalar
 311 FDot                                                  computes the n-dimensional vector dot-product
+312 CreateMatrix                                          creates the handle to a matrix
 === ===================================================== =======================================================================================================================================================================================================================
 
 

include/dxc/DXIL/DxilConstants.h

@@ -754,6 +754,7 @@ enum class OpCode : unsigned {
       160, // create resource handle from resource struct for library
 
   // Linear Algebra Operations
+  CreateMatrix = 312, // creates the handle to a matrix
   MatVecMul =
       305, // Multiplies a MxK dimension matrix and a K sized input vector
   MatVecMulAdd = 306, // multiplies a MxK dimension matrix and a K sized input
@@ -1089,7 +1090,7 @@ enum class OpCode : unsigned {
   NumOpCodes_Dxil_1_8 = 258,
   NumOpCodes_Dxil_1_9 = 312,
 
-  NumOpCodes = 312 // exclusive last value of enumeration
+  NumOpCodes = 313 // exclusive last value of enumeration
 };
 // OPCODE-ENUM:END
 
@@ -1232,6 +1233,7 @@ enum class OpCodeClass : unsigned {
   CreateHandleForLib,
 
   // Linear Algebra Operations
+  CreateMatrix,
   MatVecMul,
   MatVecMulAdd,
   OuterProductAccumulate,
@@ -1425,7 +1427,7 @@ enum class OpCodeClass : unsigned {
   NumOpClasses_Dxil_1_8 = 174,
   NumOpClasses_Dxil_1_9 = 196,
 
-  NumOpClasses = 196 // exclusive last value of enumeration
+  NumOpClasses = 197 // exclusive last value of enumeration
 };
 // OPCODECLASS-ENUM:END
 

include/dxc/DXIL/DxilInstructions.h

@@ -10231,5 +10231,25 @@ struct DxilInst_FDot {
   llvm::Value *get_b() const { return Instr->getOperand(2); }
   void set_b(llvm::Value *val) { Instr->setOperand(2, val); }
 };
+
+/// This instruction creates the handle to a matrix
+struct DxilInst_CreateMatrix {
+  llvm::Instruction *Instr;
+  // Construction and identification
+  DxilInst_CreateMatrix(llvm::Instruction *pInstr) : Instr(pInstr) {}
+  operator bool() const {
+    return hlsl::OP::IsDxilOpFuncCallInst(Instr,
+                                          hlsl::OP::OpCode::CreateMatrix);
+  }
+  // Validation support
+  bool isAllowed() const { return true; }
+  bool isArgumentListValid() const {
+    if (1 != llvm::dyn_cast<llvm::CallInst>(Instr)->getNumArgOperands())
+      return false;
+    return true;
+  }
+  // Metadata
+  bool requiresUniformInputs() const { return false; }
+};
 // INSTR-HELPER:END
 } // namespace hlsl

include/dxc/DXIL/DxilOperations.h

@@ -83,6 +83,7 @@ class OP {
   llvm::LLVMContext &GetCtx() { return m_Ctx; }
   llvm::Module *GetModule() { return m_pModule; }
   llvm::Type *GetHandleType() const;
+  llvm::Type *GetMatrixRefType() const;
   llvm::Type *GetHitObjectType() const;
   llvm::Type *GetNodeHandleType() const;
   llvm::Type *GetNodeRecordHandleType() const;
@@ -188,6 +189,7 @@ class OP {
   llvm::Type *m_pSplitDoubleType;
   llvm::Type *m_pFourI32Type;
   llvm::Type *m_pFourI16Type;
+  llvm::Type *m_pMatrixRefType;
 
   DXIL::LowPrecisionMode m_LowPrecisionMode;
 

include/dxc/DXIL/DxilUtil.h

@@ -164,6 +164,8 @@ bool IsHLSLObjectType(llvm::Type *Ty);
 bool IsHLSLRayQueryType(llvm::Type *Ty);
 llvm::Type *GetHLSLHitObjectType(llvm::Module *M);
 bool IsHLSLHitObjectType(llvm::Type *Ty);
+llvm::Type *GetHLSLMatrixRefType(llvm::Module *M);
+bool IsHLSLMatrixRefType(llvm::Type *Ty);
 bool IsHLSLResourceDescType(llvm::Type *Ty);
 bool IsResourceSingleComponent(llvm::Type *Ty);
 uint8_t GetResourceComponentCount(llvm::Type *Ty);

include/dxc/HlslIntrinsicOp.h

@@ -111,6 +111,7 @@ enum class IntrinsicOp {
   IOP___builtin_MatVecMulAdd = 391,
   IOP___builtin_OuterProductAccumulate = 392,
   IOP___builtin_VectorAccumulate = 393,
+  IOP___builtin_la_CreateMatrix = 395,
   IOP_abort = 102,
   IOP_abs = 103,
   IOP_acos = 104,
@@ -401,7 +402,7 @@ enum class IntrinsicOp {
   IOP_usign = 355,
   MOP_InterlockedUMax = 356,
   MOP_InterlockedUMin = 357,
-  Num_Intrinsics = 395,
+  Num_Intrinsics = 396,
 };
 inline bool HasUnsignedIntrinsicOpcode(IntrinsicOp opcode) {
   switch (opcode) {

include/dxc/dxcapi.internal.h

@@ -130,15 +130,17 @@ enum LEGAL_INTRINSIC_COMPTYPES {
   LICOMPTYPE_HIT_OBJECT = 51,
   LICOMPTYPE_RAY_QUERY = 52,
 
-  LICOMPTYPE_LINALG = 53, // f32, partial-precision-f32, f16,
+  LICOMPTYPE_MATRIX_REF = 53,
+
+  LICOMPTYPE_LINALG = 54, // f32, partial-precision-f32, f16,
                           // i32, i16, u32, u16,
                           // int8_4packed, uint8_4packed
 
 #ifdef ENABLE_SPIRV_CODEGEN
-  LICOMPTYPE_VK_BUFFER_POINTER = 54,
-  LICOMPTYPE_COUNT = 55
+  LICOMPTYPE_VK_BUFFER_POINTER = 55,
+  LICOMPTYPE_COUNT = 56
 #else
-  LICOMPTYPE_COUNT = 54
+  LICOMPTYPE_COUNT = 55
 #endif
 };
 

lib/DXIL/DxilOperations.cpp

@@ -2714,6 +2714,16 @@ const OP::OpCodeProperty OP::m_OpCodeProps[(unsigned)OP::OpCode::NumOpCodes] = {
      1,
      {{0x400}},
      {{0x3}}}, // Overloads: <hf
+
+    // Linear Algebra Operations
+    {OC::CreateMatrix,
+     "CreateMatrix",
+     OCC::CreateMatrix,
+     "createMatrix",
+     Attribute::None,
+     0,
+     {},
+     {}}, // Overloads: v
 };
 // OPCODE-OLOADS:END
 
@@ -3536,8 +3546,8 @@ void OP::GetMinShaderModelAndMask(OpCode C, bool bWithTranslation,
     return;
   }
   // Instructions: MatVecMul=305, MatVecMulAdd=306, OuterProductAccumulate=307,
-  // VectorAccumulate=308
-  if ((305 <= op && op <= 308)) {
+  // VectorAccumulate=308, CreateMatrix=312
+  if ((305 <= op && op <= 308) || op == 312) {
     major = 6;
     minor = 10;
     return;
@@ -3705,6 +3715,9 @@ OP::OP(LLVMContext &Ctx, Module *pModule)
                            Type::getInt16Ty(m_Ctx)}; // HiHi, HiLo, LoHi, LoLo
   m_pFourI16Type =
       GetOrCreateStructType(m_Ctx, FourI16Types, "dx.types.fouri16", pModule);
+
+  m_pMatrixRefType = GetOrCreateStructType(m_Ctx, Type::getInt8PtrTy(m_Ctx),
+                                           "dx.types.MatrixRef", pModule);
 }
 
 void OP::RefreshCache() {
@@ -3816,6 +3829,7 @@ Function *OP::GetOpFunc(OpCode opCode, Type *pOverloadType) {
   Type *pETy = pOverloadType;
   Type *pRes = GetHandleType();
   Type *pNodeHandle = GetNodeHandleType();
+  Type *pMatrixRef = GetMatrixRefType();
   Type *pNodeRecordHandle = GetNodeRecordHandleType();
   Type *pDim = GetDimensionsType();
   Type *pPos = GetSamplePosType();
@@ -6044,6 +6058,12 @@ Function *OP::GetOpFunc(OpCode opCode, Type *pOverloadType) {
     A(pETy);
     A(pETy);
     break;
+
+    // Linear Algebra Operations
+  case OpCode::CreateMatrix:
+    A(pMatrixRef);
+    A(pI32);
+    break;
   // OPCODE-OLOAD-FUNCS:END
   default:
     DXASSERT(false, "otherwise unhandled case");
@@ -6335,6 +6355,7 @@ llvm::Type *OP::GetOverloadType(OpCode opCode, llvm::Function *F) {
   case OpCode::ReservedC7:
   case OpCode::ReservedC8:
   case OpCode::ReservedC9:
+  case OpCode::CreateMatrix:
     return Type::getVoidTy(Ctx);
   case OpCode::CheckAccessFullyMapped:
   case OpCode::SampleIndex:
@@ -6472,6 +6493,8 @@ Type *OP::GetHandleType() const { return m_pHandleType; }
 
 Type *OP::GetNodeHandleType() const { return m_pNodeHandleType; }
 
+Type *OP::GetMatrixRefType() const { return m_pMatrixRefType; }
+
 Type *OP::GetHitObjectType() const { return m_pHitObjectType; }
 
 Type *OP::GetNodeRecordHandleType() const { return m_pNodeRecordHandleType; }

lib/DXIL/DxilUtil.cpp

@@ -577,6 +577,9 @@ bool IsHLSLObjectType(llvm::Type *Ty) {
 
     if (IsHLSLHitObjectType(Ty))
       return true;
+
+    if (IsHLSLMatrixRefType(Ty))
+      return true;
   }
   return false;
 }
@@ -612,6 +615,24 @@ bool IsHLSLHitObjectType(llvm::Type *Ty) {
   return ST->getName() == "dx.types.HitObject";
 }
 
+llvm::Type *GetHLSLMatrixRefType(llvm::Module *M) {
+  using namespace llvm;
+  StructType *MatrixRefTy = M->getTypeByName("dx.types.MatrixRef");
+  if (!MatrixRefTy)
+    MatrixRefTy = StructType::create({Type::getInt8PtrTy(M->getContext(), 0)},
+                                     "dx.types.MatrixRef", false);
+  return MatrixRefTy;
+}
+
+bool IsHLSLMatrixRefType(llvm::Type *Ty) {
+  llvm::StructType *ST = dyn_cast<llvm::StructType>(Ty);
+  if (!ST)
+    return false;
+  if (!ST->hasName())
+    return false;
+  return ST->getName() == "dx.types.MatrixRef";
+}
+
 bool IsHLSLResourceDescType(llvm::Type *Ty) {
   if (llvm::StructType *ST = dyn_cast<llvm::StructType>(Ty)) {
     if (!ST->hasName())

lib/HLSL/HLOperationLower.cpp

@@ -6598,7 +6598,10 @@ Value *TranslateSelect(CallInst *CI, IntrinsicOp IOP, OP::OpCode opcode,
 
   return Builder.CreateSelect(cond, t, f);
 }
+} // namespace
 
+// Translate linear algebra intrinsics
+namespace {
 Value *TranslateMatVecMul(CallInst *CI, IntrinsicOp IOP, OP::OpCode OpCode,
                           HLOperationLowerHelper &Helper,
                           HLObjectOperationLowerHelper *ObjHelper,
@@ -6792,6 +6795,22 @@ Value *TranslateVectorAccumulate(CallInst *CI, IntrinsicOp IOP,
                             {OpArg, InputVector, MatrixBuffer, MatrixOffset});
 }
 
+Value *TranslateLACreateMatrix(CallInst *CI, IntrinsicOp IOP, OP::OpCode OpCode,
+                               HLOperationLowerHelper &Helper,
+                               HLObjectOperationLowerHelper *ObjHelper,
+                               bool &Translated) {
+
+  hlsl::OP *HlslOP = &Helper.hlslOP;
+  IRBuilder<> Builder(CI);
+  Value *MatrixRefPtr = CI->getArgOperand(1);
+  Value *MatrixRef = TrivialDxilOperation(
+      OpCode, {nullptr}, Type::getVoidTy(CI->getContext()), CI, HlslOP);
+  Builder.CreateStore(MatrixRef, MatrixRefPtr);
+  DXASSERT(
+      CI->use_empty(),
+      "Default ctor return type is a Clang artifact. Value must not be used");
+  return nullptr;
+}
 } // namespace
 
 // Lower table.
@@ -7513,6 +7532,9 @@ IntrinsicLower gLowerTable[] = {
     {IntrinsicOp::IOP___builtin_VectorAccumulate, TranslateVectorAccumulate,
      DXIL::OpCode::VectorAccumulate},
     {IntrinsicOp::IOP_isnormal, TrivialIsSpecialFloat, DXIL::OpCode::IsNormal},
+
+    {IntrinsicOp::IOP___builtin_la_CreateMatrix, TranslateLACreateMatrix,
+     DXIL::OpCode::CreateMatrix},
 };
 } // namespace
 static_assert(

tools/clang/include/clang/AST/HlslTypes.h

@@ -393,6 +393,7 @@ DeclareConstantBufferViewType(clang::ASTContext &context,
                               clang::InheritableAttr *Attr);
 clang::CXXRecordDecl *DeclareRayQueryType(clang::ASTContext &context);
 clang::CXXRecordDecl *DeclareHitObjectType(clang::NamespaceDecl &NSDecl);
+clang::CXXRecordDecl *DeclareMatrixRefType(clang::ASTContext &context);
 clang::CXXRecordDecl *DeclareResourceType(clang::ASTContext &context,
                                           bool bSampler);
 
@@ -480,6 +481,7 @@ bool IsHLSLDynamicResourceType(clang::QualType type);
 bool IsHLSLDynamicSamplerType(clang::QualType type);
 bool IsHLSLNodeType(clang::QualType type);
 bool IsHLSLHitObjectType(clang::QualType type);
+bool IsHLSLMatrixRefType(clang::QualType type);
 
 bool IsHLSLObjectWithImplicitMemberAccess(clang::QualType type);
 bool IsHLSLObjectWithImplicitROMemberAccess(clang::QualType type);

tools/clang/include/clang/Basic/Attr.td

@@ -1190,6 +1190,14 @@ def HLSLHitObject : InheritableAttr {
   let Documentation = [Undocumented];
 }
 
+// HLSL MatrixRef Attribute
+
+def HLSLMatrixRef : InheritableAttr {
+  let Spellings = []; // No spellings!
+  let Subjects = SubjectList<[CXXRecord]>;
+  let Documentation = [Undocumented];
+}
+
 // HLSL Parameter Attributes
 
 def HLSLMaxRecords : InheritableAttr {

tools/clang/lib/AST/ASTContextHLSL.cpp

@@ -1306,6 +1306,48 @@ CXXRecordDecl *hlsl::DeclareHitObjectType(NamespaceDecl &NSDecl) {
   return RecordDecl;
 }
 
+CXXRecordDecl *hlsl::DeclareMatrixRefType(ASTContext &Context) {
+  // MatrixRef { ... }
+  BuiltinTypeDeclBuilder TypeDeclBuilder(Context.getTranslationUnitDecl(),
+                                         "__builtin_la_MatrixRef");
+  TypeDeclBuilder.startDefinition();
+
+  // Add handle to mark as HLSL object.
+  TypeDeclBuilder.addField("h", GetHLSLObjectHandleType(Context));
+  CXXRecordDecl *RecordDecl = TypeDeclBuilder.getRecordDecl();
+
+  CanQualType canQualType = Context.getCanonicalType(
+      Context.getRecordType(TypeDeclBuilder.getRecordDecl()));
+
+  // Add constructor that will be lowered to IOP___builtin_la_CreateMatrix.
+  CXXConstructorDecl *pConstructorDecl = nullptr;
+  TypeSourceInfo *pTypeSourceInfo = nullptr;
+  CreateConstructorDeclaration(
+      Context, RecordDecl, Context.VoidTy, {},
+      Context.DeclarationNames.getCXXConstructorName(canQualType), false,
+      &pConstructorDecl, &pTypeSourceInfo);
+  RecordDecl->addDecl(pConstructorDecl);
+  pConstructorDecl->addAttr(HLSLIntrinsicAttr::CreateImplicit(
+      Context, "op", "",
+      static_cast<int>(hlsl::IntrinsicOp::IOP___builtin_la_CreateMatrix)));
+  pConstructorDecl->addAttr(HLSLCXXOverloadAttr::CreateImplicit(Context));
+
+  // Add AvailabilityAttribute for SM6.10+
+  VersionTuple VT610 = VersionTuple(6, 10);
+  AvailabilityAttr *AAttr = AvailabilityAttr::CreateImplicit(
+      Context, &Context.Idents.get(""), clang::VersionTuple(6, 10),
+      clang::VersionTuple(), clang::VersionTuple(), false, "");
+  RecordDecl->addAttr(AAttr);
+
+  // Add the implicit HLSLMatrixRefAttr attribute to unambiguously recognize the
+  // builtin HitObject type.
+  RecordDecl->addAttr(HLSLMatrixRefAttr::CreateImplicit(Context));
+  RecordDecl->setImplicit(true);
+
+  // Add to namespace
+  return RecordDecl;
+}
+
 CXXRecordDecl *hlsl::DeclareResourceType(ASTContext &context, bool bSampler) {
   // struct ResourceDescriptor { uint8 desc; }
   StringRef Name = bSampler ? ".Sampler" : ".Resource";

tools/clang/lib/AST/HlslTypes.cpp

@@ -528,6 +528,10 @@ bool IsHLSLHitObjectType(QualType type) {
   return nullptr != getAttr<HLSLHitObjectAttr>(type);
 }
 
+bool IsHLSLMatrixRefType(QualType type) {
+  return nullptr != getAttr<HLSLMatrixRefAttr>(type);
+}
+
 DXIL::NodeIOKind GetNodeIOType(clang::QualType type) {
   if (const HLSLNodeObjectAttr *Attr = getAttr<HLSLNodeObjectAttr>(type))
     return Attr->getNodeIOType();

tools/clang/lib/CodeGen/CodeGenTypes.cpp

@@ -369,6 +369,8 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
                  .getTypePtr();
       } else if (hlsl::IsHLSLHitObjectType(T)) // HLSL Change
         return hlsl::dxilutil::GetHLSLHitObjectType(&TheModule);
+      else if (hlsl::IsHLSLMatrixRefType(T)) // HLSL Change
+        return hlsl::dxilutil::GetHLSLMatrixRefType(&TheModule);
       else
         return ConvertRecordDeclType(RT->getDecl());
     }