[ObjectYAML] Add support for DXContainer HASH

DXContainer files contain a part that has an MD5 of the generated
shader. This adds support to the ObjectYAML tooling to expand the hash
part data and hash iteself in preparation for adding hashing support to
DirectX code generation.

Reviewed By: python3kgae

Differential Revision: https://reviews.llvm.org/D136632

diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index ad921d6..44b77b1 100644 (file)
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -50,6 +50,8 @@ struct ShaderHash {
   uint32_t Flags; // dxbc::HashFlags
   uint8_t Digest[16];
 
+  bool isPopulated();
+
   void swapBytes() { sys::swapByteOrder(Flags); }
 };
 

diff --git a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
index 43afb47..7907bfc 100644 (file)
--- a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
+++ b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
@@ -2,6 +2,7 @@
 #ifdef CONTAINER_PART
 CONTAINER_PART(DXIL)
 CONTAINER_PART(SFI0)
+CONTAINER_PART(HASH)
 
 #undef CONTAINER_PART
 #endif 

diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h
index 91f4688..5f92b30 100644 (file)
--- a/llvm/include/llvm/Object/DXContainer.h
+++ b/llvm/include/llvm/Object/DXContainer.h
@@ -36,11 +36,13 @@ private:
   SmallVector<uint32_t, 4> PartOffsets;
   Optional<DXILData> DXIL;
   Optional<uint64_t> ShaderFlags;
+  Optional<dxbc::ShaderHash> Hash;
 
   Error parseHeader();
   Error parsePartOffsets();
   Error parseDXILHeader(uint32_t Offset);
   Error parseShaderFlags(uint32_t Offset);
+  Error parseHash(uint32_t Offset);
   friend class PartIterator;
 
 public:
@@ -120,6 +122,8 @@ public:
   Optional<DXILData> getDXIL() const { return DXIL; }
 
   Optional<uint64_t> getShaderFlags() const { return ShaderFlags; }
+
+  Optional<dxbc::ShaderHash> getShaderHash() const { return Hash; }
 };
 
 } // namespace object

diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
index cd966fc..49602aa 100644 (file)
--- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
@@ -16,6 +16,7 @@
 #define LLVM_OBJECTYAML_DXCONTAINERYAML_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/DXContainer.h"
 #include "llvm/ObjectYAML/YAML.h"
 #include "llvm/Support/YAMLTraits.h"
 #include <cstdint>
@@ -61,6 +62,14 @@ struct ShaderFlags {
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 };
 
+struct ShaderHash {
+  ShaderHash() = default;
+  ShaderHash(const dxbc::ShaderHash &Data);
+
+  bool IncludesSource;
+  std::vector<llvm::yaml::Hex8> Digest;
+};
+
 struct Part {
   Part() = default;
   Part(std::string N, uint32_t S) : Name(N), Size(S) {}
@@ -68,6 +77,7 @@ struct Part {
   uint32_t Size;
   Optional<DXILProgram> Program;
   Optional<ShaderFlags> Flags;
+  Optional<ShaderHash> Hash;
 };
 
 struct Object {
@@ -101,6 +111,10 @@ template <> struct MappingTraits<DXContainerYAML::ShaderFlags> {
   static void mapping(IO &IO, DXContainerYAML::ShaderFlags &Flags);
 };
 
+template <> struct MappingTraits<DXContainerYAML::ShaderHash> {
+  static void mapping(IO &IO, DXContainerYAML::ShaderHash &Hash);
+};
+
 template <> struct MappingTraits<DXContainerYAML::Part> {
   static void mapping(IO &IO, DXContainerYAML::Part &Version);
 };

diff --git a/llvm/lib/BinaryFormat/DXContainer.cpp b/llvm/lib/BinaryFormat/DXContainer.cpp
index 6704b7b..60a89c6 100644 (file)
--- a/llvm/lib/BinaryFormat/DXContainer.cpp
+++ b/llvm/lib/BinaryFormat/DXContainer.cpp
@@ -15,7 +15,7 @@
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace llvm;
-using namespace llvm;
+using namespace llvm::dxbc;
 
 dxbc::PartType dxbc::parsePartType(StringRef S) {
 #define CONTAINER_PART(PartName) .Case(#PartName, PartType::PartName)
@@ -23,3 +23,8 @@ dxbc::PartType dxbc::parsePartType(StringRef S) {
 #include "llvm/BinaryFormat/DXContainerConstants.def"
       .Default(dxbc::PartType::Unknown);
 }
+
+bool ShaderHash::isPopulated() {
+  static uint8_t Zeros[16] = {0};
+  return Flags > 0 || 0 != memcmp(&Digest, &Zeros, 16);
+}

diff --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp
index d9667db..a83cf70 100644 (file)
--- a/llvm/lib/Object/DXContainer.cpp
+++ b/llvm/lib/Object/DXContainer.cpp
@@ -80,6 +80,17 @@ Error DXContainer::parseShaderFlags(uint32_t Offset) {
   return Error::success();
 }
 
+Error DXContainer::parseHash(uint32_t Offset) {
+  if (Hash)
+    return parseFailed("More than one HASH part is present in the file");
+  const char *Current = Data.getBuffer().data() + Offset;
+  dxbc::ShaderHash ReadHash;
+  if (Error Err = readStruct(Data.getBuffer(), Current, ReadHash))
+    return Err;
+  Hash = ReadHash;
+  return Error::success();
+}
+
 Error DXContainer::parsePartOffsets() {
   const char *Current = Data.getBuffer().data() + sizeof(dxbc::Header);
   for (uint32_t Part = 0; Part < Header.PartCount; ++Part) {
@@ -107,6 +118,10 @@ Error DXContainer::parsePartOffsets() {
       if (Error Err = parseShaderFlags(PartOffset + sizeof(dxbc::PartHeader)))
         return Err;
       break;
+    case dxbc::PartType::HASH:
+      if (Error Err = parseHash(PartOffset + sizeof(dxbc::PartHeader)))
+        return Err;
+      break;
     case dxbc::PartType::Unknown:
       break;
     }

diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
index 32b4f0b..1dc5111 100644 (file)
--- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
@@ -125,7 +125,8 @@ void DXContainerWriter::writeParts(raw_ostream &OS) {
     dxbc::PartType PT = dxbc::parsePartType(P.Name);
 
     uint64_t DataStart = OS.tell();
-    if (PT == dxbc::PartType::DXIL) {
+    switch (PT) {
+    case dxbc::PartType::DXIL: {
       if (!P.Program)
         continue;
       dxbc::ProgramHeader Header;
@@ -167,7 +168,9 @@ void DXContainerWriter::writeParts(raw_ostream &OS) {
         OS.write(reinterpret_cast<char *>(P.Program->DXIL->data()),
                  P.Program->DXIL->size());
       }
-    } else if (PT == dxbc::PartType::SFI0) {
+      break;
+    }
+    case dxbc::PartType::SFI0: {
       // If we don't have any flags we can continue here and the data will be
       // zeroed out.
       if (!P.Flags.has_value())
@@ -176,6 +179,22 @@ void DXContainerWriter::writeParts(raw_ostream &OS) {
       if (sys::IsBigEndianHost)
         sys::swapByteOrder(Flags);
       OS.write(reinterpret_cast<char *>(&Flags), sizeof(uint64_t));
+      break;
+    }
+    case dxbc::PartType::HASH: {
+      if (!P.Hash.has_value())
+        continue;
+      dxbc::ShaderHash Hash = {0, {0}};
+      if (P.Hash->IncludesSource)
+        Hash.Flags |= static_cast<uint32_t>(dxbc::HashFlags::IncludesSource);
+      memcpy(&Hash.Digest[0], &P.Hash->Digest[0], 16);
+      if (sys::IsBigEndianHost)
+        Hash.swapBytes();
+      OS.write(reinterpret_cast<char *>(&Hash), sizeof(dxbc::ShaderHash));
+      break;
+    }
+    case dxbc::PartType::Unknown:
+      break; // Skip any handling for unrecognized parts.
     }
     uint64_t BytesWritten = OS.tell() - DataStart;
     RollingOffset += BytesWritten;

diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
index d9cc7f5..1d1dd42 100644 (file)
--- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
@@ -36,6 +36,13 @@ uint64_t DXContainerYAML::ShaderFlags::getEncodedFlags() {
   return Flag;
 }
 
+DXContainerYAML::ShaderHash::ShaderHash(const dxbc::ShaderHash &Data)
+    : IncludesSource((Data.Flags & static_cast<uint32_t>(
+                                       dxbc::HashFlags::IncludesSource)) != 0),
+      Digest(16, 0) {
+  memcpy(Digest.data(), &Data.Digest[0], 16);
+}
+
 namespace yaml {
 
 void MappingTraits<DXContainerYAML::VersionTuple>::mapping(
@@ -71,12 +78,19 @@ void MappingTraits<DXContainerYAML::ShaderFlags>::mapping(
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
+void MappingTraits<DXContainerYAML::ShaderHash>::mapping(
+    IO &IO, DXContainerYAML::ShaderHash &Hash) {
+  IO.mapRequired("IncludesSource", Hash.IncludesSource);
+  IO.mapRequired("Digest", Hash.Digest);
+}
+
 void MappingTraits<DXContainerYAML::Part>::mapping(IO &IO,
                                                    DXContainerYAML::Part &P) {
   IO.mapRequired("Name", P.Name);
   IO.mapRequired("Size", P.Size);
   IO.mapOptional("Program", P.Program);
   IO.mapOptional("Flags", P.Flags);
+  IO.mapOptional("Hash", P.Hash);
 }
 
 void MappingTraits<DXContainerYAML::Object>::mapping(

diff --git a/llvm/test/tools/obj2yaml/DXContainer/ExplicitSizeAndOffsets.yaml b/llvm/test/tools/obj2yaml/DXContainer/ExplicitSizeAndOffsets.yaml
index f807aef..040aeb0 100644 (file)
--- a/llvm/test/tools/obj2yaml/DXContainer/ExplicitSizeAndOffsets.yaml
+++ b/llvm/test/tools/obj2yaml/DXContainer/ExplicitSizeAndOffsets.yaml
@@ -7,9 +7,9 @@ Header:
   Version:
     Major:           1
     Minor:           0
-  FileSize:        172
+  FileSize:        184
   PartCount:       7
-  PartOffsets:     [ 60, 76, 92, 108, 124, 140, 156 ]
+  PartOffsets:     [ 60, 76, 92, 108, 124, 140, 168 ]
 Parts:
   - Name:            SFI0
     Size:            8
@@ -22,7 +22,7 @@ Parts:
   - Name:            STAT
     Size:            8
   - Name:            HASH
-    Size:            8
+    Size:            20
   - Name:            CXIL
     Size:            8
 ...
@@ -34,9 +34,9 @@ Parts:
 # CHECK-NEXT:   Version:
 # CHECK-NEXT:     Major:           1
 # CHECK-NEXT:     Minor:           0
-# CHECK-NEXT:   FileSize:        172
+# CHECK-NEXT:   FileSize:        184
 # CHECK-NEXT:   PartCount:       7
-# CHECK-NEXT:   PartOffsets:     [ 60, 76, 92, 108, 124, 140, 156 ]
+# CHECK-NEXT:   PartOffsets:     [ 60, 76, 92, 108, 124, 140, 168 ]
 # CHECK-NEXT: Parts:
 # CHECK-NEXT:   - Name:            SFI0
 # CHECK-NEXT:     Size:            8
@@ -49,7 +49,7 @@ Parts:
 # CHECK-NEXT:   - Name:            STAT
 # CHECK-NEXT:     Size:            8
 # CHECK-NEXT:   - Name:            HASH
-# CHECK-NEXT:     Size:            8
+# CHECK-NEXT:     Size:            20
 # CHECK-NEXT:   - Name:            CXIL
 # CHECK-NEXT:     Size:            8
 # CHECK-NEXT: ...

diff --git a/llvm/test/tools/obj2yaml/DXContainer/OmitSizeAndOffsets.yaml b/llvm/test/tools/obj2yaml/DXContainer/OmitSizeAndOffsets.yaml
index 3930afa..62efd41 100644 (file)
--- a/llvm/test/tools/obj2yaml/DXContainer/OmitSizeAndOffsets.yaml
+++ b/llvm/test/tools/obj2yaml/DXContainer/OmitSizeAndOffsets.yaml
@@ -20,7 +20,7 @@ Parts:
   - Name:            STAT
     Size:            8
   - Name:            HASH
-    Size:            8
+    Size:            20
   - Name:            CXIL
     Size:            8
 ...
@@ -32,9 +32,9 @@ Parts:
 # CHECK-NEXT:   Version:
 # CHECK-NEXT:     Major:           1
 # CHECK-NEXT:     Minor:           0
-# CHECK-NEXT:   FileSize:        172
+# CHECK-NEXT:   FileSize:        184
 # CHECK-NEXT:   PartCount:       7
-# CHECK-NEXT:   PartOffsets:     [ 60, 76, 92, 108, 124, 140, 156 ]
+# CHECK-NEXT:   PartOffsets:     [ 60, 76, 92, 108, 124, 140, 168 ]
 # CHECK-NEXT: Parts:
 # CHECK-NEXT:   - Name:            SFI0
 # CHECK-NEXT:     Size:            8
@@ -47,7 +47,7 @@ Parts:
 # CHECK-NEXT:   - Name:            STAT
 # CHECK-NEXT:     Size:            8
 # CHECK-NEXT:   - Name:            HASH
-# CHECK-NEXT:     Size:            8
+# CHECK-NEXT:     Size:            20
 # CHECK-NEXT:   - Name:            CXIL
 # CHECK-NEXT:     Size:            8
 # CHECK-NEXT: ...

diff --git a/llvm/test/tools/obj2yaml/DXContainer/ShaderFlags.yaml b/llvm/test/tools/obj2yaml/DXContainer/ShaderFlags.yaml
index 93e884c..2a5a3ff 100644 (file)
--- a/llvm/test/tools/obj2yaml/DXContainer/ShaderFlags.yaml
+++ b/llvm/test/tools/obj2yaml/DXContainer/ShaderFlags.yaml
@@ -54,7 +54,7 @@ Parts:
   - Name:            STAT
     Size:            8
   - Name:            HASH
-    Size:            8
+    Size:            20
   - Name:            CXIL
     Size:            8
 ...

diff --git a/llvm/test/tools/obj2yaml/DXContainer/ShaderFlagsEmpty.yaml b/llvm/test/tools/obj2yaml/DXContainer/ShaderFlagsEmpty.yaml
index 1bbc5d2..ef73eb8 100644 (file)
--- a/llvm/test/tools/obj2yaml/DXContainer/ShaderFlagsEmpty.yaml
+++ b/llvm/test/tools/obj2yaml/DXContainer/ShaderFlagsEmpty.yaml
@@ -53,7 +53,7 @@ Parts:
   - Name:            STAT
     Size:            8
   - Name:            HASH
-    Size:            8
+    Size:            20
   - Name:            CXIL
     Size:            8
 ...

diff --git a/llvm/test/tools/obj2yaml/DXContainer/ShaderHash.yaml b/llvm/test/tools/obj2yaml/DXContainer/ShaderHash.yaml
new file mode 100644 (file)
index 0000000..5809434
--- /dev/null
+++ b/llvm/test/tools/obj2yaml/DXContainer/ShaderHash.yaml
@@ -0,0 +1,33 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s 
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       1
+Parts:
+  - Name:            HASH
+    Size:            20
+    Hash:
+      IncludesSource:  true
+      Digest:          [ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 
+                         0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80 ]
+...
+
+# CHECK: --- !dxcontainer
+# CHECK-NEXT: Header:
+# CHECK-NEXT:   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+# CHECK-NEXT:                      0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+# CHECK:        FileSize:        64
+# CHECK-NEXT:   PartCount:       1
+# CHECK-NEXT:   PartOffsets:     [ 36 ]
+# CHECK:        - Name:            HASH
+# CHECK-NEXT:     Size:            20
+# CHECK-NEXT:     Hash:
+# CHECK-NEXT:       IncludesSource:  true
+# CHECK-NEXT:       Digest:          [ 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x10,
+# CHECK-NEXT:                          0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80 ]
+# CHECK-NEXT: ...

diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
index d8036ba..61b6a29 100644 (file)
--- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
+++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
@@ -67,6 +67,12 @@ dumpDXContainer(MemoryBufferRef Source) {
         NewPart.Flags = DXContainerYAML::ShaderFlags(*Flags);
       break;
     }
+    case dxbc::PartType::HASH: {
+      Optional<dxbc::ShaderHash> Hash = Container.getShaderHash();
+      if (Hash && Hash->isPopulated())
+        NewPart.Hash = DXContainerYAML::ShaderHash(*Hash);
+      break;
+    }
     case dxbc::PartType::Unknown:
       break;
     }