From 21c9452305484dde63caed7806384b9e5084a6d2 Mon Sep 17 00:00:00 2001 From: Chris Bieneman Date: Thu, 19 May 2022 13:13:56 -0500 Subject: [PATCH] [DX][ObjYAML] Support for parsing DXIL part This patch adds support for parsing the DXIL part data into the ObjectYAML tooling. The DXIL part has additional headers describing the shader and bitcode data and stores serialized bitcode after the headers. Depends on D124945 Reviewed By: kuhar Differential Revision: https://reviews.llvm.org/D126795 --- llvm/include/llvm/BinaryFormat/DXContainer.h | 34 ++++++++++++++ llvm/include/llvm/Object/DXContainer.h | 8 ++++ llvm/include/llvm/ObjectYAML/DXContainerYAML.h | 17 +++++++ llvm/lib/Object/DXContainer.cpp | 18 ++++++++ llvm/lib/ObjectYAML/DXContainerEmitter.cpp | 43 ++++++++++++++++-- llvm/lib/ObjectYAML/DXContainerYAML.cpp | 13 ++++++ llvm/test/tools/obj2yaml/DXContainer/DXILPart.yaml | 52 ++++++++++++++++++++++ llvm/tools/obj2yaml/dxcontainer2yaml.cpp | 20 ++++++++- 8 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 llvm/test/tools/obj2yaml/DXContainer/DXILPart.yaml diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h index d9a231d..9e912c7 100644 --- a/llvm/include/llvm/BinaryFormat/DXContainer.h +++ b/llvm/include/llvm/BinaryFormat/DXContainer.h @@ -91,6 +91,40 @@ struct PartHeader { // Structure is followed directly by part data: uint8_t PartData[PartSize]. }; +struct BitcodeHeader { + uint8_t Magic[4]; // ACSII "DXIL". + uint8_t MajorVersion; // DXIL version. + uint8_t MinorVersion; // DXIL version. + uint16_t Unused; + uint32_t Offset; // Offset to LLVM bitcode (from start of header). + uint32_t Size; // Size of LLVM bitcode (in bytes). + // Followed by uint8_t[BitcodeHeader.Size] at &BitcodeHeader + Header.Offset + + void swapBytes() { + sys::swapByteOrder(MinorVersion); + sys::swapByteOrder(MajorVersion); + sys::swapByteOrder(Offset); + sys::swapByteOrder(Size); + } +}; + +struct ProgramHeader { + uint8_t MinorVersion : 4; + uint8_t MajorVersion : 4; + uint8_t Unused; + uint16_t ShaderKind; + uint32_t Size; // Size in uint32_t words including this header. + BitcodeHeader Bitcode; + + void swapBytes() { + sys::swapByteOrder(ShaderKind); + sys::swapByteOrder(Size); + Bitcode.swapBytes(); + } +}; + +static_assert(sizeof(ProgramHeader) == 24, "ProgramHeader Size incorrect!"); + } // namespace dxbc } // namespace llvm diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h index 3877abf..7aa7d8e 100644 --- a/llvm/include/llvm/Object/DXContainer.h +++ b/llvm/include/llvm/Object/DXContainer.h @@ -15,6 +15,7 @@ #ifndef LLVM_OBJECT_DXCONTAINER_H #define LLVM_OBJECT_DXCONTAINER_H +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/DXContainer.h" @@ -24,15 +25,20 @@ namespace llvm { namespace object { class DXContainer { +public: + using DXILData = std::pair; + private: DXContainer(MemoryBufferRef O); MemoryBufferRef Data; dxbc::Header Header; SmallVector PartOffsets; + Optional DXIL; Error parseHeader(); Error parsePartOffsets(); + Error parseDXILHeader(uint32_t Offset); friend class PartIterator; public: @@ -108,6 +114,8 @@ public: static Expected create(MemoryBufferRef Object); const dxbc::Header &getHeader() const { return Header; } + + Optional getDXIL() const { return DXIL; } }; } // namespace object diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h index e4acec0..d1c0cd9 100644 --- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h +++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h @@ -41,9 +41,22 @@ struct FileHeader { Optional> PartOffsets; }; +struct DXILProgram { + uint8_t MajorVersion; + uint8_t MinorVersion; + uint16_t ShaderKind; + Optional Size; + uint16_t DXILMajorVersion; + uint16_t DXILMinorVersion; + Optional DXILOffset; + Optional DXILSize; + Optional> DXIL; +}; + struct Part { std::string Name; uint32_t Size; + Optional Program; }; struct Object { @@ -69,6 +82,10 @@ template <> struct MappingTraits { static void mapping(IO &IO, DXContainerYAML::FileHeader &Header); }; +template <> struct MappingTraits { + static void mapping(IO &IO, DXContainerYAML::DXILProgram &Program); +}; + template <> struct MappingTraits { static void mapping(IO &IO, DXContainerYAML::Part &Version); }; diff --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp index 2a5281c..ca859c1 100644 --- a/llvm/lib/Object/DXContainer.cpp +++ b/llvm/lib/Object/DXContainer.cpp @@ -53,6 +53,18 @@ Error DXContainer::parseHeader() { return readStruct(Data.getBuffer(), Data.getBuffer().data(), Header); } +Error DXContainer::parseDXILHeader(uint32_t Offset) { + if (DXIL) + return parseFailed("More than one DXIL part is present in the file"); + const char *Current = Data.getBuffer().data() + Offset; + dxbc::ProgramHeader Header; + if (Error Err = readStruct(Data.getBuffer(), Current, Header)) + return Err; + Current += offsetof(dxbc::ProgramHeader, Bitcode) + Header.Bitcode.Offset; + DXIL.emplace(std::make_pair(Header, Current)); + return Error::success(); +} + Error DXContainer::parsePartOffsets() { const char *Current = Data.getBuffer().data() + sizeof(dxbc::Header); for (uint32_t Part = 0; Part < Header.PartCount; ++Part) { @@ -68,6 +80,12 @@ Error DXContainer::parsePartOffsets() { if (PartOffset > Data.getBufferSize() - sizeof(dxbc::PartHeader)) return parseFailed("Part offset points beyond boundary of the file"); PartOffsets.push_back(PartOffset); + + // If this isn't a dxil part stop here... + if (Data.getBuffer().substr(PartOffset, 4) != "DXIL") + continue; + if (Error Err = parseDXILHeader(PartOffset + sizeof(dxbc::PartHeader))) + return Err; } return Error::success(); } diff --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp index 4729085..e3ab1ef 100644 --- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp +++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp @@ -104,14 +104,14 @@ void DXContainerWriter::writeHeader(raw_ostream &OS) { OS.write(reinterpret_cast(Offsets.data()), Offsets.size() * sizeof(uint32_t)); } + void DXContainerWriter::writeParts(raw_ostream &OS) { uint32_t RollingOffset = sizeof(dxbc::Header) + (ObjectFile.Header.PartCount * sizeof(uint32_t)); for (auto I : llvm::zip(ObjectFile.Parts, *ObjectFile.Header.PartOffsets)) { if (RollingOffset < std::get<1>(I)) { uint32_t PadBytes = std::get<1>(I) - RollingOffset; - std::vector FillData(PadBytes, 0); - OS.write(reinterpret_cast(FillData.data()), PadBytes); + OS.write_zeros(PadBytes); } DXContainerYAML::Part P = std::get<0>(I); OS.write(P.Name.c_str(), 4); @@ -120,7 +120,44 @@ void DXContainerWriter::writeParts(raw_ostream &OS) { OS.write(reinterpret_cast(&P.Size), sizeof(uint32_t)); RollingOffset = std::get<1>(I) + sizeof(dxbc::PartHeader); - // TODO: Write Part data + if (P.Name == "DXIL" && P.Program) { + dxbc::ProgramHeader Header; + Header.MajorVersion = P.Program->MajorVersion; + Header.MinorVersion = P.Program->MinorVersion; + Header.ShaderKind = P.Program->ShaderKind; + memcpy(Header.Bitcode.Magic, "DXIL", 4); + Header.Bitcode.MajorVersion = P.Program->DXILMajorVersion; + Header.Bitcode.MinorVersion = P.Program->DXILMinorVersion; + + // Compute the optional fields if needed... + if (P.Program->DXILOffset) + Header.Bitcode.Offset = P.Program->DXILOffset.getValue(); + else + Header.Bitcode.Offset = sizeof(dxbc::BitcodeHeader); + + if (P.Program->DXILSize) + Header.Bitcode.Size = P.Program->DXILSize.getValue(); + else + Header.Bitcode.Size = P.Program->DXIL ? P.Program->DXIL->size() : 0; + + if (P.Program->Size) + Header.Size = P.Program->Size.getValue(); + else + Header.Size = sizeof(dxbc::ProgramHeader) + Header.Bitcode.Size; + if (sys::IsBigEndianHost) + Header.swapBytes(); + OS.write(reinterpret_cast(&Header), + sizeof(dxbc::ProgramHeader)); + if (P.Program->DXIL) { + if (Header.Bitcode.Offset > sizeof(dxbc::BitcodeHeader)) { + uint32_t PadBytes = + Header.Bitcode.Offset - sizeof(dxbc::BitcodeHeader); + OS.write_zeros(PadBytes); + } + OS.write(reinterpret_cast(P.Program->DXIL->data()), + P.Program->DXIL->size()); + } + } } } diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp index 52f1205..de8f425 100644 --- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp +++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp @@ -31,10 +31,23 @@ void MappingTraits::mapping( IO.mapOptional("PartOffsets", Header.PartOffsets); } +void MappingTraits::mapping( + IO &IO, DXContainerYAML::DXILProgram &Program) { + IO.mapRequired("MajorVersion", Program.MajorVersion); + IO.mapRequired("MinorVersion", Program.MinorVersion); + IO.mapRequired("ShaderKind", Program.ShaderKind); + IO.mapOptional("Size", Program.Size); + IO.mapRequired("DXIMMajorVersion", Program.DXILMajorVersion); + IO.mapRequired("DXILMinorVersion", Program.DXILMinorVersion); + IO.mapOptional("DXILSize", Program.DXILSize); + IO.mapOptional("DXIL", Program.DXIL); +} + void MappingTraits::mapping(IO &IO, DXContainerYAML::Part &P) { IO.mapRequired("Name", P.Name); IO.mapRequired("Size", P.Size); + IO.mapOptional("Program", P.Program); } void MappingTraits::mapping( diff --git a/llvm/test/tools/obj2yaml/DXContainer/DXILPart.yaml b/llvm/test/tools/obj2yaml/DXContainer/DXILPart.yaml new file mode 100644 index 0000000..e4beedc --- /dev/null +++ b/llvm/test/tools/obj2yaml/DXContainer/DXILPart.yaml @@ -0,0 +1,52 @@ +# RUN: yaml2obj %s | obj2yaml | FileCheck %s + +--- !dxcontainer +Header: + Hash: [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ] + Version: + Major: 1 + Minor: 0 + FileSize: 3548 + PartCount: 7 + PartOffsets: [ 60, 76, 92, 108, 236, 1932, 1960 ] +Parts: + - Name: SFI0 + Size: 8 + - Name: ISG1 + Size: 8 + - Name: OSG1 + Size: 8 + - Name: PSV0 + Size: 120 + - Name: STAT + Size: 1688 + - Name: HASH + Size: 20 + - Name: DXIL + Size: 28 + Program: + MajorVersion: 6 + MinorVersion: 5 + ShaderKind: 5 + Size: 8 + DXIMMajorVersion: 1 + DXILMinorVersion: 5 + DXILSize: 4 + DXIL: [ 0x42, 0x43, 0xC0, 0xDE, ] +... + + + + +#CHECK: - Name: DXIL +#CHECK-NEXT: Size: 28 +#CHECK-NEXT: Program: +#CHECK-NEXT: MajorVersion: 6 +#CHECK-NEXT: MinorVersion: 5 +#CHECK-NEXT: ShaderKind: 5 +#CHECK-NEXT: Size: 8 +#CHECK-NEXT: DXIMMajorVersion: 1 +#CHECK-NEXT: DXILMinorVersion: 5 +#CHECK-NEXT: DXILSize: 4 +#CHECK-NEXT: DXIL: [ 0x42, 0x43, 0xC0, 0xDE diff --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp index 0d79baa..a57b8cf 100644 --- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp +++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp @@ -38,8 +38,24 @@ dumpDXContainer(MemoryBufferRef Source) { Obj->Header.PartOffsets = std::vector(); for (const auto P : Container) { Obj->Header.PartOffsets->push_back(P.Offset); - Obj->Parts.push_back( - DXContainerYAML::Part{P.Part.getName().str(), P.Part.Size}); + if (P.Part.getName() == "DXIL") { + Optional DXIL = Container.getDXIL(); + assert(DXIL.hasValue() && "Since we are iterating and found a DXIL part, " + "this should never not have a value"); + Obj->Parts.push_back(DXContainerYAML::Part{ + P.Part.getName().str(), P.Part.Size, + DXContainerYAML::DXILProgram{ + DXIL->first.MajorVersion, DXIL->first.MinorVersion, + DXIL->first.ShaderKind, DXIL->first.Size, + DXIL->first.Bitcode.MajorVersion, + DXIL->first.Bitcode.MinorVersion, DXIL->first.Bitcode.Offset, + DXIL->first.Bitcode.Size, + std::vector( + DXIL->second, DXIL->second + DXIL->first.Bitcode.Size)}}); + } else { + Obj->Parts.push_back( + DXContainerYAML::Part{P.Part.getName().str(), P.Part.Size, None}); + } } return Obj.release(); -- 2.7.4