From: Pyry Haulos Date: Thu, 12 Nov 2015 22:41:16 +0000 (-0800) Subject: Improve prebuilt SPIR-V binary storage X-Git-Tag: upstream/0.1.0~812^2~512 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=b10f41aed29c5cb350ef7091ff3cfa165426e9ba;p=platform%2Fupstream%2FVK-GL-CTS.git Improve prebuilt SPIR-V binary storage This change improves storage usage of pre-built SPIR-V binary registry by eliminating duplicate binaries. With the current test set the size of binary registry drops from almost 400MiB down to 25MiB. A binary index file (trie) is used to map from binary identifiers to actual binary files. Change-Id: I09b015c00bd30df00e7cf96a39549ddbb66ece61 --- diff --git a/external/vulkancts/framework/vulkan/vkBinaryRegistry.cpp b/external/vulkancts/framework/vulkan/vkBinaryRegistry.cpp index b247583..99efa56 100644 --- a/external/vulkancts/framework/vulkan/vkBinaryRegistry.cpp +++ b/external/vulkancts/framework/vulkan/vkBinaryRegistry.cpp @@ -34,57 +34,354 @@ #include "vkBinaryRegistry.hpp" #include "tcuResource.hpp" +#include "tcuFormatUtil.hpp" #include "deFilePath.hpp" #include "deStringUtil.hpp" +#include "deString.h" +#include "deInt32.h" -#include #include +#include +#include +#include namespace vk { +namespace BinaryRegistryDetail +{ using std::string; using std::vector; -static string getProgramPath (const ProgramIdentifier& id) +namespace +{ + +string getProgramPath (const std::string& dirName, deUint32 index) +{ + return de::FilePath::join(dirName, de::toString(tcu::toHex(index)) + ".spv").getPath(); +} + +string getIndexPath (const std::string& dirName) +{ + return de::FilePath::join(dirName, "index.bin").getPath(); +} + +void writeBinary (const std::string& dstDir, deUint32 index, const ProgramBinary& binary) +{ + const de::FilePath fullPath = getProgramPath(dstDir, index); + + if (!de::FilePath(fullPath.getDirName()).exists()) + de::createDirectoryAndParents(fullPath.getDirName().c_str()); + + { + std::ofstream out (fullPath.getPath(), std::ios_base::binary); + + if (!out.is_open() || !out.good()) + throw tcu::Exception("Failed to open " + string(fullPath.getPath())); + + out.write((const char*)binary.getBinary(), binary.getSize()); + out.close(); + } +} + +deUint32 binaryHash (const ProgramBinary* binary) +{ + return deMemoryHash(binary->getBinary(), binary->getSize()); +} + +deBool binaryEqual (const ProgramBinary* a, const ProgramBinary* b) { - const vector casePathComps = de::splitString(id.testCasePath, '.'); - std::ostringstream path; + if (a->getSize() == b->getSize()) + return deMemoryEqual(a->getBinary(), b->getBinary(), a->getSize()); + else + return DE_FALSE; +} - for (size_t compNdx = 0; compNdx < casePathComps.size(); compNdx++) - path << casePathComps[compNdx] << '/'; +std::vector getSearchPath (const ProgramIdentifier& id) +{ + const std::string combinedStr = id.testCasePath + '#' + id.programName; + const size_t strLen = combinedStr.size(); + const size_t numWords = strLen/4 + 1; // Must always end up with at least one 0 byte + vector words (numWords, 0u); - path << id.programName << ".spv"; + deMemcpy(&words[0], combinedStr.c_str(), strLen); - return path.str(); + return words; } +const deUint32* findBinaryIndex (BinaryIndexAccess* index, const ProgramIdentifier& id) +{ + const vector words = getSearchPath(id); + size_t nodeNdx = 0; + size_t wordNdx = 0; + + for (;;) + { + const BinaryIndexNode& curNode = (*index)[nodeNdx]; + + if (curNode.word == words[wordNdx]) + { + if (wordNdx+1 < words.size()) + { + TCU_CHECK_INTERNAL((size_t)curNode.index < index->size()); + + nodeNdx = curNode.index; + wordNdx += 1; + } + else if (wordNdx+1 == words.size()) + return &curNode.index; + else + return DE_NULL; + } + else if (curNode.word != 0) + { + nodeNdx += 1; + + // Index should always be null-terminated + TCU_CHECK_INTERNAL(nodeNdx < index->size()); + } + else + return DE_NULL; + } + + return DE_NULL; +} + +//! Sparse index node used for final binary index construction +struct SparseIndexNode +{ + deUint32 word; + deUint32 index; + std::vector children; + + SparseIndexNode (deUint32 word_, deUint32 index_) + : word (word_) + , index (index_) + {} + + SparseIndexNode (void) + : word (0) + , index (0) + {} + + ~SparseIndexNode (void) + { + for (size_t ndx = 0; ndx < children.size(); ndx++) + delete children[ndx]; + } +}; + +#if defined(DE_DEBUG) +bool isNullByteTerminated (deUint32 word) +{ + deUint8 bytes[4]; + deMemcpy(bytes, &word, sizeof(word)); + return bytes[3] == 0; +} +#endif + +void addToSparseIndex (SparseIndexNode* group, const deUint32* words, size_t numWords, deUint32 index) +{ + const deUint32 curWord = words[0]; + SparseIndexNode* child = DE_NULL; + + for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++) + { + if (group->children[childNdx]->word == curWord) + { + child = group->children[childNdx]; + break; + } + } + + DE_ASSERT(numWords > 1 || !child); + + if (!child) + { + group->children.reserve(group->children.size()+1); + group->children.push_back(new SparseIndexNode(curWord, numWords == 1 ? index : 0)); + + child = group->children.back(); + } + + if (numWords > 1) + addToSparseIndex(child, words+1, numWords-1, index); + else + DE_ASSERT(isNullByteTerminated(curWord)); +} + +// Prepares sparse index for finalization. Ensures that child with word = 0 is moved +// to the end, or one is added if there is no such child already. +void normalizeSparseIndex (SparseIndexNode* group) +{ + int zeroChildPos = -1; + + for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++) + { + normalizeSparseIndex(group->children[childNdx]); + + if (group->children[childNdx]->word == 0) + { + DE_ASSERT(zeroChildPos < 0); + zeroChildPos = (int)childNdx; + } + } + + if (zeroChildPos >= 0) + { + // Move child with word = 0 to last + while (zeroChildPos != (int)group->children.size()-1) + { + std::swap(group->children[zeroChildPos], group->children[zeroChildPos+1]); + zeroChildPos += 1; + } + } + else if (!group->children.empty()) + { + group->children.reserve(group->children.size()+1); + group->children.push_back(new SparseIndexNode(0, 0)); + } +} + +deUint32 getIndexSize (const SparseIndexNode* group) +{ + size_t numNodes = group->children.size(); + + for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++) + numNodes += getIndexSize(group->children[childNdx]); + + DE_ASSERT(numNodes <= std::numeric_limits::max()); + + return (deUint32)numNodes; +} + +deUint32 addAndCountNodes (BinaryIndexNode* index, deUint32 baseOffset, const SparseIndexNode* group) +{ + const deUint32 numLocalNodes = (deUint32)group->children.size(); + deUint32 curOffset = numLocalNodes; + + // Must be normalized prior to construction of final index + DE_ASSERT(group->children.empty() || group->children.back()->word == 0); + + for (size_t childNdx = 0; childNdx < numLocalNodes; childNdx++) + { + const SparseIndexNode* child = group->children[childNdx]; + const deUint32 subtreeSize = addAndCountNodes(index+curOffset, baseOffset+curOffset, child); + + index[childNdx].word = child->word; + + if (subtreeSize == 0) + index[childNdx].index = child->index; + else + { + DE_ASSERT(child->index == 0); + index[childNdx].index = baseOffset+curOffset; + } + + curOffset += subtreeSize; + } + + return curOffset; +} + +void buildFinalIndex (std::vector* dst, const SparseIndexNode* root) +{ + const deUint32 indexSize = getIndexSize(root); + + DE_ASSERT(indexSize > 0); + + dst->resize(indexSize); + addAndCountNodes(&(*dst)[0], 0, root); +} + +} // anonymous + // BinaryRegistryWriter +DE_IMPLEMENT_POOL_HASH(BinaryHash, const ProgramBinary*, deUint32, binaryHash, binaryEqual); + BinaryRegistryWriter::BinaryRegistryWriter (const std::string& dstPath) - : m_dstPath(dstPath) + : m_dstPath (dstPath) + , m_binaryIndexMap (DE_NULL) { + m_binaryIndexMap = BinaryHash_create(m_memPool.getRawPool()); + + if (!m_binaryIndexMap) + throw std::bad_alloc(); } BinaryRegistryWriter::~BinaryRegistryWriter (void) { + for (BinaryVector::const_iterator binaryIter = m_compactedBinaries.begin(); + binaryIter != m_compactedBinaries.end(); + ++binaryIter) + delete *binaryIter; } void BinaryRegistryWriter::storeProgram (const ProgramIdentifier& id, const ProgramBinary& binary) { - const de::FilePath fullPath = de::FilePath::join(m_dstPath, getProgramPath(id)); + const deUint32* const indexPtr = BinaryHash_find(m_binaryIndexMap, &binary); + deUint32 index = indexPtr ? *indexPtr : ~0u; - if (!de::FilePath(fullPath.getDirName()).exists()) - de::createDirectoryAndParents(fullPath.getDirName().c_str()); + DE_ASSERT(binary.getFormat() == vk::PROGRAM_FORMAT_SPIRV); + if (!indexPtr) { - std::ofstream out (fullPath.getPath(), std::ios_base::binary); + ProgramBinary* const binaryClone = new ProgramBinary(binary); - if (!out.is_open() || !out.good()) - throw tcu::Exception("Failed to open " + string(fullPath.getPath())); + try + { + index = (deUint32)m_compactedBinaries.size(); + m_compactedBinaries.push_back(binaryClone); + } + catch (...) + { + delete binaryClone; + throw; + } - out.write((const char*)binary.getBinary(), binary.getSize()); - out.close(); + writeBinary(m_dstPath, index, binary); + + if (!BinaryHash_insert(m_binaryIndexMap, binaryClone, index)) + throw std::bad_alloc(); + } + + DE_ASSERT((size_t)index < m_compactedBinaries.size()); + + m_binaryIndices.push_back(BinaryIndex(id, index)); +} + +void BinaryRegistryWriter::writeIndex (void) const +{ + const de::FilePath indexPath = getIndexPath(m_dstPath); + std::vector index; + + { + de::UniquePtr sparseIndex (new SparseIndexNode()); + + for (size_t progNdx = 0; progNdx < m_binaryIndices.size(); progNdx++) + { + const std::vector searchPath = getSearchPath(m_binaryIndices[progNdx].id); + addToSparseIndex(sparseIndex.get(), &searchPath[0], searchPath.size(), m_binaryIndices[progNdx].index); + } + + normalizeSparseIndex(sparseIndex.get()); + buildFinalIndex(&index, sparseIndex.get()); + } + + // Even in empty index there is always terminating node for the root group + DE_ASSERT(!index.empty()); + + if (!de::FilePath(indexPath.getDirName()).exists()) + de::createDirectoryAndParents(indexPath.getDirName().c_str()); + + { + std::ofstream indexOut(indexPath.getPath(), std::ios_base::binary); + + if (!indexOut.is_open() || !indexOut.good()) + throw tcu::InternalError(string("Failed to open program binary index file ") + indexPath.getPath()); + + indexOut.write((const char*)&index[0], index.size()*sizeof(BinaryIndexNode)); } } @@ -102,25 +399,46 @@ BinaryRegistryReader::~BinaryRegistryReader (void) ProgramBinary* BinaryRegistryReader::loadProgram (const ProgramIdentifier& id) const { - const string fullPath = de::FilePath::join(m_srcPath, getProgramPath(id)).getPath(); + if (!m_binaryIndex) + { + try + { + m_binaryIndex = BinaryIndexPtr(new BinaryIndexAccess(de::MovePtr(m_archive.getResource(getIndexPath(m_srcPath).c_str())))); + } + catch (const tcu::ResourceError& e) + { + throw ProgramNotFoundException(id, string("Failed to open binary index (") + e.what() + ")"); + } + } - try { - de::UniquePtr progRes (m_archive.getResource(fullPath.c_str())); - const int progSize = progRes->getSize(); - vector bytes (progSize); + const deUint32* indexPos = findBinaryIndex(m_binaryIndex.get(), id); - TCU_CHECK_INTERNAL(!bytes.empty()); + if (indexPos) + { + const string fullPath = getProgramPath(m_srcPath, *indexPos); - progRes->read(&bytes[0], progSize); + try + { + de::UniquePtr progRes (m_archive.getResource(fullPath.c_str())); + const int progSize = progRes->getSize(); + vector bytes (progSize); - return new ProgramBinary(vk::PROGRAM_FORMAT_SPIRV, bytes.size(), &bytes[0]); - } - catch (const tcu::ResourceError&) - { - throw ProgramNotFoundException(id); + TCU_CHECK_INTERNAL(!bytes.empty()); + + progRes->read(&bytes[0], progSize); + + return new ProgramBinary(vk::PROGRAM_FORMAT_SPIRV, bytes.size(), &bytes[0]); + } + catch (const tcu::ResourceError& e) + { + throw ProgramNotFoundException(id, e.what()); + } + } + else + throw ProgramNotFoundException(id, "Program not found in index"); } } - +} // BinaryRegistryDetail } // vk diff --git a/external/vulkancts/framework/vulkan/vkBinaryRegistry.hpp b/external/vulkancts/framework/vulkan/vkBinaryRegistry.hpp index ede6d72..714e6f6 100644 --- a/external/vulkancts/framework/vulkan/vkBinaryRegistry.hpp +++ b/external/vulkancts/framework/vulkan/vkBinaryRegistry.hpp @@ -36,14 +36,19 @@ #include "vkDefs.hpp" #include "vkPrograms.hpp" +#include "tcuResource.hpp" +#include "deMemPool.hpp" +#include "dePoolHash.h" +#include "deUniquePtr.hpp" -namespace tcu -{ -class Archive; -} +#include +#include +#include namespace vk { +namespace BinaryRegistryDetail +{ struct ProgramIdentifier { @@ -57,26 +62,142 @@ struct ProgramIdentifier } }; +inline bool operator< (const ProgramIdentifier& a, const ProgramIdentifier& b) +{ + return (a.testCasePath < b.testCasePath) || ((a.testCasePath == b.testCasePath) && (a.programName < b.programName)); +} + class ProgramNotFoundException : public tcu::ResourceError { public: - ProgramNotFoundException (const ProgramIdentifier& id) - : tcu::ResourceError("Program " + id.testCasePath + " / '" + id.programName + "' not found") + ProgramNotFoundException (const ProgramIdentifier& id, const std::string& reason) + : tcu::ResourceError("Program " + id.testCasePath + " / '" + id.programName + "' not found: " + reason) { } }; +// Program Binary Index +// -------------------- +// +// When SPIR-V binaries are stored on disk, duplicate binaries are eliminated +// to save a significant amount of space. Many tests use identical binaries and +// just storing each compiled binary without de-duplication would be incredibly +// wasteful. +// +// To locate binary that corresponds given ProgramIdentifier, a program binary +// index is needed. Since that index is accessed every time a test requests shader +// binary, it must be fast to load (to reduce statup cost), and fast to access. +// +// Simple trie is used to store binary indices. It is laid out as an array of +// BinaryIndexNodes. Nodes store 4-byte pieces (words) of search string, rather +// than just a single character. This gives more regular memory layout in exchange +// of a little wasted storage. +// +// Search strings are created by splitting original string into 4-byte words and +// appending one or more terminating 0 bytes. +// +// For each node where word doesn't have trailing 0 bytes (not terminated), the +// index points into a offset of its child list. Children for each node are stored +// consecutively, and the list is terminated by child with word = 0. +// +// If word contains one or more trailing 0 bytes, index denotes the binary index +// instead of index of the child list. + +struct BinaryIndexNode +{ + deUint32 word; //!< 4 bytes of search string. + deUint32 index; //!< Binary index if word ends with 0 bytes, or index of first child node otherwise. +}; + +template +class LazyResource +{ +public: + LazyResource (de::MovePtr resource); + + const Element& operator[] (size_t ndx); + size_t size (void) const { return m_elements.size(); } + +private: + enum + { + ELEMENTS_PER_PAGE_LOG2 = 10 + }; + + inline size_t getPageForElement (size_t elemNdx) const { return elemNdx >> ELEMENTS_PER_PAGE_LOG2; } + inline bool isPageResident (size_t pageNdx) const { return m_isPageResident[pageNdx]; } + + void makePageResident (size_t pageNdx); + + de::UniquePtr m_resource; + + std::vector m_elements; + std::vector m_isPageResident; +}; + +template +LazyResource::LazyResource (de::MovePtr resource) + : m_resource(resource) +{ + const size_t resSize = m_resource->getSize(); + const size_t numElements = resSize/sizeof(Element); + const size_t numPages = (numElements >> ELEMENTS_PER_PAGE_LOG2) + ((numElements & ((1u< +const Element& LazyResource::operator[] (size_t ndx) +{ + const size_t pageNdx = getPageForElement(ndx); + + if (ndx >= m_elements.size()) + throw std::out_of_range(""); + + if (!isPageResident(pageNdx)) + makePageResident(pageNdx); + + return m_elements[ndx]; +} + +template +void LazyResource::makePageResident (size_t pageNdx) +{ + const size_t pageSize = (size_t)(1<getPosition() != pageOffset) + m_resource->setPosition((int)pageOffset); + + m_resource->read((deUint8*)&m_elements[pageNdx << ELEMENTS_PER_PAGE_LOG2], (int)numBytesToRead); + m_isPageResident[pageNdx] = true; +} + +typedef LazyResource BinaryIndexAccess; + +DE_DECLARE_POOL_HASH(BinaryHash, const ProgramBinary*, deUint32); + class BinaryRegistryReader { public: - BinaryRegistryReader (const tcu::Archive& archive, const std::string& srcPath); - ~BinaryRegistryReader (void); + BinaryRegistryReader (const tcu::Archive& archive, const std::string& srcPath); + ~BinaryRegistryReader (void); - ProgramBinary* loadProgram (const ProgramIdentifier& id) const; + ProgramBinary* loadProgram (const ProgramIdentifier& id) const; private: - const tcu::Archive& m_archive; - const std::string m_srcPath; + typedef de::MovePtr BinaryIndexPtr; + + const tcu::Archive& m_archive; + const std::string m_srcPath; + + mutable BinaryIndexPtr m_binaryIndex; }; class BinaryRegistryWriter @@ -86,11 +207,39 @@ public: ~BinaryRegistryWriter (void); void storeProgram (const ProgramIdentifier& id, const ProgramBinary& binary); + void writeIndex (void) const; private: - const std::string m_dstPath; + struct BinaryIndex + { + ProgramIdentifier id; + deUint32 index; + + BinaryIndex (const ProgramIdentifier& id_, + deUint32 index_) + : id (id_) + , index (index_) + {} + }; + + typedef std::vector BinaryVector; + typedef std::vector BinaryIndexVector; + + const std::string& m_dstPath; + + de::MemPool m_memPool; + BinaryHash* m_binaryIndexMap; //!< ProgramBinary -> slot in m_compactedBinaries + BinaryVector m_compactedBinaries; + BinaryIndexVector m_binaryIndices; //!< ProgramIdentifier -> slot in m_compactedBinaries }; +} // BinaryRegistryDetail + +using BinaryRegistryDetail::BinaryRegistryReader; +using BinaryRegistryDetail::BinaryRegistryWriter; +using BinaryRegistryDetail::ProgramIdentifier; +using BinaryRegistryDetail::ProgramNotFoundException; + } // vk #endif // _VKBINARYREGISTRY_HPP diff --git a/external/vulkancts/modules/vulkan/vktBuildPrograms.cpp b/external/vulkancts/modules/vulkan/vktBuildPrograms.cpp index 864c5d7..edf1f8d 100644 --- a/external/vulkancts/modules/vulkan/vktBuildPrograms.cpp +++ b/external/vulkancts/modules/vulkan/vktBuildPrograms.cpp @@ -197,6 +197,9 @@ BuildStats buildPrograms (tcu::TestContext& testCtx, const std::string& dstPath, iterator.next(); } + if (mode == BUILDMODE_BUILD) + writer->writeIndex(); + return stats; } diff --git a/external/vulkancts/modules/vulkan/vktTestPackage.cpp b/external/vulkancts/modules/vulkan/vktTestPackage.cpp index 55e5d12..32b5309 100644 --- a/external/vulkancts/modules/vulkan/vktTestPackage.cpp +++ b/external/vulkancts/modules/vulkan/vktTestPackage.cpp @@ -72,9 +72,12 @@ vk::ProgramBinary* compileProgram (const vk::SpirVAsmSource& source, vk::SpirVPr } template -vk::ProgramBinary* buildProgram (const std::string& casePath, IteratorType iter, vkt::Context* context, vk::BinaryCollection* progCollection) +vk::ProgramBinary* buildProgram (const std::string& casePath, + IteratorType iter, + const vk::BinaryRegistryReader& prebuiltBinRegistry, + tcu::TestLog& log, + vk::BinaryCollection* progCollection) { - tcu::TestLog& log = context->getTestContext().getLog(); const vk::ProgramIdentifier progId (casePath, iter.getName()); const tcu::ScopedLogSection progSection (log, iter.getName(), "Program: " + iter.getName()); de::MovePtr binProg; @@ -88,11 +91,9 @@ vk::ProgramBinary* buildProgram (const std::string& casePath, IteratorType iter, catch (const tcu::NotSupportedError& err) { // Try to load from cache - const vk::BinaryRegistryReader registry (context->getTestContext().getArchive(), "vulkan/prebuilt"); - log << err << tcu::TestLog::Message << "Building from source not supported, loading stored binary instead" << tcu::TestLog::EndMessage; - binProg = de::MovePtr(registry.loadProgram(progId)); + binProg = de::MovePtr(prebuiltBinRegistry.loadProgram(progId)); log << iter.getProgram(); } @@ -105,11 +106,13 @@ vk::ProgramBinary* buildProgram (const std::string& casePath, IteratorType iter, TCU_CHECK_INTERNAL(binProg); - vk::ProgramBinary* returnBinary = binProg.get(); + { + vk::ProgramBinary* const returnBinary = binProg.get(); - progCollection->add(progId.programName, binProg); + progCollection->add(progId.programName, binProg); - return returnBinary; + return returnBinary; + } } } // anonymous(compilation) @@ -137,6 +140,8 @@ public: private: vk::BinaryCollection m_progCollection; + vk::BinaryRegistryReader m_prebuiltBinRegistry; + de::UniquePtr m_library; Context m_context; @@ -149,9 +154,10 @@ static MovePtr createLibrary (tcu::TestContext& testCtx) } TestCaseExecutor::TestCaseExecutor (tcu::TestContext& testCtx) - : m_library (createLibrary(testCtx)) - , m_context (testCtx, m_library->getPlatformInterface(), m_progCollection) - , m_instance (DE_NULL) + : m_prebuiltBinRegistry (testCtx.getArchive(), "vulkan/prebuilt") + , m_library (createLibrary(testCtx)) + , m_context (testCtx, m_library->getPlatformInterface(), m_progCollection) + , m_instance (DE_NULL) { } @@ -176,7 +182,7 @@ void TestCaseExecutor::init (tcu::TestCase* testCase, const std::string& casePat for (vk::GlslSourceCollection::Iterator progIter = sourceProgs.glslSources.begin(); progIter != sourceProgs.glslSources.end(); ++progIter) { - vk::ProgramBinary* binProg = buildProgram(casePath, progIter, &m_context, &m_progCollection); + vk::ProgramBinary* binProg = buildProgram(casePath, progIter, m_prebuiltBinRegistry, log, &m_progCollection); try { @@ -194,7 +200,7 @@ void TestCaseExecutor::init (tcu::TestCase* testCase, const std::string& casePat for (vk::SpirVAsmCollection::Iterator asmIterator = sourceProgs.spirvAsmSources.begin(); asmIterator != sourceProgs.spirvAsmSources.end(); ++asmIterator) { - buildProgram(casePath, asmIterator, &m_context, &m_progCollection); + buildProgram(casePath, asmIterator, m_prebuiltBinRegistry, log, &m_progCollection); } DE_ASSERT(!m_instance);