1 /*-------------------------------------------------------------------------
5 * Copyright (c) 2015 Google Inc.
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
21 * \brief Program binary registry.
22 *//*--------------------------------------------------------------------*/
24 #include "vkBinaryRegistry.hpp"
25 #include "tcuResource.hpp"
26 #include "tcuFormatUtil.hpp"
27 #include "deFilePath.hpp"
28 #include "deStringUtil.hpp"
29 #include "deDirectoryIterator.hpp"
41 namespace BinaryRegistryDetail
50 string getProgramFileName (deUint32 index)
52 return de::toString(tcu::toHex(index)) + ".spv";
55 string getProgramPath (const std::string& dirName, deUint32 index)
57 return de::FilePath::join(dirName, getProgramFileName(index)).getPath();
60 bool isHexChr (char c)
62 return de::inRange(c, '0', '9') || de::inRange(c, 'a', 'f') || de::inRange(c, 'A', 'F');
65 bool isProgramFileName (const std::string& name)
67 // 0x + 00000000 + .spv
68 if (name.length() != (2 + 8 + 4))
79 for (size_t ndx = 2; ndx < 10; ++ndx)
81 if (!isHexChr(name[ndx]))
88 deUint32 getProgramIndexFromName (const std::string& name)
90 DE_ASSERT(isProgramFileName(name));
93 std::stringstream str;
95 str << std::hex << name.substr(2,10);
98 DE_ASSERT(getProgramFileName(index) == name);
103 string getIndexPath (const std::string& dirName)
105 return de::FilePath::join(dirName, "index.bin").getPath();
108 void writeBinary (const ProgramBinary& binary, const std::string& dstPath)
110 const de::FilePath filePath(dstPath);
112 if (!de::FilePath(filePath.getDirName()).exists())
113 de::createDirectoryAndParents(filePath.getDirName().c_str());
116 std::ofstream out (dstPath.c_str(), std::ios_base::binary);
118 if (!out.is_open() || !out.good())
119 throw tcu::Exception("Failed to open " + dstPath);
121 out.write((const char*)binary.getBinary(), binary.getSize());
126 void writeBinary (const std::string& dstDir, deUint32 index, const ProgramBinary& binary)
128 writeBinary(binary, getProgramPath(dstDir, index));
131 ProgramBinary* readBinary (const std::string& srcPath)
133 std::ifstream in (srcPath.c_str(), std::ios::binary | std::ios::ate);
134 const size_t size = (size_t)in.tellg();
136 if (!in.is_open() || !in.good())
137 throw tcu::Exception("Failed to open " + srcPath);
140 throw tcu::Exception("Malformed binary, size = 0");
142 in.seekg(0, std::ios::beg);
145 std::vector<deUint8> bytes (size);
147 in.read((char*)&bytes[0], size);
148 DE_ASSERT(bytes[0] != 0);
150 return new ProgramBinary(vk::PROGRAM_FORMAT_SPIRV, bytes.size(), &bytes[0]);
154 deUint32 binaryHash (const ProgramBinary* binary)
156 return deMemoryHash(binary->getBinary(), binary->getSize());
159 deBool binaryEqual (const ProgramBinary* a, const ProgramBinary* b)
161 if (a->getSize() == b->getSize())
162 return deMemoryEqual(a->getBinary(), b->getBinary(), a->getSize());
167 std::vector<deUint32> getSearchPath (const ProgramIdentifier& id)
169 const std::string combinedStr = id.testCasePath + '#' + id.programName;
170 const size_t strLen = combinedStr.size();
171 const size_t numWords = strLen/4 + 1; // Must always end up with at least one 0 byte
172 vector<deUint32> words (numWords, 0u);
174 deMemcpy(&words[0], combinedStr.c_str(), strLen);
179 const deUint32* findBinaryIndex (BinaryIndexAccess* index, const ProgramIdentifier& id)
181 const vector<deUint32> words = getSearchPath(id);
187 const BinaryIndexNode& curNode = (*index)[nodeNdx];
189 if (curNode.word == words[wordNdx])
191 if (wordNdx+1 < words.size())
193 TCU_CHECK_INTERNAL((size_t)curNode.index < index->size());
195 nodeNdx = curNode.index;
198 else if (wordNdx+1 == words.size())
199 return &curNode.index;
203 else if (curNode.word != 0)
207 // Index should always be null-terminated
208 TCU_CHECK_INTERNAL(nodeNdx < index->size());
217 //! Sparse index node used for final binary index construction
218 struct SparseIndexNode
222 std::vector<SparseIndexNode*> children;
224 SparseIndexNode (deUint32 word_, deUint32 index_)
229 SparseIndexNode (void)
234 ~SparseIndexNode (void)
236 for (size_t ndx = 0; ndx < children.size(); ndx++)
237 delete children[ndx];
241 #if defined(DE_DEBUG)
242 bool isNullByteTerminated (deUint32 word)
245 deMemcpy(bytes, &word, sizeof(word));
246 return bytes[3] == 0;
250 void addToSparseIndex (SparseIndexNode* group, const deUint32* words, size_t numWords, deUint32 index)
252 const deUint32 curWord = words[0];
253 SparseIndexNode* child = DE_NULL;
255 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
257 if (group->children[childNdx]->word == curWord)
259 child = group->children[childNdx];
264 DE_ASSERT(numWords > 1 || !child);
268 group->children.reserve(group->children.size()+1);
269 group->children.push_back(new SparseIndexNode(curWord, numWords == 1 ? index : 0));
271 child = group->children.back();
275 addToSparseIndex(child, words+1, numWords-1, index);
277 DE_ASSERT(isNullByteTerminated(curWord));
280 // Prepares sparse index for finalization. Ensures that child with word = 0 is moved
281 // to the end, or one is added if there is no such child already.
282 void normalizeSparseIndex (SparseIndexNode* group)
284 int zeroChildPos = -1;
286 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
288 normalizeSparseIndex(group->children[childNdx]);
290 if (group->children[childNdx]->word == 0)
292 DE_ASSERT(zeroChildPos < 0);
293 zeroChildPos = (int)childNdx;
297 if (zeroChildPos >= 0)
299 // Move child with word = 0 to last
300 while (zeroChildPos != (int)group->children.size()-1)
302 std::swap(group->children[zeroChildPos], group->children[zeroChildPos+1]);
306 else if (!group->children.empty())
308 group->children.reserve(group->children.size()+1);
309 group->children.push_back(new SparseIndexNode(0, 0));
313 deUint32 getIndexSize (const SparseIndexNode* group)
315 size_t numNodes = group->children.size();
317 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
318 numNodes += getIndexSize(group->children[childNdx]);
320 DE_ASSERT(numNodes <= std::numeric_limits<deUint32>::max());
322 return (deUint32)numNodes;
325 deUint32 addAndCountNodes (BinaryIndexNode* index, deUint32 baseOffset, const SparseIndexNode* group)
327 const deUint32 numLocalNodes = (deUint32)group->children.size();
328 deUint32 curOffset = numLocalNodes;
330 // Must be normalized prior to construction of final index
331 DE_ASSERT(group->children.empty() || group->children.back()->word == 0);
333 for (size_t childNdx = 0; childNdx < numLocalNodes; childNdx++)
335 const SparseIndexNode* child = group->children[childNdx];
336 const deUint32 subtreeSize = addAndCountNodes(index+curOffset, baseOffset+curOffset, child);
338 index[childNdx].word = child->word;
340 if (subtreeSize == 0)
341 index[childNdx].index = child->index;
344 DE_ASSERT(child->index == 0);
345 index[childNdx].index = baseOffset+curOffset;
348 curOffset += subtreeSize;
354 void buildFinalIndex (std::vector<BinaryIndexNode>* dst, const SparseIndexNode* root)
356 const deUint32 indexSize = getIndexSize(root);
360 dst->resize(indexSize);
361 addAndCountNodes(&(*dst)[0], 0, root);
365 // Generate empty index
368 (*dst)[0].index = 0u;
372 void buildBinaryIndex (std::vector<BinaryIndexNode>* dst, size_t numEntries, const ProgramIdentifierIndex* entries)
374 de::UniquePtr<SparseIndexNode> sparseIndex (new SparseIndexNode());
376 for (size_t ndx = 0; ndx < numEntries; ndx++)
378 const std::vector<deUint32> searchPath = getSearchPath(entries[ndx].id);
379 addToSparseIndex(sparseIndex.get(), &searchPath[0], searchPath.size(), entries[ndx].index);
382 normalizeSparseIndex(sparseIndex.get());
383 buildFinalIndex(dst, sparseIndex.get());
390 DE_IMPLEMENT_POOL_HASH(BinaryIndexHashImpl, const ProgramBinary*, deUint32, binaryHash, binaryEqual);
392 BinaryIndexHash::BinaryIndexHash (void)
393 : m_hash(BinaryIndexHashImpl_create(m_memPool.getRawPool()))
396 throw std::bad_alloc();
399 BinaryIndexHash::~BinaryIndexHash (void)
403 deUint32* BinaryIndexHash::find (const ProgramBinary* binary) const
405 return BinaryIndexHashImpl_find(m_hash, binary);
408 void BinaryIndexHash::insert (const ProgramBinary* binary, deUint32 index)
410 if (!BinaryIndexHashImpl_insert(m_hash, binary, index))
411 throw std::bad_alloc();
414 // BinaryRegistryWriter
416 BinaryRegistryWriter::BinaryRegistryWriter (const std::string& dstPath)
419 if (de::FilePath(dstPath).exists())
420 initFromPath(dstPath);
423 BinaryRegistryWriter::~BinaryRegistryWriter (void)
425 for (BinaryVector::const_iterator binaryIter = m_binaries.begin();
426 binaryIter != m_binaries.end();
428 delete binaryIter->binary;
431 void BinaryRegistryWriter::initFromPath (const std::string& srcPath)
433 DE_ASSERT(m_binaries.empty());
435 for (de::DirectoryIterator iter(srcPath); iter.hasItem(); iter.next())
437 const de::FilePath path = iter.getItem();
438 const std::string baseName = path.getBaseName();
440 if (isProgramFileName(baseName))
442 const deUint32 index = getProgramIndexFromName(baseName);
443 const de::UniquePtr<ProgramBinary> binary (readBinary(path.getPath()));
445 addBinary(index, *binary);
446 // \note referenceCount is left to 0 and will only be incremented
447 // if binary is reused (added via addProgram()).
452 void BinaryRegistryWriter::addProgram (const ProgramIdentifier& id, const ProgramBinary& binary)
454 const deUint32* const indexPtr = findBinary(binary);
455 deUint32 index = indexPtr ? *indexPtr : ~0u;
459 index = getNextSlot();
460 addBinary(index, binary);
463 m_binaries[index].referenceCount += 1;
464 m_binaryIndices.push_back(ProgramIdentifierIndex(id, index));
467 deUint32* BinaryRegistryWriter::findBinary (const ProgramBinary& binary) const
469 return m_binaryHash.find(&binary);
472 deUint32 BinaryRegistryWriter::getNextSlot (void) const
474 const deUint32 index = (deUint32)m_binaries.size();
476 if ((size_t)index != m_binaries.size())
477 throw std::bad_alloc(); // Overflow
482 void BinaryRegistryWriter::addBinary (deUint32 index, const ProgramBinary& binary)
484 DE_ASSERT(binary.getFormat() == vk::PROGRAM_FORMAT_SPIRV);
485 DE_ASSERT(findBinary(binary) == DE_NULL);
487 ProgramBinary* const binaryClone = new ProgramBinary(binary);
491 if (m_binaries.size() < (size_t)index+1)
492 m_binaries.resize(index+1);
494 DE_ASSERT(!m_binaries[index].binary);
495 DE_ASSERT(m_binaries[index].referenceCount == 0);
497 m_binaries[index].binary = binaryClone;
498 // \note referenceCount is not incremented here
506 m_binaryHash.insert(binaryClone, index);
509 void BinaryRegistryWriter::write (void) const
511 writeToPath(m_dstPath);
514 void BinaryRegistryWriter::writeToPath (const std::string& dstPath) const
516 if (!de::FilePath(dstPath).exists())
517 de::createDirectoryAndParents(dstPath.c_str());
519 DE_ASSERT(m_binaries.size() <= 0xffffffffu);
520 for (size_t binaryNdx = 0; binaryNdx < m_binaries.size(); ++binaryNdx)
522 const BinarySlot& slot = m_binaries[binaryNdx];
524 if (slot.referenceCount > 0)
526 DE_ASSERT(slot.binary);
527 writeBinary(dstPath, (deUint32)binaryNdx, *slot.binary);
531 // Delete stale binary if such exists
532 const std::string progPath = getProgramPath(dstPath, (deUint32)binaryNdx);
534 if (de::FilePath(progPath).exists())
535 deDeleteFile(progPath.c_str());
542 const de::FilePath indexPath = getIndexPath(dstPath);
543 std::vector<BinaryIndexNode> index;
545 buildBinaryIndex(&index, m_binaryIndices.size(), !m_binaryIndices.empty() ? &m_binaryIndices[0] : DE_NULL);
547 // Even in empty index there is always terminating node for the root group
548 DE_ASSERT(!index.empty());
550 if (!de::FilePath(indexPath.getDirName()).exists())
551 de::createDirectoryAndParents(indexPath.getDirName().c_str());
554 std::ofstream indexOut(indexPath.getPath(), std::ios_base::binary);
556 if (!indexOut.is_open() || !indexOut.good())
557 throw tcu::InternalError(string("Failed to open program binary index file ") + indexPath.getPath());
559 indexOut.write((const char*)&index[0], index.size()*sizeof(BinaryIndexNode));
564 // BinaryRegistryReader
566 BinaryRegistryReader::BinaryRegistryReader (const tcu::Archive& archive, const std::string& srcPath)
567 : m_archive (archive)
568 , m_srcPath (srcPath)
572 BinaryRegistryReader::~BinaryRegistryReader (void)
576 ProgramBinary* BinaryRegistryReader::loadProgram (const ProgramIdentifier& id) const
582 m_binaryIndex = BinaryIndexPtr(new BinaryIndexAccess(de::MovePtr<tcu::Resource>(m_archive.getResource(getIndexPath(m_srcPath).c_str()))));
584 catch (const tcu::ResourceError& e)
586 throw ProgramNotFoundException(id, string("Failed to open binary index (") + e.what() + ")");
591 const deUint32* indexPos = findBinaryIndex(m_binaryIndex.get(), id);
595 const string fullPath = getProgramPath(m_srcPath, *indexPos);
599 de::UniquePtr<tcu::Resource> progRes (m_archive.getResource(fullPath.c_str()));
600 const int progSize = progRes->getSize();
601 vector<deUint8> bytes (progSize);
603 TCU_CHECK_INTERNAL(!bytes.empty());
605 progRes->read(&bytes[0], progSize);
607 return new ProgramBinary(vk::PROGRAM_FORMAT_SPIRV, bytes.size(), &bytes[0]);
609 catch (const tcu::ResourceError& e)
611 throw ProgramNotFoundException(id, e.what());
615 throw ProgramNotFoundException(id, "Program not found in index");
619 } // BinaryRegistryDetail