1 /*-------------------------------------------------------------------------
5 * Copyright (c) 2015 Google Inc.
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
21 * \brief Program binary registry.
22 *//*--------------------------------------------------------------------*/
24 #include "vkBinaryRegistry.hpp"
25 #include "tcuResource.hpp"
26 #include "tcuFormatUtil.hpp"
27 #include "deFilePath.hpp"
28 #include "deStringUtil.hpp"
39 namespace BinaryRegistryDetail
48 string getProgramPath (const std::string& dirName, deUint32 index)
50 return de::FilePath::join(dirName, de::toString(tcu::toHex(index)) + ".spv").getPath();
53 string getIndexPath (const std::string& dirName)
55 return de::FilePath::join(dirName, "index.bin").getPath();
58 void writeBinary (const std::string& dstDir, deUint32 index, const ProgramBinary& binary)
60 const de::FilePath fullPath = getProgramPath(dstDir, index);
62 if (!de::FilePath(fullPath.getDirName()).exists())
63 de::createDirectoryAndParents(fullPath.getDirName().c_str());
66 std::ofstream out (fullPath.getPath(), std::ios_base::binary);
68 if (!out.is_open() || !out.good())
69 throw tcu::Exception("Failed to open " + string(fullPath.getPath()));
71 out.write((const char*)binary.getBinary(), binary.getSize());
76 deUint32 binaryHash (const ProgramBinary* binary)
78 return deMemoryHash(binary->getBinary(), binary->getSize());
81 deBool binaryEqual (const ProgramBinary* a, const ProgramBinary* b)
83 if (a->getSize() == b->getSize())
84 return deMemoryEqual(a->getBinary(), b->getBinary(), a->getSize());
89 std::vector<deUint32> getSearchPath (const ProgramIdentifier& id)
91 const std::string combinedStr = id.testCasePath + '#' + id.programName;
92 const size_t strLen = combinedStr.size();
93 const size_t numWords = strLen/4 + 1; // Must always end up with at least one 0 byte
94 vector<deUint32> words (numWords, 0u);
96 deMemcpy(&words[0], combinedStr.c_str(), strLen);
101 const deUint32* findBinaryIndex (BinaryIndexAccess* index, const ProgramIdentifier& id)
103 const vector<deUint32> words = getSearchPath(id);
109 const BinaryIndexNode& curNode = (*index)[nodeNdx];
111 if (curNode.word == words[wordNdx])
113 if (wordNdx+1 < words.size())
115 TCU_CHECK_INTERNAL((size_t)curNode.index < index->size());
117 nodeNdx = curNode.index;
120 else if (wordNdx+1 == words.size())
121 return &curNode.index;
125 else if (curNode.word != 0)
129 // Index should always be null-terminated
130 TCU_CHECK_INTERNAL(nodeNdx < index->size());
139 //! Sparse index node used for final binary index construction
140 struct SparseIndexNode
144 std::vector<SparseIndexNode*> children;
146 SparseIndexNode (deUint32 word_, deUint32 index_)
151 SparseIndexNode (void)
156 ~SparseIndexNode (void)
158 for (size_t ndx = 0; ndx < children.size(); ndx++)
159 delete children[ndx];
163 #if defined(DE_DEBUG)
164 bool isNullByteTerminated (deUint32 word)
167 deMemcpy(bytes, &word, sizeof(word));
168 return bytes[3] == 0;
172 void addToSparseIndex (SparseIndexNode* group, const deUint32* words, size_t numWords, deUint32 index)
174 const deUint32 curWord = words[0];
175 SparseIndexNode* child = DE_NULL;
177 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
179 if (group->children[childNdx]->word == curWord)
181 child = group->children[childNdx];
186 DE_ASSERT(numWords > 1 || !child);
190 group->children.reserve(group->children.size()+1);
191 group->children.push_back(new SparseIndexNode(curWord, numWords == 1 ? index : 0));
193 child = group->children.back();
197 addToSparseIndex(child, words+1, numWords-1, index);
199 DE_ASSERT(isNullByteTerminated(curWord));
202 // Prepares sparse index for finalization. Ensures that child with word = 0 is moved
203 // to the end, or one is added if there is no such child already.
204 void normalizeSparseIndex (SparseIndexNode* group)
206 int zeroChildPos = -1;
208 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
210 normalizeSparseIndex(group->children[childNdx]);
212 if (group->children[childNdx]->word == 0)
214 DE_ASSERT(zeroChildPos < 0);
215 zeroChildPos = (int)childNdx;
219 if (zeroChildPos >= 0)
221 // Move child with word = 0 to last
222 while (zeroChildPos != (int)group->children.size()-1)
224 std::swap(group->children[zeroChildPos], group->children[zeroChildPos+1]);
228 else if (!group->children.empty())
230 group->children.reserve(group->children.size()+1);
231 group->children.push_back(new SparseIndexNode(0, 0));
235 deUint32 getIndexSize (const SparseIndexNode* group)
237 size_t numNodes = group->children.size();
239 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
240 numNodes += getIndexSize(group->children[childNdx]);
242 DE_ASSERT(numNodes <= std::numeric_limits<deUint32>::max());
244 return (deUint32)numNodes;
247 deUint32 addAndCountNodes (BinaryIndexNode* index, deUint32 baseOffset, const SparseIndexNode* group)
249 const deUint32 numLocalNodes = (deUint32)group->children.size();
250 deUint32 curOffset = numLocalNodes;
252 // Must be normalized prior to construction of final index
253 DE_ASSERT(group->children.empty() || group->children.back()->word == 0);
255 for (size_t childNdx = 0; childNdx < numLocalNodes; childNdx++)
257 const SparseIndexNode* child = group->children[childNdx];
258 const deUint32 subtreeSize = addAndCountNodes(index+curOffset, baseOffset+curOffset, child);
260 index[childNdx].word = child->word;
262 if (subtreeSize == 0)
263 index[childNdx].index = child->index;
266 DE_ASSERT(child->index == 0);
267 index[childNdx].index = baseOffset+curOffset;
270 curOffset += subtreeSize;
276 void buildFinalIndex (std::vector<BinaryIndexNode>* dst, const SparseIndexNode* root)
278 const deUint32 indexSize = getIndexSize(root);
282 dst->resize(indexSize);
283 addAndCountNodes(&(*dst)[0], 0, root);
287 // Generate empty index
290 (*dst)[0].index = 0u;
296 // BinaryRegistryWriter
298 DE_IMPLEMENT_POOL_HASH(BinaryHash, const ProgramBinary*, deUint32, binaryHash, binaryEqual);
300 BinaryRegistryWriter::BinaryRegistryWriter (const std::string& dstPath)
301 : m_dstPath (dstPath)
302 , m_binaryIndexMap (DE_NULL)
304 m_binaryIndexMap = BinaryHash_create(m_memPool.getRawPool());
306 if (!m_binaryIndexMap)
307 throw std::bad_alloc();
310 BinaryRegistryWriter::~BinaryRegistryWriter (void)
312 for (BinaryVector::const_iterator binaryIter = m_compactedBinaries.begin();
313 binaryIter != m_compactedBinaries.end();
318 void BinaryRegistryWriter::storeProgram (const ProgramIdentifier& id, const ProgramBinary& binary)
320 const deUint32* const indexPtr = BinaryHash_find(m_binaryIndexMap, &binary);
321 deUint32 index = indexPtr ? *indexPtr : ~0u;
323 DE_ASSERT(binary.getFormat() == vk::PROGRAM_FORMAT_SPIRV);
327 ProgramBinary* const binaryClone = new ProgramBinary(binary);
331 index = (deUint32)m_compactedBinaries.size();
332 m_compactedBinaries.push_back(binaryClone);
340 writeBinary(m_dstPath, index, binary);
342 if (!BinaryHash_insert(m_binaryIndexMap, binaryClone, index))
343 throw std::bad_alloc();
346 DE_ASSERT((size_t)index < m_compactedBinaries.size());
348 m_binaryIndices.push_back(BinaryIndex(id, index));
351 void BinaryRegistryWriter::writeIndex (void) const
353 const de::FilePath indexPath = getIndexPath(m_dstPath);
354 std::vector<BinaryIndexNode> index;
357 de::UniquePtr<SparseIndexNode> sparseIndex (new SparseIndexNode());
359 for (size_t progNdx = 0; progNdx < m_binaryIndices.size(); progNdx++)
361 const std::vector<deUint32> searchPath = getSearchPath(m_binaryIndices[progNdx].id);
362 addToSparseIndex(sparseIndex.get(), &searchPath[0], searchPath.size(), m_binaryIndices[progNdx].index);
365 normalizeSparseIndex(sparseIndex.get());
366 buildFinalIndex(&index, sparseIndex.get());
369 // Even in empty index there is always terminating node for the root group
370 DE_ASSERT(!index.empty());
372 if (!de::FilePath(indexPath.getDirName()).exists())
373 de::createDirectoryAndParents(indexPath.getDirName().c_str());
376 std::ofstream indexOut(indexPath.getPath(), std::ios_base::binary);
378 if (!indexOut.is_open() || !indexOut.good())
379 throw tcu::InternalError(string("Failed to open program binary index file ") + indexPath.getPath());
381 indexOut.write((const char*)&index[0], index.size()*sizeof(BinaryIndexNode));
385 // BinaryRegistryReader
387 BinaryRegistryReader::BinaryRegistryReader (const tcu::Archive& archive, const std::string& srcPath)
388 : m_archive (archive)
389 , m_srcPath (srcPath)
393 BinaryRegistryReader::~BinaryRegistryReader (void)
397 ProgramBinary* BinaryRegistryReader::loadProgram (const ProgramIdentifier& id) const
403 m_binaryIndex = BinaryIndexPtr(new BinaryIndexAccess(de::MovePtr<tcu::Resource>(m_archive.getResource(getIndexPath(m_srcPath).c_str()))));
405 catch (const tcu::ResourceError& e)
407 throw ProgramNotFoundException(id, string("Failed to open binary index (") + e.what() + ")");
412 const deUint32* indexPos = findBinaryIndex(m_binaryIndex.get(), id);
416 const string fullPath = getProgramPath(m_srcPath, *indexPos);
420 de::UniquePtr<tcu::Resource> progRes (m_archive.getResource(fullPath.c_str()));
421 const int progSize = progRes->getSize();
422 vector<deUint8> bytes (progSize);
424 TCU_CHECK_INTERNAL(!bytes.empty());
426 progRes->read(&bytes[0], progSize);
428 return new ProgramBinary(vk::PROGRAM_FORMAT_SPIRV, bytes.size(), &bytes[0]);
430 catch (const tcu::ResourceError& e)
432 throw ProgramNotFoundException(id, e.what());
436 throw ProgramNotFoundException(id, "Program not found in index");
440 } // BinaryRegistryDetail