1 /*-------------------------------------------------------------------------
5 * Copyright (c) 2015 Google Inc.
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
21 * \brief Program binary registry.
22 *//*--------------------------------------------------------------------*/
24 #include "vkBinaryRegistry.hpp"
25 #include "tcuResource.hpp"
26 #include "tcuFormatUtil.hpp"
27 #include "deFilePath.hpp"
28 #include "deStringUtil.hpp"
39 namespace BinaryRegistryDetail
48 string getProgramPath (const std::string& dirName, deUint32 index)
50 return de::FilePath::join(dirName, de::toString(tcu::toHex(index)) + ".spv").getPath();
53 string getIndexPath (const std::string& dirName)
55 return de::FilePath::join(dirName, "index.bin").getPath();
58 void writeBinary (const std::string& dstDir, deUint32 index, const ProgramBinary& binary)
60 const de::FilePath fullPath = getProgramPath(dstDir, index);
62 if (!de::FilePath(fullPath.getDirName()).exists())
63 de::createDirectoryAndParents(fullPath.getDirName().c_str());
66 std::ofstream out (fullPath.getPath(), std::ios_base::binary);
68 if (!out.is_open() || !out.good())
69 throw tcu::Exception("Failed to open " + string(fullPath.getPath()));
71 out.write((const char*)binary.getBinary(), binary.getSize());
76 deUint32 binaryHash (const ProgramBinary* binary)
78 return deMemoryHash(binary->getBinary(), binary->getSize());
81 deBool binaryEqual (const ProgramBinary* a, const ProgramBinary* b)
83 if (a->getSize() == b->getSize())
84 return deMemoryEqual(a->getBinary(), b->getBinary(), a->getSize());
89 std::vector<deUint32> getSearchPath (const ProgramIdentifier& id)
91 const std::string combinedStr = id.testCasePath + '#' + id.programName;
92 const size_t strLen = combinedStr.size();
93 const size_t numWords = strLen/4 + 1; // Must always end up with at least one 0 byte
94 vector<deUint32> words (numWords, 0u);
96 deMemcpy(&words[0], combinedStr.c_str(), strLen);
101 const deUint32* findBinaryIndex (BinaryIndexAccess* index, const ProgramIdentifier& id)
103 const vector<deUint32> words = getSearchPath(id);
109 const BinaryIndexNode& curNode = (*index)[nodeNdx];
111 if (curNode.word == words[wordNdx])
113 if (wordNdx+1 < words.size())
115 TCU_CHECK_INTERNAL((size_t)curNode.index < index->size());
117 nodeNdx = curNode.index;
120 else if (wordNdx+1 == words.size())
121 return &curNode.index;
125 else if (curNode.word != 0)
129 // Index should always be null-terminated
130 TCU_CHECK_INTERNAL(nodeNdx < index->size());
139 //! Sparse index node used for final binary index construction
140 struct SparseIndexNode
144 std::vector<SparseIndexNode*> children;
146 SparseIndexNode (deUint32 word_, deUint32 index_)
151 SparseIndexNode (void)
156 ~SparseIndexNode (void)
158 for (size_t ndx = 0; ndx < children.size(); ndx++)
159 delete children[ndx];
163 #if defined(DE_DEBUG)
164 bool isNullByteTerminated (deUint32 word)
167 deMemcpy(bytes, &word, sizeof(word));
168 return bytes[3] == 0;
172 void addToSparseIndex (SparseIndexNode* group, const deUint32* words, size_t numWords, deUint32 index)
174 const deUint32 curWord = words[0];
175 SparseIndexNode* child = DE_NULL;
177 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
179 if (group->children[childNdx]->word == curWord)
181 child = group->children[childNdx];
186 DE_ASSERT(numWords > 1 || !child);
190 group->children.reserve(group->children.size()+1);
191 group->children.push_back(new SparseIndexNode(curWord, numWords == 1 ? index : 0));
193 child = group->children.back();
197 addToSparseIndex(child, words+1, numWords-1, index);
199 DE_ASSERT(isNullByteTerminated(curWord));
202 // Prepares sparse index for finalization. Ensures that child with word = 0 is moved
203 // to the end, or one is added if there is no such child already.
204 void normalizeSparseIndex (SparseIndexNode* group)
206 int zeroChildPos = -1;
208 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
210 normalizeSparseIndex(group->children[childNdx]);
212 if (group->children[childNdx]->word == 0)
214 DE_ASSERT(zeroChildPos < 0);
215 zeroChildPos = (int)childNdx;
219 if (zeroChildPos >= 0)
221 // Move child with word = 0 to last
222 while (zeroChildPos != (int)group->children.size()-1)
224 std::swap(group->children[zeroChildPos], group->children[zeroChildPos+1]);
228 else if (!group->children.empty())
230 group->children.reserve(group->children.size()+1);
231 group->children.push_back(new SparseIndexNode(0, 0));
235 deUint32 getIndexSize (const SparseIndexNode* group)
237 size_t numNodes = group->children.size();
239 for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
240 numNodes += getIndexSize(group->children[childNdx]);
242 DE_ASSERT(numNodes <= std::numeric_limits<deUint32>::max());
244 return (deUint32)numNodes;
247 deUint32 addAndCountNodes (BinaryIndexNode* index, deUint32 baseOffset, const SparseIndexNode* group)
249 const deUint32 numLocalNodes = (deUint32)group->children.size();
250 deUint32 curOffset = numLocalNodes;
252 // Must be normalized prior to construction of final index
253 DE_ASSERT(group->children.empty() || group->children.back()->word == 0);
255 for (size_t childNdx = 0; childNdx < numLocalNodes; childNdx++)
257 const SparseIndexNode* child = group->children[childNdx];
258 const deUint32 subtreeSize = addAndCountNodes(index+curOffset, baseOffset+curOffset, child);
260 index[childNdx].word = child->word;
262 if (subtreeSize == 0)
263 index[childNdx].index = child->index;
266 DE_ASSERT(child->index == 0);
267 index[childNdx].index = baseOffset+curOffset;
270 curOffset += subtreeSize;
276 void buildFinalIndex (std::vector<BinaryIndexNode>* dst, const SparseIndexNode* root)
278 const deUint32 indexSize = getIndexSize(root);
280 DE_ASSERT(indexSize > 0);
282 dst->resize(indexSize);
283 addAndCountNodes(&(*dst)[0], 0, root);
288 // BinaryRegistryWriter
290 DE_IMPLEMENT_POOL_HASH(BinaryHash, const ProgramBinary*, deUint32, binaryHash, binaryEqual);
292 BinaryRegistryWriter::BinaryRegistryWriter (const std::string& dstPath)
293 : m_dstPath (dstPath)
294 , m_binaryIndexMap (DE_NULL)
296 m_binaryIndexMap = BinaryHash_create(m_memPool.getRawPool());
298 if (!m_binaryIndexMap)
299 throw std::bad_alloc();
302 BinaryRegistryWriter::~BinaryRegistryWriter (void)
304 for (BinaryVector::const_iterator binaryIter = m_compactedBinaries.begin();
305 binaryIter != m_compactedBinaries.end();
310 void BinaryRegistryWriter::storeProgram (const ProgramIdentifier& id, const ProgramBinary& binary)
312 const deUint32* const indexPtr = BinaryHash_find(m_binaryIndexMap, &binary);
313 deUint32 index = indexPtr ? *indexPtr : ~0u;
315 DE_ASSERT(binary.getFormat() == vk::PROGRAM_FORMAT_SPIRV);
319 ProgramBinary* const binaryClone = new ProgramBinary(binary);
323 index = (deUint32)m_compactedBinaries.size();
324 m_compactedBinaries.push_back(binaryClone);
332 writeBinary(m_dstPath, index, binary);
334 if (!BinaryHash_insert(m_binaryIndexMap, binaryClone, index))
335 throw std::bad_alloc();
338 DE_ASSERT((size_t)index < m_compactedBinaries.size());
340 m_binaryIndices.push_back(BinaryIndex(id, index));
343 void BinaryRegistryWriter::writeIndex (void) const
345 const de::FilePath indexPath = getIndexPath(m_dstPath);
346 std::vector<BinaryIndexNode> index;
349 de::UniquePtr<SparseIndexNode> sparseIndex (new SparseIndexNode());
351 for (size_t progNdx = 0; progNdx < m_binaryIndices.size(); progNdx++)
353 const std::vector<deUint32> searchPath = getSearchPath(m_binaryIndices[progNdx].id);
354 addToSparseIndex(sparseIndex.get(), &searchPath[0], searchPath.size(), m_binaryIndices[progNdx].index);
357 normalizeSparseIndex(sparseIndex.get());
358 buildFinalIndex(&index, sparseIndex.get());
361 // Even in empty index there is always terminating node for the root group
362 DE_ASSERT(!index.empty());
364 if (!de::FilePath(indexPath.getDirName()).exists())
365 de::createDirectoryAndParents(indexPath.getDirName().c_str());
368 std::ofstream indexOut(indexPath.getPath(), std::ios_base::binary);
370 if (!indexOut.is_open() || !indexOut.good())
371 throw tcu::InternalError(string("Failed to open program binary index file ") + indexPath.getPath());
373 indexOut.write((const char*)&index[0], index.size()*sizeof(BinaryIndexNode));
377 // BinaryRegistryReader
379 BinaryRegistryReader::BinaryRegistryReader (const tcu::Archive& archive, const std::string& srcPath)
380 : m_archive (archive)
381 , m_srcPath (srcPath)
385 BinaryRegistryReader::~BinaryRegistryReader (void)
389 ProgramBinary* BinaryRegistryReader::loadProgram (const ProgramIdentifier& id) const
395 m_binaryIndex = BinaryIndexPtr(new BinaryIndexAccess(de::MovePtr<tcu::Resource>(m_archive.getResource(getIndexPath(m_srcPath).c_str()))));
397 catch (const tcu::ResourceError& e)
399 throw ProgramNotFoundException(id, string("Failed to open binary index (") + e.what() + ")");
404 const deUint32* indexPos = findBinaryIndex(m_binaryIndex.get(), id);
408 const string fullPath = getProgramPath(m_srcPath, *indexPos);
412 de::UniquePtr<tcu::Resource> progRes (m_archive.getResource(fullPath.c_str()));
413 const int progSize = progRes->getSize();
414 vector<deUint8> bytes (progSize);
416 TCU_CHECK_INTERNAL(!bytes.empty());
418 progRes->read(&bytes[0], progSize);
420 return new ProgramBinary(vk::PROGRAM_FORMAT_SPIRV, bytes.size(), &bytes[0]);
422 catch (const tcu::ResourceError& e)
424 throw ProgramNotFoundException(id, e.what());
428 throw ProgramNotFoundException(id, "Program not found in index");
432 } // BinaryRegistryDetail