external/vulkancts/framework/vulkan/vkBinaryRegistry.cpp

   1 /*-------------------------------------------------------------------------
   2  * Vulkan CTS Framework
   3  * --------------------
   4  *
   5  * Copyright (c) 2015 Google Inc.
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Program binary registry.
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "vkBinaryRegistry.hpp"
  25 #include "tcuResource.hpp"
  26 #include "tcuFormatUtil.hpp"
  27 #include "deFilePath.hpp"
  28 #include "deStringUtil.hpp"
  29 #include "deString.h"
  30 #include "deInt32.h"
  31
  32 #include <sstream>
  33 #include <fstream>
  34 #include <stdexcept>
  35 #include <limits>
  36
  37 namespace vk
  38 {
  39 namespace BinaryRegistryDetail
  40 {
  41
  42 using std::string;
  43 using std::vector;
  44
  45 namespace
  46 {
  47
  48 string getProgramPath (const std::string& dirName, deUint32 index)
  49 {
  50         return de::FilePath::join(dirName, de::toString(tcu::toHex(index)) + ".spv").getPath();
  51 }
  52
  53 string getIndexPath (const std::string& dirName)
  54 {
  55         return de::FilePath::join(dirName, "index.bin").getPath();
  56 }
  57
  58 void writeBinary (const std::string& dstDir, deUint32 index, const ProgramBinary& binary)
  59 {
  60         const de::FilePath      fullPath        = getProgramPath(dstDir, index);
  61
  62         if (!de::FilePath(fullPath.getDirName()).exists())
  63                 de::createDirectoryAndParents(fullPath.getDirName().c_str());
  64
  65         {
  66                 std::ofstream   out             (fullPath.getPath(), std::ios_base::binary);
  67
  68                 if (!out.is_open() || !out.good())
  69                         throw tcu::Exception("Failed to open " + string(fullPath.getPath()));
  70
  71                 out.write((const char*)binary.getBinary(), binary.getSize());
  72                 out.close();
  73         }
  74 }
  75
  76 deUint32 binaryHash (const ProgramBinary* binary)
  77 {
  78         return deMemoryHash(binary->getBinary(), binary->getSize());
  79 }
  80
  81 deBool binaryEqual (const ProgramBinary* a, const ProgramBinary* b)
  82 {
  83         if (a->getSize() == b->getSize())
  84                 return deMemoryEqual(a->getBinary(), b->getBinary(), a->getSize());
  85         else
  86                 return DE_FALSE;
  87 }
  88
  89 std::vector<deUint32> getSearchPath (const ProgramIdentifier& id)
  90 {
  91         const std::string       combinedStr             = id.testCasePath + '#' + id.programName;
  92         const size_t            strLen                  = combinedStr.size();
  93         const size_t            numWords                = strLen/4 + 1;         // Must always end up with at least one 0 byte
  94         vector<deUint32>        words                   (numWords, 0u);
  95
  96         deMemcpy(&words[0], combinedStr.c_str(), strLen);
  97
  98         return words;
  99 }
 100
 101 const deUint32* findBinaryIndex (BinaryIndexAccess* index, const ProgramIdentifier& id)
 102 {
 103         const vector<deUint32>  words   = getSearchPath(id);
 104         size_t                                  nodeNdx = 0;
 105         size_t                                  wordNdx = 0;
 106
 107         for (;;)
 108         {
 109                 const BinaryIndexNode&  curNode = (*index)[nodeNdx];
 110
 111                 if (curNode.word == words[wordNdx])
 112                 {
 113                         if (wordNdx+1 < words.size())
 114                         {
 115                                 TCU_CHECK_INTERNAL((size_t)curNode.index < index->size());
 116
 117                                 nodeNdx  = curNode.index;
 118                                 wordNdx += 1;
 119                         }
 120                         else if (wordNdx+1 == words.size())
 121                                 return &curNode.index;
 122                         else
 123                                 return DE_NULL;
 124                 }
 125                 else if (curNode.word != 0)
 126                 {
 127                         nodeNdx += 1;
 128
 129                         // Index should always be null-terminated
 130                         TCU_CHECK_INTERNAL(nodeNdx < index->size());
 131                 }
 132                 else
 133                         return DE_NULL;
 134         }
 135
 136         return DE_NULL;
 137 }
 138
 139 //! Sparse index node used for final binary index construction
 140 struct SparseIndexNode
 141 {
 142         deUint32                                                word;
 143         deUint32                                                index;
 144         std::vector<SparseIndexNode*>   children;
 145
 146         SparseIndexNode (deUint32 word_, deUint32 index_)
 147                 : word  (word_)
 148                 , index (index_)
 149         {}
 150
 151         SparseIndexNode (void)
 152                 : word  (0)
 153                 , index (0)
 154         {}
 155
 156         ~SparseIndexNode (void)
 157         {
 158                 for (size_t ndx = 0; ndx < children.size(); ndx++)
 159                         delete children[ndx];
 160         }
 161 };
 162
 163 #if defined(DE_DEBUG)
 164 bool isNullByteTerminated (deUint32 word)
 165 {
 166         deUint8 bytes[4];
 167         deMemcpy(bytes, &word, sizeof(word));
 168         return bytes[3] == 0;
 169 }
 170 #endif
 171
 172 void addToSparseIndex (SparseIndexNode* group, const deUint32* words, size_t numWords, deUint32 index)
 173 {
 174         const deUint32          curWord = words[0];
 175         SparseIndexNode*        child   = DE_NULL;
 176
 177         for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
 178         {
 179                 if (group->children[childNdx]->word == curWord)
 180                 {
 181                         child = group->children[childNdx];
 182                         break;
 183                 }
 184         }
 185
 186         DE_ASSERT(numWords > 1 || !child);
 187
 188         if (!child)
 189         {
 190                 group->children.reserve(group->children.size()+1);
 191                 group->children.push_back(new SparseIndexNode(curWord, numWords == 1 ? index : 0));
 192
 193                 child = group->children.back();
 194         }
 195
 196         if (numWords > 1)
 197                 addToSparseIndex(child, words+1, numWords-1, index);
 198         else
 199                 DE_ASSERT(isNullByteTerminated(curWord));
 200 }
 201
 202 // Prepares sparse index for finalization. Ensures that child with word = 0 is moved
 203 // to the end, or one is added if there is no such child already.
 204 void normalizeSparseIndex (SparseIndexNode* group)
 205 {
 206         int             zeroChildPos    = -1;
 207
 208         for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
 209         {
 210                 normalizeSparseIndex(group->children[childNdx]);
 211
 212                 if (group->children[childNdx]->word == 0)
 213                 {
 214                         DE_ASSERT(zeroChildPos < 0);
 215                         zeroChildPos = (int)childNdx;
 216                 }
 217         }
 218
 219         if (zeroChildPos >= 0)
 220         {
 221                 // Move child with word = 0 to last
 222                 while (zeroChildPos != (int)group->children.size()-1)
 223                 {
 224                         std::swap(group->children[zeroChildPos], group->children[zeroChildPos+1]);
 225                         zeroChildPos += 1;
 226                 }
 227         }
 228         else if (!group->children.empty())
 229         {
 230                 group->children.reserve(group->children.size()+1);
 231                 group->children.push_back(new SparseIndexNode(0, 0));
 232         }
 233 }
 234
 235 deUint32 getIndexSize (const SparseIndexNode* group)
 236 {
 237         size_t  numNodes        = group->children.size();
 238
 239         for (size_t childNdx = 0; childNdx < group->children.size(); childNdx++)
 240                 numNodes += getIndexSize(group->children[childNdx]);
 241
 242         DE_ASSERT(numNodes <= std::numeric_limits<deUint32>::max());
 243
 244         return (deUint32)numNodes;
 245 }
 246
 247 deUint32 addAndCountNodes (BinaryIndexNode* index, deUint32 baseOffset, const SparseIndexNode* group)
 248 {
 249         const deUint32  numLocalNodes   = (deUint32)group->children.size();
 250         deUint32                curOffset               = numLocalNodes;
 251
 252         // Must be normalized prior to construction of final index
 253         DE_ASSERT(group->children.empty() || group->children.back()->word == 0);
 254
 255         for (size_t childNdx = 0; childNdx < numLocalNodes; childNdx++)
 256         {
 257                 const SparseIndexNode*  child           = group->children[childNdx];
 258                 const deUint32                  subtreeSize     = addAndCountNodes(index+curOffset, baseOffset+curOffset, child);
 259
 260                 index[childNdx].word = child->word;
 261
 262                 if (subtreeSize == 0)
 263                         index[childNdx].index = child->index;
 264                 else
 265                 {
 266                         DE_ASSERT(child->index == 0);
 267                         index[childNdx].index = baseOffset+curOffset;
 268                 }
 269
 270                 curOffset += subtreeSize;
 271         }
 272
 273         return curOffset;
 274 }
 275
 276 void buildFinalIndex (std::vector<BinaryIndexNode>* dst, const SparseIndexNode* root)
 277 {
 278         const deUint32  indexSize       = getIndexSize(root);
 279
 280         if (indexSize > 0)
 281         {
 282                 dst->resize(indexSize);
 283                 addAndCountNodes(&(*dst)[0], 0, root);
 284         }
 285         else
 286         {
 287                 // Generate empty index
 288                 dst->resize(1);
 289                 (*dst)[0].word  = 0u;
 290                 (*dst)[0].index = 0u;
 291         }
 292 }
 293
 294 } // anonymous
 295
 296 // BinaryRegistryWriter
 297
 298 DE_IMPLEMENT_POOL_HASH(BinaryHash, const ProgramBinary*, deUint32, binaryHash, binaryEqual);
 299
 300 BinaryRegistryWriter::BinaryRegistryWriter (const std::string& dstPath)
 301         : m_dstPath                     (dstPath)
 302         , m_binaryIndexMap      (DE_NULL)
 303 {
 304         m_binaryIndexMap = BinaryHash_create(m_memPool.getRawPool());
 305
 306         if (!m_binaryIndexMap)
 307                 throw std::bad_alloc();
 308 }
 309
 310 BinaryRegistryWriter::~BinaryRegistryWriter (void)
 311 {
 312         for (BinaryVector::const_iterator binaryIter = m_compactedBinaries.begin();
 313                  binaryIter != m_compactedBinaries.end();
 314                  ++binaryIter)
 315                 delete *binaryIter;
 316 }
 317
 318 void BinaryRegistryWriter::storeProgram (const ProgramIdentifier& id, const ProgramBinary& binary)
 319 {
 320         const deUint32* const   indexPtr        = BinaryHash_find(m_binaryIndexMap, &binary);
 321         deUint32                                index           = indexPtr ? *indexPtr : ~0u;
 322
 323         DE_ASSERT(binary.getFormat() == vk::PROGRAM_FORMAT_SPIRV);
 324
 325         if (!indexPtr)
 326         {
 327                 ProgramBinary* const    binaryClone             = new ProgramBinary(binary);
 328
 329                 try
 330                 {
 331                         index = (deUint32)m_compactedBinaries.size();
 332                         m_compactedBinaries.push_back(binaryClone);
 333                 }
 334                 catch (...)
 335                 {
 336                         delete binaryClone;
 337                         throw;
 338                 }
 339
 340                 writeBinary(m_dstPath, index, binary);
 341
 342                 if (!BinaryHash_insert(m_binaryIndexMap, binaryClone, index))
 343                         throw std::bad_alloc();
 344         }
 345
 346         DE_ASSERT((size_t)index < m_compactedBinaries.size());
 347
 348         m_binaryIndices.push_back(BinaryIndex(id, index));
 349 }
 350
 351 void BinaryRegistryWriter::writeIndex (void) const
 352 {
 353         const de::FilePath                              indexPath       = getIndexPath(m_dstPath);
 354         std::vector<BinaryIndexNode>    index;
 355
 356         {
 357                 de::UniquePtr<SparseIndexNode>  sparseIndex     (new SparseIndexNode());
 358
 359                 for (size_t progNdx = 0; progNdx < m_binaryIndices.size(); progNdx++)
 360                 {
 361                         const std::vector<deUint32>     searchPath      = getSearchPath(m_binaryIndices[progNdx].id);
 362                         addToSparseIndex(sparseIndex.get(), &searchPath[0], searchPath.size(), m_binaryIndices[progNdx].index);
 363                 }
 364
 365                 normalizeSparseIndex(sparseIndex.get());
 366                 buildFinalIndex(&index, sparseIndex.get());
 367         }
 368
 369         // Even in empty index there is always terminating node for the root group
 370         DE_ASSERT(!index.empty());
 371
 372         if (!de::FilePath(indexPath.getDirName()).exists())
 373                 de::createDirectoryAndParents(indexPath.getDirName().c_str());
 374
 375         {
 376                 std::ofstream indexOut(indexPath.getPath(), std::ios_base::binary);
 377
 378                 if (!indexOut.is_open() || !indexOut.good())
 379                         throw tcu::InternalError(string("Failed to open program binary index file ") + indexPath.getPath());
 380
 381                 indexOut.write((const char*)&index[0], index.size()*sizeof(BinaryIndexNode));
 382         }
 383 }
 384
 385 // BinaryRegistryReader
 386
 387 BinaryRegistryReader::BinaryRegistryReader (const tcu::Archive& archive, const std::string& srcPath)
 388         : m_archive     (archive)
 389         , m_srcPath     (srcPath)
 390 {
 391 }
 392
 393 BinaryRegistryReader::~BinaryRegistryReader (void)
 394 {
 395 }
 396
 397 ProgramBinary* BinaryRegistryReader::loadProgram (const ProgramIdentifier& id) const
 398 {
 399         if (!m_binaryIndex)
 400         {
 401                 try
 402                 {
 403                         m_binaryIndex = BinaryIndexPtr(new BinaryIndexAccess(de::MovePtr<tcu::Resource>(m_archive.getResource(getIndexPath(m_srcPath).c_str()))));
 404                 }
 405                 catch (const tcu::ResourceError& e)
 406                 {
 407                         throw ProgramNotFoundException(id, string("Failed to open binary index (") + e.what() + ")");
 408                 }
 409         }
 410
 411         {
 412                 const deUint32* indexPos        = findBinaryIndex(m_binaryIndex.get(), id);
 413
 414                 if (indexPos)
 415                 {
 416                         const string    fullPath        = getProgramPath(m_srcPath, *indexPos);
 417
 418                         try
 419                         {
 420                                 de::UniquePtr<tcu::Resource>    progRes         (m_archive.getResource(fullPath.c_str()));
 421                                 const int                                               progSize        = progRes->getSize();
 422                                 vector<deUint8>                                 bytes           (progSize);
 423
 424                                 TCU_CHECK_INTERNAL(!bytes.empty());
 425
 426                                 progRes->read(&bytes[0], progSize);
 427
 428                                 return new ProgramBinary(vk::PROGRAM_FORMAT_SPIRV, bytes.size(), &bytes[0]);
 429                         }
 430                         catch (const tcu::ResourceError& e)
 431                         {
 432                                 throw ProgramNotFoundException(id, e.what());
 433                         }
 434                 }
 435                 else
 436                         throw ProgramNotFoundException(id, "Program not found in index");
 437         }
 438 }
 439
 440 } // BinaryRegistryDetail
 441 } // vk