external/vulkancts/modules/vulkan/memory/vktMemoryMappingTests.cpp

   1 /*-------------------------------------------------------------------------
   2  * Vulkan Conformance Tests
   3  * ------------------------
   4  *
   5  * Copyright (c) 2015 Google Inc.
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Simple memory mapping tests.
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "vktMemoryMappingTests.hpp"
  25
  26 #include "vktTestCaseUtil.hpp"
  27
  28 #include "tcuMaybe.hpp"
  29 #include "tcuResultCollector.hpp"
  30 #include "tcuTestLog.hpp"
  31 #include "tcuPlatform.hpp"
  32
  33 #include "vkDeviceUtil.hpp"
  34 #include "vkPlatform.hpp"
  35 #include "vkQueryUtil.hpp"
  36 #include "vkRef.hpp"
  37 #include "vkRefUtil.hpp"
  38 #include "vkStrUtil.hpp"
  39 #include "vkAllocationCallbackUtil.hpp"
  40
  41 #include "deRandom.hpp"
  42 #include "deSharedPtr.hpp"
  43 #include "deStringUtil.hpp"
  44 #include "deUniquePtr.hpp"
  45 #include "deSTLUtil.hpp"
  46
  47 #include <string>
  48 #include <vector>
  49 #include <algorithm>
  50
  51 using tcu::Maybe;
  52 using tcu::TestLog;
  53
  54 using de::SharedPtr;
  55
  56 using std::string;
  57 using std::vector;
  58 using std::pair;
  59
  60 using namespace vk;
  61
  62 namespace vkt
  63 {
  64 namespace memory
  65 {
  66 namespace
  67 {
  68 template<typename T>
  69 T divRoundUp (const T& a, const T& b)
  70 {
  71         return (a / b) + (a % b == 0 ? 0 : 1);
  72 }
  73
  74 template<typename T>
  75 T roundDownToMultiple (const T& a, const T& b)
  76 {
  77         return b * (a / b);
  78 }
  79
  80 template<typename T>
  81 T roundUpToMultiple (const T& a, const T& b)
  82 {
  83         return b * (a / b + (a % b != 0 ? 1 : 0));
  84 }
  85
  86 // \note Bit vector that guarantees that each value takes only one bit.
  87 // std::vector<bool> is often optimized to only take one bit for each bool, but
  88 // that is implementation detail and in this case we really need to known how much
  89 // memory is used.
  90 class BitVector
  91 {
  92 public:
  93         enum
  94         {
  95                 BLOCK_BIT_SIZE = 8 * sizeof(deUint32)
  96         };
  97
  98         BitVector (size_t size, bool value = false)
  99                 : m_data(divRoundUp<size_t>(size, (size_t)BLOCK_BIT_SIZE), value ? ~0x0u : 0x0u)
 100         {
 101         }
 102
 103         bool get (size_t ndx) const
 104         {
 105                 return (m_data[ndx / BLOCK_BIT_SIZE] & (0x1u << (deUint32)(ndx % BLOCK_BIT_SIZE))) != 0;
 106         }
 107
 108         void set (size_t ndx, bool value)
 109         {
 110                 if (value)
 111                         m_data[ndx / BLOCK_BIT_SIZE] |= 0x1u << (deUint32)(ndx % BLOCK_BIT_SIZE);
 112                 else
 113                         m_data[ndx / BLOCK_BIT_SIZE] &= ~(0x1u << (deUint32)(ndx % BLOCK_BIT_SIZE));
 114         }
 115
 116         void setRange (size_t offset, size_t count, bool value)
 117         {
 118                 size_t ndx = offset;
 119
 120                 for (; (ndx < offset + count) && ((ndx % BLOCK_BIT_SIZE) != 0); ndx++)
 121                 {
 122                         DE_ASSERT(ndx >= offset);
 123                         DE_ASSERT(ndx < offset + count);
 124                         set(ndx, value);
 125                 }
 126
 127                 {
 128                         const size_t endOfFullBlockNdx = roundDownToMultiple<size_t>(offset + count, BLOCK_BIT_SIZE);
 129
 130                         if (ndx < endOfFullBlockNdx)
 131                         {
 132                                 deMemset(&m_data[ndx / BLOCK_BIT_SIZE], (value ? 0xFF : 0x0), (endOfFullBlockNdx - ndx) / 8);
 133                                 ndx = endOfFullBlockNdx;
 134                         }
 135                 }
 136
 137                 for (; ndx < offset + count; ndx++)
 138                 {
 139                         DE_ASSERT(ndx >= offset);
 140                         DE_ASSERT(ndx < offset + count);
 141                         set(ndx, value);
 142                 }
 143         }
 144
 145         void vectorAnd (const BitVector& other, size_t offset, size_t count)
 146         {
 147                 size_t ndx = offset;
 148
 149                 for (; ndx < offset + count && (ndx % BLOCK_BIT_SIZE) != 0; ndx++)
 150                 {
 151                         DE_ASSERT(ndx >= offset);
 152                         DE_ASSERT(ndx < offset + count);
 153                         set(ndx, other.get(ndx) && get(ndx));
 154                 }
 155
 156                 for (; ndx < roundDownToMultiple<size_t>(offset + count, BLOCK_BIT_SIZE); ndx += BLOCK_BIT_SIZE)
 157                 {
 158                         DE_ASSERT(ndx >= offset);
 159                         DE_ASSERT(ndx < offset + count);
 160                         DE_ASSERT(ndx % BLOCK_BIT_SIZE == 0);
 161                         DE_ASSERT(ndx + BLOCK_BIT_SIZE <= offset + count);
 162                         m_data[ndx / BLOCK_BIT_SIZE] &= other.m_data[ndx / BLOCK_BIT_SIZE];
 163                 }
 164
 165                 for (; ndx < offset + count; ndx++)
 166                 {
 167                         DE_ASSERT(ndx >= offset);
 168                         DE_ASSERT(ndx < offset + count);
 169                         set(ndx, other.get(ndx) && get(ndx));
 170                 }
 171         }
 172
 173 private:
 174         vector<deUint32>        m_data;
 175 };
 176
 177 class ReferenceMemory
 178 {
 179 public:
 180         ReferenceMemory (size_t size, size_t atomSize)
 181                 : m_atomSize    (atomSize)
 182                 , m_bytes               (size, 0xDEu)
 183                 , m_defined             (size, false)
 184                 , m_flushed             (size / atomSize, false)
 185         {
 186                 DE_ASSERT(size % m_atomSize == 0);
 187         }
 188
 189         void write (size_t pos, deUint8 value)
 190         {
 191                 m_bytes[pos] = value;
 192                 m_defined.set(pos, true);
 193                 m_flushed.set(pos / m_atomSize, false);
 194         }
 195
 196         bool read (size_t pos, deUint8 value)
 197         {
 198                 const bool isOk = !m_defined.get(pos)
 199                                                 || m_bytes[pos] == value;
 200
 201                 m_bytes[pos] = value;
 202                 m_defined.set(pos, true);
 203
 204                 return isOk;
 205         }
 206
 207         bool modifyXor (size_t pos, deUint8 value, deUint8 mask)
 208         {
 209                 const bool isOk = !m_defined.get(pos)
 210                                                 || m_bytes[pos] == value;
 211
 212                 m_bytes[pos] = value ^ mask;
 213                 m_defined.set(pos, true);
 214                 m_flushed.set(pos / m_atomSize, false);
 215
 216                 return isOk;
 217         }
 218
 219         void flush (size_t offset, size_t size)
 220         {
 221                 DE_ASSERT((offset % m_atomSize) == 0);
 222                 DE_ASSERT((size % m_atomSize) == 0);
 223
 224                 m_flushed.setRange(offset / m_atomSize, size / m_atomSize, true);
 225         }
 226
 227         void invalidate (size_t offset, size_t size)
 228         {
 229                 DE_ASSERT((offset % m_atomSize) == 0);
 230                 DE_ASSERT((size % m_atomSize) == 0);
 231
 232                 if (m_atomSize == 1)
 233                 {
 234                         m_defined.vectorAnd(m_flushed, offset, size);
 235                 }
 236                 else
 237                 {
 238                         for (size_t ndx = 0; ndx < size / m_atomSize; ndx++)
 239                         {
 240                                 if (!m_flushed.get((offset / m_atomSize) + ndx))
 241                                         m_defined.setRange(offset + ndx * m_atomSize, m_atomSize, false);
 242                         }
 243                 }
 244         }
 245
 246
 247 private:
 248         const size_t    m_atomSize;
 249         vector<deUint8> m_bytes;
 250         BitVector               m_defined;
 251         BitVector               m_flushed;
 252 };
 253
 254 struct MemoryType
 255 {
 256         MemoryType              (deUint32 index_, const VkMemoryType& type_)
 257                 : index (index_)
 258                 , type  (type_)
 259         {
 260         }
 261
 262         MemoryType              (void)
 263                 : index (~0u)
 264         {
 265         }
 266
 267         deUint32                index;
 268         VkMemoryType    type;
 269 };
 270
 271 size_t computeDeviceMemorySystemMemFootprint (const DeviceInterface& vk, VkDevice device)
 272 {
 273         AllocationCallbackRecorder      callbackRecorder        (getSystemAllocator());
 274
 275         {
 276                 // 1 B allocation from memory type 0
 277                 const VkMemoryAllocateInfo      allocInfo       =
 278                 {
 279                         VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
 280                         DE_NULL,
 281                         1u,
 282                         0u,
 283                 };
 284                 const Unique<VkDeviceMemory>                    memory                  (allocateMemory(vk, device, &allocInfo));
 285                 AllocationCallbackValidationResults             validateRes;
 286
 287                 validateAllocationCallbacks(callbackRecorder, &validateRes);
 288
 289                 TCU_CHECK(validateRes.violations.empty());
 290
 291                 return getLiveSystemAllocationTotal(validateRes)
 292                            + sizeof(void*)*validateRes.liveAllocations.size(); // allocation overhead
 293         }
 294 }
 295
 296 Move<VkDeviceMemory> allocMemory (const DeviceInterface& vk, VkDevice device, VkDeviceSize pAllocInfo_allocationSize, deUint32 pAllocInfo_memoryTypeIndex)
 297 {
 298         const VkMemoryAllocateInfo pAllocInfo =
 299         {
 300                 VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
 301                 DE_NULL,
 302                 pAllocInfo_allocationSize,
 303                 pAllocInfo_memoryTypeIndex,
 304         };
 305         return allocateMemory(vk, device, &pAllocInfo);
 306 }
 307
 308 struct MemoryRange
 309 {
 310         MemoryRange (VkDeviceSize offset_ = ~(VkDeviceSize)0, VkDeviceSize size_ = ~(VkDeviceSize)0)
 311                 : offset        (offset_)
 312                 , size          (size_)
 313         {
 314         }
 315
 316         VkDeviceSize    offset;
 317         VkDeviceSize    size;
 318 };
 319
 320 struct TestConfig
 321 {
 322         TestConfig (void)
 323                 : allocationSize        (~(VkDeviceSize)0)
 324         {
 325         }
 326
 327         VkDeviceSize            allocationSize;
 328         deUint32                        seed;
 329
 330         MemoryRange                     mapping;
 331         vector<MemoryRange>     flushMappings;
 332         vector<MemoryRange>     invalidateMappings;
 333         bool                            remap;
 334 };
 335
 336 bool compareAndLogBuffer (TestLog& log, size_t size, const deUint8* result, const deUint8* reference)
 337 {
 338         size_t  failedBytes     = 0;
 339         size_t  firstFailed     = (size_t)-1;
 340
 341         for (size_t ndx = 0; ndx < size; ndx++)
 342         {
 343                 if (result[ndx] != reference[ndx])
 344                 {
 345                         failedBytes++;
 346
 347                         if (firstFailed == (size_t)-1)
 348                                 firstFailed = ndx;
 349                 }
 350         }
 351
 352         if (failedBytes > 0)
 353         {
 354                 log << TestLog::Message << "Comparison failed. Failed bytes " << failedBytes << ". First failed at offset " << firstFailed << "." << TestLog::EndMessage;
 355
 356                 std::ostringstream      expectedValues;
 357                 std::ostringstream      resultValues;
 358
 359                 for (size_t ndx = firstFailed; ndx < firstFailed + 10 && ndx < size; ndx++)
 360                 {
 361                         if (ndx != firstFailed)
 362                         {
 363                                 expectedValues << ", ";
 364                                 resultValues << ", ";
 365                         }
 366
 367                         expectedValues << reference[ndx];
 368                         resultValues << result[ndx];
 369                 }
 370
 371                 if (firstFailed + 10 < size)
 372                 {
 373                         expectedValues << "...";
 374                         resultValues << "...";
 375                 }
 376
 377                 log << TestLog::Message << "Expected values at offset: " << firstFailed << ", " << expectedValues.str() << TestLog::EndMessage;
 378                 log << TestLog::Message << "Result values at offset: " << firstFailed << ", " << resultValues.str() << TestLog::EndMessage;
 379
 380                 return false;
 381         }
 382         else
 383                 return true;
 384 }
 385
 386 tcu::TestStatus testMemoryMapping (Context& context, const TestConfig config)
 387 {
 388         TestLog&                                                                log                                     = context.getTestContext().getLog();
 389         tcu::ResultCollector                                    result                          (log);
 390         const VkPhysicalDevice                                  physicalDevice          = context.getPhysicalDevice();
 391         const VkDevice                                                  device                          = context.getDevice();
 392         const InstanceInterface&                                vki                                     = context.getInstanceInterface();
 393         const DeviceInterface&                                  vkd                                     = context.getDeviceInterface();
 394         const VkPhysicalDeviceMemoryProperties  memoryProperties        = getPhysicalDeviceMemoryProperties(vki, physicalDevice);
 395         // \todo [2016-05-27 misojarvi] Remove once drivers start reporting correctly nonCoherentAtomSize that is at least 1.
 396         const VkDeviceSize                                              nonCoherentAtomSize     = context.getDeviceProperties().limits.nonCoherentAtomSize != 0
 397                                                                                                                                 ? context.getDeviceProperties().limits.nonCoherentAtomSize
 398                                                                                                                                 : 1;
 399
 400         {
 401                 const tcu::ScopedLogSection     section (log, "TestCaseInfo", "TestCaseInfo");
 402
 403                 log << TestLog::Message << "Seed: " << config.seed << TestLog::EndMessage;
 404                 log << TestLog::Message << "Allocation size: " << config.allocationSize << " * atom" <<  TestLog::EndMessage;
 405                 log << TestLog::Message << "Mapping, offset: " << config.mapping.offset << " * atom, size: " << config.mapping.size << " * atom" << TestLog::EndMessage;
 406
 407                 if (!config.flushMappings.empty())
 408                 {
 409                         log << TestLog::Message << "Invalidating following ranges:" << TestLog::EndMessage;
 410
 411                         for (size_t ndx = 0; ndx < config.flushMappings.size(); ndx++)
 412                                 log << TestLog::Message << "\tOffset: " << config.flushMappings[ndx].offset << " * atom, Size: " << config.flushMappings[ndx].size << " * atom" << TestLog::EndMessage;
 413                 }
 414
 415                 if (config.remap)
 416                         log << TestLog::Message << "Remapping memory between flush and invalidation." << TestLog::EndMessage;
 417
 418                 if (!config.invalidateMappings.empty())
 419                 {
 420                         log << TestLog::Message << "Flushing following ranges:" << TestLog::EndMessage;
 421
 422                         for (size_t ndx = 0; ndx < config.invalidateMappings.size(); ndx++)
 423                                 log << TestLog::Message << "\tOffset: " << config.invalidateMappings[ndx].offset << " * atom, Size: " << config.invalidateMappings[ndx].size << " * atom" << TestLog::EndMessage;
 424                 }
 425         }
 426
 427         for (deUint32 memoryTypeIndex = 0; memoryTypeIndex < memoryProperties.memoryTypeCount; memoryTypeIndex++)
 428         {
 429                 try
 430                 {
 431                         const tcu::ScopedLogSection             section         (log, "MemoryType" + de::toString(memoryTypeIndex), "MemoryType" + de::toString(memoryTypeIndex));
 432                         const VkMemoryType&                             memoryType      = memoryProperties.memoryTypes[memoryTypeIndex];
 433                         const VkMemoryHeap&                             memoryHeap      = memoryProperties.memoryHeaps[memoryType.heapIndex];
 434                         const VkDeviceSize                              atomSize        = (memoryType.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0
 435                                                                                                                 ? 1
 436                                                                                                                 : nonCoherentAtomSize;
 437
 438                         log << TestLog::Message << "MemoryType: " << memoryType << TestLog::EndMessage;
 439                         log << TestLog::Message << "MemoryHeap: " << memoryHeap << TestLog::EndMessage;
 440                         log << TestLog::Message << "AtomSize: " << atomSize << TestLog::EndMessage;
 441                         log << TestLog::Message << "AllocationSize: " << config.allocationSize * atomSize <<  TestLog::EndMessage;
 442                         log << TestLog::Message << "Mapping, offset: " << config.mapping.offset * atomSize << ", size: " << config.mapping.size * atomSize << TestLog::EndMessage;
 443
 444                         if (!config.flushMappings.empty())
 445                         {
 446                                 log << TestLog::Message << "Invalidating following ranges:" << TestLog::EndMessage;
 447
 448                                 for (size_t ndx = 0; ndx < config.flushMappings.size(); ndx++)
 449                                         log << TestLog::Message << "\tOffset: " << config.flushMappings[ndx].offset * atomSize << ", Size: " << config.flushMappings[ndx].size * atomSize << TestLog::EndMessage;
 450                         }
 451
 452                         if (!config.invalidateMappings.empty())
 453                         {
 454                                 log << TestLog::Message << "Flushing following ranges:" << TestLog::EndMessage;
 455
 456                                 for (size_t ndx = 0; ndx < config.invalidateMappings.size(); ndx++)
 457                                         log << TestLog::Message << "\tOffset: " << config.invalidateMappings[ndx].offset * atomSize << ", Size: " << config.invalidateMappings[ndx].size * atomSize << TestLog::EndMessage;
 458                         }
 459
 460                         if ((memoryType.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
 461                         {
 462                                 log << TestLog::Message << "Memory type doesn't support mapping." << TestLog::EndMessage;
 463                         }
 464                         else if (memoryHeap.size <= 4 * atomSize * config.allocationSize)
 465                         {
 466                                 log << TestLog::Message << "Memory types heap is too small." << TestLog::EndMessage;
 467                         }
 468                         else
 469                         {
 470                                 const Unique<VkDeviceMemory>    memory                          (allocMemory(vkd, device, config.allocationSize * atomSize, memoryTypeIndex));
 471                                 de::Random                                              rng                                     (config.seed);
 472                                 vector<deUint8>                                 reference                       ((size_t)(config.allocationSize * atomSize));
 473                                 deUint8*                                                mapping                         = DE_NULL;
 474
 475                                 {
 476                                         void* ptr;
 477                                         VK_CHECK(vkd.mapMemory(device, *memory, config.mapping.offset * atomSize, config.mapping.size * atomSize, 0u, &ptr));
 478                                         TCU_CHECK(ptr);
 479
 480                                         mapping = (deUint8*)ptr;
 481                                 }
 482
 483                                 for (VkDeviceSize ndx = 0; ndx < config.mapping.size * atomSize; ndx++)
 484                                 {
 485                                         const deUint8 val = rng.getUint8();
 486
 487                                         mapping[ndx]                                                                                            = val;
 488                                         reference[(size_t)(config.mapping.offset * atomSize + ndx)]     = val;
 489                                 }
 490
 491                                 if (!config.flushMappings.empty())
 492                                 {
 493                                         vector<VkMappedMemoryRange> ranges;
 494
 495                                         for (size_t ndx = 0; ndx < config.flushMappings.size(); ndx++)
 496                                         {
 497                                                 const VkMappedMemoryRange range =
 498                                                 {
 499                                                         VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
 500                                                         DE_NULL,
 501
 502                                                         *memory,
 503                                                         config.flushMappings[ndx].offset * atomSize,
 504                                                         config.flushMappings[ndx].size * atomSize
 505                                                 };
 506
 507                                                 ranges.push_back(range);
 508                                         }
 509
 510                                         VK_CHECK(vkd.flushMappedMemoryRanges(device, (deUint32)ranges.size(), &ranges[0]));
 511                                 }
 512
 513                                 if (config.remap)
 514                                 {
 515                                         void* ptr;
 516                                         vkd.unmapMemory(device, *memory);
 517                                         VK_CHECK(vkd.mapMemory(device, *memory, config.mapping.offset * atomSize, config.mapping.size * atomSize, 0u, &ptr));
 518                                         TCU_CHECK(ptr);
 519
 520                                         mapping = (deUint8*)ptr;
 521                                 }
 522
 523                                 if (!config.invalidateMappings.empty())
 524                                 {
 525                                         vector<VkMappedMemoryRange> ranges;
 526
 527                                         for (size_t ndx = 0; ndx < config.invalidateMappings.size(); ndx++)
 528                                         {
 529                                                 const VkMappedMemoryRange range =
 530                                                 {
 531                                                         VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
 532                                                         DE_NULL,
 533
 534                                                         *memory,
 535                                                         config.invalidateMappings[ndx].offset * atomSize,
 536                                                         config.invalidateMappings[ndx].size * atomSize
 537                                                 };
 538
 539                                                 ranges.push_back(range);
 540                                         }
 541
 542                                         VK_CHECK(vkd.invalidateMappedMemoryRanges(device, (deUint32)ranges.size(), &ranges[0]));
 543                                 }
 544
 545                                 if (!compareAndLogBuffer(log, (size_t)(config.mapping.size * atomSize), mapping, &reference[(size_t)(config.mapping.offset * atomSize)]))
 546                                         result.fail("Unexpected values read from mapped memory.");
 547
 548                                 vkd.unmapMemory(device, *memory);
 549                         }
 550                 }
 551                 catch (const tcu::TestError& error)
 552                 {
 553                         result.fail(error.getMessage());
 554                 }
 555         }
 556
 557         return tcu::TestStatus(result.getResult(), result.getMessage());
 558 }
 559
 560 class MemoryMapping
 561 {
 562 public:
 563                                                 MemoryMapping   (const MemoryRange&     range,
 564                                                                                  void*                          ptr,
 565                                                                                  ReferenceMemory&       reference);
 566
 567         void                            randomRead              (de::Random& rng);
 568         void                            randomWrite             (de::Random& rng);
 569         void                            randomModify    (de::Random& rng);
 570
 571         const MemoryRange&      getRange                (void) const { return m_range; }
 572
 573 private:
 574         MemoryRange                     m_range;
 575         void*                           m_ptr;
 576         ReferenceMemory&        m_reference;
 577 };
 578
 579 MemoryMapping::MemoryMapping (const MemoryRange&        range,
 580                                                           void*                                 ptr,
 581                                                           ReferenceMemory&              reference)
 582         : m_range               (range)
 583         , m_ptr                 (ptr)
 584         , m_reference   (reference)
 585 {
 586         DE_ASSERT(range.size > 0);
 587 }
 588
 589 void MemoryMapping::randomRead (de::Random& rng)
 590 {
 591         const size_t count = (size_t)rng.getInt(0, 100);
 592
 593         for (size_t ndx = 0; ndx < count; ndx++)
 594         {
 595                 const size_t    pos     = (size_t)(rng.getUint64() % (deUint64)m_range.size);
 596                 const deUint8   val     = ((deUint8*)m_ptr)[pos];
 597
 598                 TCU_CHECK(m_reference.read((size_t)(m_range.offset + pos), val));
 599         }
 600 }
 601
 602 void MemoryMapping::randomWrite (de::Random& rng)
 603 {
 604         const size_t count = (size_t)rng.getInt(0, 100);
 605
 606         for (size_t ndx = 0; ndx < count; ndx++)
 607         {
 608                 const size_t    pos     = (size_t)(rng.getUint64() % (deUint64)m_range.size);
 609                 const deUint8   val     = rng.getUint8();
 610
 611                 ((deUint8*)m_ptr)[pos]  = val;
 612                 m_reference.write((size_t)(m_range.offset + pos), val);
 613         }
 614 }
 615
 616 void MemoryMapping::randomModify (de::Random& rng)
 617 {
 618         const size_t count = (size_t)rng.getInt(0, 100);
 619
 620         for (size_t ndx = 0; ndx < count; ndx++)
 621         {
 622                 const size_t    pos             = (size_t)(rng.getUint64() % (deUint64)m_range.size);
 623                 const deUint8   val             = ((deUint8*)m_ptr)[pos];
 624                 const deUint8   mask    = rng.getUint8();
 625
 626                 ((deUint8*)m_ptr)[pos]  = val ^ mask;
 627                 TCU_CHECK(m_reference.modifyXor((size_t)(m_range.offset + pos), val, mask));
 628         }
 629 }
 630
 631 VkDeviceSize randomSize (de::Random& rng, VkDeviceSize atomSize, VkDeviceSize maxSize)
 632 {
 633         const VkDeviceSize maxSizeInAtoms = maxSize / atomSize;
 634
 635         DE_ASSERT(maxSizeInAtoms > 0);
 636
 637         return maxSizeInAtoms > 1
 638                         ? atomSize * (1 + (VkDeviceSize)(rng.getUint64() % (deUint64)maxSizeInAtoms))
 639                         : atomSize;
 640 }
 641
 642 VkDeviceSize randomOffset (de::Random& rng, VkDeviceSize atomSize, VkDeviceSize maxOffset)
 643 {
 644         const VkDeviceSize maxOffsetInAtoms = maxOffset / atomSize;
 645
 646         return maxOffsetInAtoms > 0
 647                         ? atomSize * (VkDeviceSize)(rng.getUint64() % (deUint64)(maxOffsetInAtoms + 1))
 648                         : 0;
 649 }
 650
 651 void randomRanges (de::Random& rng, vector<VkMappedMemoryRange>& ranges, size_t count, VkDeviceMemory memory, VkDeviceSize minOffset, VkDeviceSize maxSize, VkDeviceSize atomSize)
 652 {
 653         ranges.resize(count);
 654
 655         for (size_t rangeNdx = 0; rangeNdx < count; rangeNdx++)
 656         {
 657                 const VkDeviceSize      size    = randomSize(rng, atomSize, maxSize);
 658                 const VkDeviceSize      offset  = minOffset + randomOffset(rng, atomSize, maxSize - size);
 659
 660                 const VkMappedMemoryRange range =
 661                 {
 662                         VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
 663                         DE_NULL,
 664
 665                         memory,
 666                         offset,
 667                         size
 668                 };
 669                 ranges[rangeNdx] = range;
 670         }
 671 }
 672
 673 class MemoryObject
 674 {
 675 public:
 676                                                         MemoryObject                    (const DeviceInterface&         vkd,
 677                                                                                                          VkDevice                                       device,
 678                                                                                                          VkDeviceSize                           size,
 679                                                                                                          deUint32                                       memoryTypeIndex,
 680                                                                                                          VkDeviceSize                           atomSize,
 681                                                                                                          VkDeviceSize                           memoryUsage,
 682                                                                                                          VkDeviceSize                           referenceMemoryUsage);
 683
 684                                                         ~MemoryObject                   (void);
 685
 686         MemoryMapping*                  mapRandom                               (const DeviceInterface& vkd, VkDevice device, de::Random& rng);
 687         void                                    unmap                                   (void);
 688
 689         void                                    randomFlush                             (const DeviceInterface& vkd, VkDevice device, de::Random& rng);
 690         void                                    randomInvalidate                (const DeviceInterface& vkd, VkDevice device, de::Random& rng);
 691
 692         VkDeviceSize                    getSize                                 (void) const { return m_size; }
 693         MemoryMapping*                  getMapping                              (void) { return m_mapping; }
 694
 695         VkDeviceSize                    getMemoryUsage                  (void) const { return m_memoryUsage; }
 696         VkDeviceSize                    getReferenceMemoryUsage (void) const { return m_referenceMemoryUsage; }
 697 private:
 698         const DeviceInterface&  m_vkd;
 699         const VkDevice                  m_device;
 700
 701         const deUint32                  m_memoryTypeIndex;
 702         const VkDeviceSize              m_size;
 703         const VkDeviceSize              m_atomSize;
 704         const VkDeviceSize              m_memoryUsage;
 705         const VkDeviceSize              m_referenceMemoryUsage;
 706
 707         Move<VkDeviceMemory>    m_memory;
 708
 709         MemoryMapping*                  m_mapping;
 710         ReferenceMemory                 m_referenceMemory;
 711 };
 712
 713 MemoryObject::MemoryObject (const DeviceInterface&              vkd,
 714                                                         VkDevice                                        device,
 715                                                         VkDeviceSize                            size,
 716                                                         deUint32                                        memoryTypeIndex,
 717                                                         VkDeviceSize                            atomSize,
 718                                                         VkDeviceSize                            memoryUsage,
 719                                                         VkDeviceSize                            referenceMemoryUsage)
 720         : m_vkd                                         (vkd)
 721         , m_device                                      (device)
 722         , m_memoryTypeIndex                     (memoryTypeIndex)
 723         , m_size                                        (size)
 724         , m_atomSize                            (atomSize)
 725         , m_memoryUsage                         (memoryUsage)
 726         , m_referenceMemoryUsage        (referenceMemoryUsage)
 727         , m_mapping                                     (DE_NULL)
 728         , m_referenceMemory                     ((size_t)size, (size_t)m_atomSize)
 729 {
 730         m_memory = allocMemory(m_vkd, m_device, m_size, m_memoryTypeIndex);
 731 }
 732
 733 MemoryObject::~MemoryObject (void)
 734 {
 735         delete m_mapping;
 736 }
 737
 738 MemoryMapping* MemoryObject::mapRandom (const DeviceInterface& vkd, VkDevice device, de::Random& rng)
 739 {
 740         const VkDeviceSize      size    = randomSize(rng, m_atomSize, m_size);
 741         const VkDeviceSize      offset  = randomOffset(rng, m_atomSize, m_size - size);
 742         void*                           ptr;
 743
 744         DE_ASSERT(!m_mapping);
 745
 746         VK_CHECK(vkd.mapMemory(device, *m_memory, offset, size, 0u, &ptr));
 747         TCU_CHECK(ptr);
 748         m_mapping = new MemoryMapping(MemoryRange(offset, size), ptr, m_referenceMemory);
 749
 750         return m_mapping;
 751 }
 752
 753 void MemoryObject::unmap (void)
 754 {
 755         m_vkd.unmapMemory(m_device, *m_memory);
 756
 757         delete m_mapping;
 758         m_mapping = DE_NULL;
 759 }
 760
 761 void MemoryObject::randomFlush (const DeviceInterface& vkd, VkDevice device, de::Random& rng)
 762 {
 763         const size_t                            rangeCount      = (size_t)rng.getInt(1, 10);
 764         vector<VkMappedMemoryRange>     ranges          (rangeCount);
 765
 766         randomRanges(rng, ranges, rangeCount, *m_memory, m_mapping->getRange().offset, m_mapping->getRange().size, m_atomSize);
 767
 768         for (size_t rangeNdx = 0; rangeNdx < ranges.size(); rangeNdx++)
 769                 m_referenceMemory.flush((size_t)ranges[rangeNdx].offset, (size_t)ranges[rangeNdx].size);
 770
 771         VK_CHECK(vkd.flushMappedMemoryRanges(device, (deUint32)ranges.size(), ranges.empty() ? DE_NULL : &ranges[0]));
 772 }
 773
 774 void MemoryObject::randomInvalidate (const DeviceInterface& vkd, VkDevice device, de::Random& rng)
 775 {
 776         const size_t                            rangeCount      = (size_t)rng.getInt(1, 10);
 777         vector<VkMappedMemoryRange>     ranges          (rangeCount);
 778
 779         randomRanges(rng, ranges, rangeCount, *m_memory, m_mapping->getRange().offset, m_mapping->getRange().size, m_atomSize);
 780
 781         for (size_t rangeNdx = 0; rangeNdx < ranges.size(); rangeNdx++)
 782                 m_referenceMemory.invalidate((size_t)ranges[rangeNdx].offset, (size_t)ranges[rangeNdx].size);
 783
 784         VK_CHECK(vkd.invalidateMappedMemoryRanges(device, (deUint32)ranges.size(), ranges.empty() ? DE_NULL : &ranges[0]));
 785 }
 786
 787 enum
 788 {
 789         MAX_MEMORY_USAGE_DIV = 2, // Use only 1/2 of each memory heap.
 790         MAX_MEMORY_ALLOC_DIV = 2, // Do not alloc more than 1/2 of available space.
 791 };
 792
 793 template<typename T>
 794 void removeFirstEqual (vector<T>& vec, const T& val)
 795 {
 796         for (size_t ndx = 0; ndx < vec.size(); ndx++)
 797         {
 798                 if (vec[ndx] == val)
 799                 {
 800                         vec[ndx] = vec.back();
 801                         vec.pop_back();
 802                         return;
 803                 }
 804         }
 805 }
 806
 807 enum MemoryClass
 808 {
 809         MEMORY_CLASS_SYSTEM = 0,
 810         MEMORY_CLASS_DEVICE,
 811
 812         MEMORY_CLASS_LAST
 813 };
 814
 815 // \todo [2016-04-20 pyry] Consider estimating memory fragmentation
 816 class TotalMemoryTracker
 817 {
 818 public:
 819                                         TotalMemoryTracker      (void)
 820         {
 821                 std::fill(DE_ARRAY_BEGIN(m_usage), DE_ARRAY_END(m_usage), 0);
 822         }
 823
 824         void                    allocate                        (MemoryClass memClass, VkDeviceSize size)
 825         {
 826                 m_usage[memClass] += size;
 827         }
 828
 829         void                    free                            (MemoryClass memClass, VkDeviceSize size)
 830         {
 831                 DE_ASSERT(size <= m_usage[memClass]);
 832                 m_usage[memClass] -= size;
 833         }
 834
 835         VkDeviceSize    getUsage                        (MemoryClass memClass) const
 836         {
 837                 return m_usage[memClass];
 838         }
 839
 840         VkDeviceSize    getTotalUsage           (void) const
 841         {
 842                 VkDeviceSize total = 0;
 843                 for (int ndx = 0; ndx < MEMORY_CLASS_LAST; ++ndx)
 844                         total += getUsage((MemoryClass)ndx);
 845                 return total;
 846         }
 847
 848 private:
 849         VkDeviceSize    m_usage[MEMORY_CLASS_LAST];
 850 };
 851
 852 VkDeviceSize getHostPageSize (void)
 853 {
 854         return 4096;
 855 }
 856
 857 VkDeviceSize getMinAtomSize (VkDeviceSize nonCoherentAtomSize, const vector<MemoryType>& memoryTypes)
 858 {
 859         for (size_t ndx = 0; ndx < memoryTypes.size(); ndx++)
 860         {
 861                 if ((memoryTypes[ndx].type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
 862                         return 1;
 863         }
 864
 865         return nonCoherentAtomSize;
 866 }
 867
 868 class MemoryHeap
 869 {
 870 public:
 871         MemoryHeap (const VkMemoryHeap&                 heap,
 872                                 const vector<MemoryType>&       memoryTypes,
 873                                 const PlatformMemoryLimits&     memoryLimits,
 874                                 const VkDeviceSize                      nonCoherentAtomSize,
 875                                 TotalMemoryTracker&                     totalMemTracker)
 876                 : m_heap                                (heap)
 877                 , m_memoryTypes                 (memoryTypes)
 878                 , m_limits                              (memoryLimits)
 879                 , m_nonCoherentAtomSize (nonCoherentAtomSize)
 880                 , m_minAtomSize                 (getMinAtomSize(nonCoherentAtomSize, memoryTypes))
 881                 , m_totalMemTracker             (totalMemTracker)
 882                 , m_usage                               (0)
 883         {
 884         }
 885
 886         ~MemoryHeap (void)
 887         {
 888                 for (vector<MemoryObject*>::iterator iter = m_objects.begin(); iter != m_objects.end(); ++iter)
 889                         delete *iter;
 890         }
 891
 892         bool                                                            full                    (void) const;
 893         bool                                                            empty                   (void) const
 894         {
 895                 return m_usage == 0 && !full();
 896         }
 897
 898         MemoryObject*                                           allocateRandom  (const DeviceInterface& vkd, VkDevice device, de::Random& rng);
 899
 900         MemoryObject*                                           getRandomObject (de::Random& rng) const
 901         {
 902                 return rng.choose<MemoryObject*>(m_objects.begin(), m_objects.end());
 903         }
 904
 905         void                                                            free                    (MemoryObject* object)
 906         {
 907                 removeFirstEqual(m_objects, object);
 908                 m_usage -= object->getMemoryUsage();
 909                 m_totalMemTracker.free(MEMORY_CLASS_SYSTEM, object->getReferenceMemoryUsage());
 910                 m_totalMemTracker.free(getMemoryClass(), object->getMemoryUsage());
 911                 delete object;
 912         }
 913
 914 private:
 915         MemoryClass                                                     getMemoryClass  (void) const
 916         {
 917                 if ((m_heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
 918                         return MEMORY_CLASS_DEVICE;
 919                 else
 920                         return MEMORY_CLASS_SYSTEM;
 921         }
 922
 923         const VkMemoryHeap                      m_heap;
 924         const vector<MemoryType>        m_memoryTypes;
 925         const PlatformMemoryLimits&     m_limits;
 926         const VkDeviceSize                      m_nonCoherentAtomSize;
 927         const VkDeviceSize                      m_minAtomSize;
 928         TotalMemoryTracker&                     m_totalMemTracker;
 929
 930         VkDeviceSize                            m_usage;
 931         vector<MemoryObject*>           m_objects;
 932 };
 933
 934 // Heap is full if there is not enough memory to allocate minimal memory object.
 935 bool MemoryHeap::full (void) const
 936 {
 937         DE_ASSERT(m_usage <= m_heap.size/MAX_MEMORY_USAGE_DIV);
 938
 939         const VkDeviceSize      availableInHeap         = m_heap.size/MAX_MEMORY_USAGE_DIV - m_usage;
 940         const bool                      isUMA                           = m_limits.totalDeviceLocalMemory == 0;
 941         const MemoryClass       memClass                        = getMemoryClass();
 942         const VkDeviceSize      minAllocationSize       = de::max(m_minAtomSize, memClass == MEMORY_CLASS_DEVICE ? m_limits.devicePageSize : getHostPageSize());
 943         // Memory required for reference. One byte and one bit for each byte and one bit per each m_atomSize.
 944         const VkDeviceSize      minReferenceSize        = minAllocationSize
 945                                                                                         + divRoundUp<VkDeviceSize>(minAllocationSize,  8)
 946                                                                                         + divRoundUp<VkDeviceSize>(minAllocationSize,  m_minAtomSize * 8);
 947
 948         if (isUMA)
 949         {
 950                 const VkDeviceSize      totalUsage      = m_totalMemTracker.getTotalUsage();
 951                 const VkDeviceSize      totalSysMem     = (VkDeviceSize)m_limits.totalSystemMemory;
 952
 953                 DE_ASSERT(totalUsage <= totalSysMem);
 954
 955                 return (minAllocationSize + minReferenceSize) > (totalSysMem - totalUsage)
 956                                 || minAllocationSize > availableInHeap;
 957         }
 958         else
 959         {
 960                 const VkDeviceSize      totalUsage              = m_totalMemTracker.getTotalUsage();
 961                 const VkDeviceSize      totalSysMem             = (VkDeviceSize)m_limits.totalSystemMemory;
 962
 963                 const VkDeviceSize      totalMemClass   = memClass == MEMORY_CLASS_SYSTEM
 964                                                                                         ? m_limits.totalSystemMemory
 965                                                                                         : m_limits.totalDeviceLocalMemory;
 966                 const VkDeviceSize      usedMemClass    = m_totalMemTracker.getUsage(memClass);
 967
 968                 DE_ASSERT(usedMemClass <= totalMemClass);
 969
 970                 return minAllocationSize > availableInHeap
 971                                 || minAllocationSize > (totalMemClass - usedMemClass)
 972                                 || minReferenceSize > (totalSysMem - totalUsage);
 973         }
 974 }
 975
 976 MemoryObject* MemoryHeap::allocateRandom (const DeviceInterface& vkd, VkDevice device, de::Random& rng)
 977 {
 978         pair<MemoryType, VkDeviceSize> memoryTypeMaxSizePair;
 979
 980         // Pick random memory type
 981         {
 982                 vector<pair<MemoryType, VkDeviceSize> > memoryTypes;
 983
 984                 const VkDeviceSize      availableInHeap         = m_heap.size/MAX_MEMORY_USAGE_DIV - m_usage;
 985                 const bool                      isUMA                           = m_limits.totalDeviceLocalMemory == 0;
 986                 const MemoryClass       memClass                        = getMemoryClass();
 987
 988                 // Collect memory types that can be allocated and the maximum size of allocation.
 989                 // Memory type can be only allocated if minimal memory allocation is less than available memory.
 990                 for (size_t memoryTypeNdx = 0; memoryTypeNdx < m_memoryTypes.size(); memoryTypeNdx++)
 991                 {
 992                         const MemoryType        type                                            = m_memoryTypes[memoryTypeNdx];
 993                         const VkDeviceSize      atomSize                                        = (type.type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0
 994                                                                                                                         ? 1
 995                                                                                                                         : m_nonCoherentAtomSize;
 996                         const VkDeviceSize      allocationSizeGranularity       = de::max(atomSize, memClass == MEMORY_CLASS_DEVICE ? m_limits.devicePageSize : getHostPageSize());
 997                         const VkDeviceSize      minAllocationSize                       = allocationSizeGranularity;
 998                         const VkDeviceSize      minReferenceSize                        = minAllocationSize
 999                                                                                                                         + divRoundUp<VkDeviceSize>(minAllocationSize,  8)
1000                                                                                                                         + divRoundUp<VkDeviceSize>(minAllocationSize,  atomSize * 8);
1001
1002                         if (isUMA)
1003                         {
1004                                 // Max memory size calculation is little tricky since reference memory requires 1/n bits per byte.
1005                                 const VkDeviceSize      totalUsage                              = m_totalMemTracker.getTotalUsage();
1006                                 const VkDeviceSize      totalSysMem                             = (VkDeviceSize)m_limits.totalSystemMemory;
1007                                 const VkDeviceSize      availableBits                   = (totalSysMem - totalUsage) * 8;
1008                                 // availableBits == maxAllocationSizeBits + maxAllocationReferenceSizeBits
1009                                 // maxAllocationReferenceSizeBits == maxAllocationSizeBits + (maxAllocationSizeBits / 8) + (maxAllocationSizeBits / atomSizeBits)
1010                                 // availableBits == maxAllocationSizeBits + maxAllocationSizeBits + (maxAllocationSizeBits / 8) + (maxAllocationSizeBits / atomSizeBits)
1011                                 // availableBits == 2 * maxAllocationSizeBits + (maxAllocationSizeBits / 8) + (maxAllocationSizeBits / atomSizeBits)
1012                                 // availableBits == (2 + 1/8 + 1/atomSizeBits) * maxAllocationSizeBits
1013                                 // 8 * availableBits == (16 + 1 + 8/atomSizeBits) * maxAllocationSizeBits
1014                                 // atomSizeBits * 8 * availableBits == (17 * atomSizeBits + 8) * maxAllocationSizeBits
1015                                 // maxAllocationSizeBits == atomSizeBits * 8 * availableBits / (17 * atomSizeBits + 8)
1016                                 // maxAllocationSizeBytes == maxAllocationSizeBits / 8
1017                                 // maxAllocationSizeBytes == atomSizeBits * availableBits / (17 * atomSizeBits + 8)
1018                                 // atomSizeBits = atomSize * 8
1019                                 // maxAllocationSizeBytes == atomSize * 8 * availableBits / (17 * atomSize * 8 + 8)
1020                                 // maxAllocationSizeBytes == atomSize * availableBits / (17 * atomSize + 1)
1021                                 const VkDeviceSize      maxAllocationSize               = roundDownToMultiple(((atomSize * availableBits) / (17 * atomSize + 1)), allocationSizeGranularity);
1022
1023                                 DE_ASSERT(totalUsage <= totalSysMem);
1024                                 DE_ASSERT(maxAllocationSize <= totalSysMem);
1025
1026                                 if (minAllocationSize + minReferenceSize <= (totalSysMem - totalUsage) && minAllocationSize <= availableInHeap)
1027                                 {
1028                                         DE_ASSERT(maxAllocationSize >= minAllocationSize);
1029                                         memoryTypes.push_back(std::make_pair(type, maxAllocationSize));
1030                                 }
1031                         }
1032                         else
1033                         {
1034                                 // Max memory size calculation is little tricky since reference memory requires 1/n bits per byte.
1035                                 const VkDeviceSize      totalUsage                      = m_totalMemTracker.getTotalUsage();
1036                                 const VkDeviceSize      totalSysMem                     = (VkDeviceSize)m_limits.totalSystemMemory;
1037
1038                                 const VkDeviceSize      totalMemClass           = memClass == MEMORY_CLASS_SYSTEM
1039                                                                                                                 ? m_limits.totalSystemMemory
1040                                                                                                                 : m_limits.totalDeviceLocalMemory;
1041                                 const VkDeviceSize      usedMemClass            = m_totalMemTracker.getUsage(memClass);
1042                                 // availableRefBits = maxRefBits + maxRefBits/8 + maxRefBits/atomSizeBits
1043                                 // availableRefBits = maxRefBits * (1 + 1/8 + 1/atomSizeBits)
1044                                 // 8 * availableRefBits = maxRefBits * (8 + 1 + 8/atomSizeBits)
1045                                 // 8 * atomSizeBits * availableRefBits = maxRefBits * (9 * atomSizeBits + 8)
1046                                 // maxRefBits = 8 * atomSizeBits * availableRefBits / (9 * atomSizeBits + 8)
1047                                 // atomSizeBits = atomSize * 8
1048                                 // maxRefBits = 8 * atomSize * 8 * availableRefBits / (9 * atomSize * 8 + 8)
1049                                 // maxRefBits = atomSize * 8 * availableRefBits / (9 * atomSize + 1)
1050                                 // maxRefBytes = atomSize * availableRefBits / (9 * atomSize + 1)
1051                                 const VkDeviceSize      maxAllocationSize       = roundDownToMultiple(de::min(totalMemClass - usedMemClass, (atomSize * 8 * (totalSysMem - totalUsage)) / (9 * atomSize + 1)), allocationSizeGranularity);
1052
1053                                 DE_ASSERT(usedMemClass <= totalMemClass);
1054
1055                                 if (minAllocationSize <= availableInHeap
1056                                                 && minAllocationSize <= (totalMemClass - usedMemClass)
1057                                                 && minReferenceSize <= (totalSysMem - totalUsage))
1058                                 {
1059                                         DE_ASSERT(maxAllocationSize >= minAllocationSize);
1060                                         memoryTypes.push_back(std::make_pair(type, maxAllocationSize));
1061                                 }
1062
1063                         }
1064                 }
1065
1066                 memoryTypeMaxSizePair = rng.choose<pair<MemoryType, VkDeviceSize> >(memoryTypes.begin(), memoryTypes.end());
1067         }
1068
1069         const MemoryType                type                                            = memoryTypeMaxSizePair.first;
1070         const VkDeviceSize              maxAllocationSize                       = memoryTypeMaxSizePair.second / MAX_MEMORY_ALLOC_DIV;
1071         const VkDeviceSize              atomSize                                        = (type.type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0
1072                                                                                                                 ? 1
1073                                                                                                                 : m_nonCoherentAtomSize;
1074         const VkDeviceSize              allocationSizeGranularity       = de::max(atomSize, getMemoryClass() == MEMORY_CLASS_DEVICE ? m_limits.devicePageSize : getHostPageSize());
1075         const VkDeviceSize              size                                            = randomSize(rng, atomSize, maxAllocationSize);
1076         const VkDeviceSize              memoryUsage                                     = roundUpToMultiple(size, allocationSizeGranularity);
1077         const VkDeviceSize              referenceMemoryUsage            = size + divRoundUp<VkDeviceSize>(size, 8) + divRoundUp<VkDeviceSize>(size / atomSize, 8);
1078
1079         DE_ASSERT(size <= maxAllocationSize);
1080
1081         MemoryObject* const             object  = new MemoryObject(vkd, device, size, type.index, atomSize, memoryUsage, referenceMemoryUsage);
1082
1083         m_usage += memoryUsage;
1084         m_totalMemTracker.allocate(getMemoryClass(), memoryUsage);
1085         m_totalMemTracker.allocate(MEMORY_CLASS_SYSTEM, referenceMemoryUsage);
1086         m_objects.push_back(object);
1087
1088         return object;
1089 }
1090
1091 size_t getMemoryObjectSystemSize (Context& context)
1092 {
1093         return computeDeviceMemorySystemMemFootprint(context.getDeviceInterface(), context.getDevice())
1094                    + sizeof(MemoryObject)
1095                    + sizeof(de::SharedPtr<MemoryObject>);
1096 }
1097
1098 size_t getMemoryMappingSystemSize (void)
1099 {
1100         return sizeof(MemoryMapping) + sizeof(de::SharedPtr<MemoryMapping>);
1101 }
1102
1103 class RandomMemoryMappingInstance : public TestInstance
1104 {
1105 public:
1106         RandomMemoryMappingInstance (Context& context, deUint32 seed)
1107                 : TestInstance                          (context)
1108                 , m_memoryObjectSysMemSize      (getMemoryObjectSystemSize(context))
1109                 , m_memoryMappingSysMemSize     (getMemoryMappingSystemSize())
1110                 , m_memoryLimits                        (getMemoryLimits(context.getTestContext().getPlatform().getVulkanPlatform()))
1111                 , m_rng                                         (seed)
1112                 , m_opNdx                                       (0)
1113         {
1114                 const VkPhysicalDevice                                  physicalDevice          = context.getPhysicalDevice();
1115                 const InstanceInterface&                                vki                                     = context.getInstanceInterface();
1116                 const VkPhysicalDeviceMemoryProperties  memoryProperties        = getPhysicalDeviceMemoryProperties(vki, physicalDevice);
1117                 // \todo [2016-05-26 misojarvi] Remove zero check once drivers report correctly 1 instead of 0
1118                 const VkDeviceSize                                              nonCoherentAtomSize     = context.getDeviceProperties().limits.nonCoherentAtomSize != 0
1119                                                                                                                                         ? context.getDeviceProperties().limits.nonCoherentAtomSize
1120                                                                                                                                         : 1;
1121
1122                 // Initialize heaps
1123                 {
1124                         vector<vector<MemoryType> >     memoryTypes     (memoryProperties.memoryHeapCount);
1125
1126                         for (deUint32 memoryTypeNdx = 0; memoryTypeNdx < memoryProperties.memoryTypeCount; memoryTypeNdx++)
1127                         {
1128                                 if (memoryProperties.memoryTypes[memoryTypeNdx].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
1129                                         memoryTypes[memoryProperties.memoryTypes[memoryTypeNdx].heapIndex].push_back(MemoryType(memoryTypeNdx, memoryProperties.memoryTypes[memoryTypeNdx]));
1130                         }
1131
1132                         for (deUint32 heapIndex = 0; heapIndex < memoryProperties.memoryHeapCount; heapIndex++)
1133                         {
1134                                 const VkMemoryHeap      heapInfo        = memoryProperties.memoryHeaps[heapIndex];
1135
1136                                 if (!memoryTypes[heapIndex].empty())
1137                                 {
1138                                         const de::SharedPtr<MemoryHeap> heap    (new MemoryHeap(heapInfo, memoryTypes[heapIndex], m_memoryLimits, nonCoherentAtomSize, m_totalMemTracker));
1139
1140                                         TCU_CHECK_INTERNAL(!heap->full());
1141
1142                                         m_memoryHeaps.push_back(heap);
1143                                 }
1144                         }
1145                 }
1146         }
1147
1148         ~RandomMemoryMappingInstance (void)
1149         {
1150         }
1151
1152         tcu::TestStatus iterate (void)
1153         {
1154                 const size_t                    opCount                                         = 100;
1155                 const float                             memoryOpProbability                     = 0.5f;         // 0.50
1156                 const float                             flushInvalidateProbability      = 0.4f;         // 0.20
1157                 const float                             mapProbability                          = 0.50f;        // 0.15
1158                 const float                             unmapProbability                        = 0.25f;        // 0.075
1159
1160                 const float                             allocProbability                        = 0.75f; // Versun free
1161
1162                 const VkDevice                  device                                          = m_context.getDevice();
1163                 const DeviceInterface&  vkd                                                     = m_context.getDeviceInterface();
1164
1165                 const VkDeviceSize              sysMemUsage                                     = (m_memoryLimits.totalDeviceLocalMemory == 0)
1166                                                                                                                         ? m_totalMemTracker.getTotalUsage()
1167                                                                                                                         : m_totalMemTracker.getUsage(MEMORY_CLASS_SYSTEM);
1168
1169                 if (!m_memoryMappings.empty() && m_rng.getFloat() < memoryOpProbability)
1170                 {
1171                         // Perform operations on mapped memory
1172                         MemoryMapping* const    mapping = m_rng.choose<MemoryMapping*>(m_memoryMappings.begin(), m_memoryMappings.end());
1173
1174                         enum Op
1175                         {
1176                                 OP_READ = 0,
1177                                 OP_WRITE,
1178                                 OP_MODIFY,
1179                                 OP_LAST
1180                         };
1181
1182                         const Op op = (Op)(m_rng.getUint32() % OP_LAST);
1183
1184                         switch (op)
1185                         {
1186                                 case OP_READ:
1187                                         mapping->randomRead(m_rng);
1188                                         break;
1189
1190                                 case OP_WRITE:
1191                                         mapping->randomWrite(m_rng);
1192                                         break;
1193
1194                                 case OP_MODIFY:
1195                                         mapping->randomModify(m_rng);
1196                                         break;
1197
1198                                 default:
1199                                         DE_FATAL("Invalid operation");
1200                         }
1201                 }
1202                 else if (!m_mappedMemoryObjects.empty() && m_rng.getFloat() < flushInvalidateProbability)
1203                 {
1204                         MemoryObject* const     object  = m_rng.choose<MemoryObject*>(m_mappedMemoryObjects.begin(), m_mappedMemoryObjects.end());
1205
1206                         if (m_rng.getBool())
1207                                 object->randomFlush(vkd, device, m_rng);
1208                         else
1209                                 object->randomInvalidate(vkd, device, m_rng);
1210                 }
1211                 else if (!m_mappedMemoryObjects.empty() && m_rng.getFloat() < unmapProbability)
1212                 {
1213                         // Unmap memory object
1214                         MemoryObject* const     object  = m_rng.choose<MemoryObject*>(m_mappedMemoryObjects.begin(), m_mappedMemoryObjects.end());
1215
1216                         // Remove mapping
1217                         removeFirstEqual(m_memoryMappings, object->getMapping());
1218
1219                         object->unmap();
1220                         removeFirstEqual(m_mappedMemoryObjects, object);
1221                         m_nonMappedMemoryObjects.push_back(object);
1222
1223                         m_totalMemTracker.free(MEMORY_CLASS_SYSTEM, (VkDeviceSize)m_memoryMappingSysMemSize);
1224                 }
1225                 else if (!m_nonMappedMemoryObjects.empty() &&
1226                                  (m_rng.getFloat() < mapProbability) &&
1227                                  (sysMemUsage+m_memoryMappingSysMemSize <= (VkDeviceSize)m_memoryLimits.totalSystemMemory))
1228                 {
1229                         // Map memory object
1230                         MemoryObject* const             object  = m_rng.choose<MemoryObject*>(m_nonMappedMemoryObjects.begin(), m_nonMappedMemoryObjects.end());
1231                         MemoryMapping*                  mapping = object->mapRandom(vkd, device, m_rng);
1232
1233                         m_memoryMappings.push_back(mapping);
1234                         m_mappedMemoryObjects.push_back(object);
1235                         removeFirstEqual(m_nonMappedMemoryObjects, object);
1236
1237                         m_totalMemTracker.allocate(MEMORY_CLASS_SYSTEM, (VkDeviceSize)m_memoryMappingSysMemSize);
1238                 }
1239                 else
1240                 {
1241                         // Sort heaps based on capacity (full or not)
1242                         vector<MemoryHeap*>             nonFullHeaps;
1243                         vector<MemoryHeap*>             nonEmptyHeaps;
1244
1245                         if (sysMemUsage+m_memoryObjectSysMemSize <= (VkDeviceSize)m_memoryLimits.totalSystemMemory)
1246                         {
1247                                 // For the duration of sorting reserve MemoryObject space from system memory
1248                                 m_totalMemTracker.allocate(MEMORY_CLASS_SYSTEM, (VkDeviceSize)m_memoryObjectSysMemSize);
1249
1250                                 for (vector<de::SharedPtr<MemoryHeap> >::const_iterator heapIter = m_memoryHeaps.begin();
1251                                          heapIter != m_memoryHeaps.end();
1252                                          ++heapIter)
1253                                 {
1254                                         if (!(*heapIter)->full())
1255                                                 nonFullHeaps.push_back(heapIter->get());
1256
1257                                         if (!(*heapIter)->empty())
1258                                                 nonEmptyHeaps.push_back(heapIter->get());
1259                                 }
1260
1261                                 m_totalMemTracker.free(MEMORY_CLASS_SYSTEM, (VkDeviceSize)m_memoryObjectSysMemSize);
1262                         }
1263                         else
1264                         {
1265                                 // Not possible to even allocate MemoryObject from system memory, look for non-empty heaps
1266                                 for (vector<de::SharedPtr<MemoryHeap> >::const_iterator heapIter = m_memoryHeaps.begin();
1267                                          heapIter != m_memoryHeaps.end();
1268                                          ++heapIter)
1269                                 {
1270                                         if (!(*heapIter)->empty())
1271                                                 nonEmptyHeaps.push_back(heapIter->get());
1272                                 }
1273                         }
1274
1275                         if (!nonFullHeaps.empty() && (nonEmptyHeaps.empty() || m_rng.getFloat() < allocProbability))
1276                         {
1277                                 // Reserve MemoryObject from sys mem first
1278                                 m_totalMemTracker.allocate(MEMORY_CLASS_SYSTEM, (VkDeviceSize)m_memoryObjectSysMemSize);
1279
1280                                 // Allocate more memory objects
1281                                 MemoryHeap* const       heap    = m_rng.choose<MemoryHeap*>(nonFullHeaps.begin(), nonFullHeaps.end());
1282                                 MemoryObject* const     object  = heap->allocateRandom(vkd, device, m_rng);
1283
1284                                 m_nonMappedMemoryObjects.push_back(object);
1285                         }
1286                         else
1287                         {
1288                                 // Free memory objects
1289                                 MemoryHeap* const               heap    = m_rng.choose<MemoryHeap*>(nonEmptyHeaps.begin(), nonEmptyHeaps.end());
1290                                 MemoryObject* const             object  = heap->getRandomObject(m_rng);
1291
1292                                 // Remove mapping
1293                                 if (object->getMapping())
1294                                 {
1295                                         removeFirstEqual(m_memoryMappings, object->getMapping());
1296                                         m_totalMemTracker.free(MEMORY_CLASS_SYSTEM, m_memoryMappingSysMemSize);
1297                                 }
1298
1299                                 removeFirstEqual(m_mappedMemoryObjects, object);
1300                                 removeFirstEqual(m_nonMappedMemoryObjects, object);
1301
1302                                 heap->free(object);
1303                                 m_totalMemTracker.free(MEMORY_CLASS_SYSTEM, (VkDeviceSize)m_memoryObjectSysMemSize);
1304                         }
1305                 }
1306
1307                 m_opNdx += 1;
1308                 if (m_opNdx == opCount)
1309                         return tcu::TestStatus::pass("Pass");
1310                 else
1311                         return tcu::TestStatus::incomplete();
1312         }
1313
1314 private:
1315         const size_t                                            m_memoryObjectSysMemSize;
1316         const size_t                                            m_memoryMappingSysMemSize;
1317         const PlatformMemoryLimits                      m_memoryLimits;
1318
1319         de::Random                                                      m_rng;
1320         size_t                                                          m_opNdx;
1321
1322         TotalMemoryTracker                                      m_totalMemTracker;
1323         vector<de::SharedPtr<MemoryHeap> >      m_memoryHeaps;
1324
1325         vector<MemoryObject*>                           m_mappedMemoryObjects;
1326         vector<MemoryObject*>                           m_nonMappedMemoryObjects;
1327         vector<MemoryMapping*>                          m_memoryMappings;
1328 };
1329
1330 enum Op
1331 {
1332         OP_NONE = 0,
1333
1334         OP_FLUSH,
1335         OP_SUB_FLUSH,
1336         OP_SUB_FLUSH_SEPARATE,
1337         OP_SUB_FLUSH_OVERLAPPING,
1338
1339         OP_INVALIDATE,
1340         OP_SUB_INVALIDATE,
1341         OP_SUB_INVALIDATE_SEPARATE,
1342         OP_SUB_INVALIDATE_OVERLAPPING,
1343
1344         OP_REMAP,
1345
1346         OP_LAST
1347 };
1348
1349 TestConfig subMappedConfig (VkDeviceSize                                allocationSize,
1350                                                         const MemoryRange&                      mapping,
1351                                                         Op                                                      op,
1352                                                         deUint32                                        seed)
1353 {
1354         TestConfig config;
1355
1356         config.allocationSize   = allocationSize;
1357         config.seed                             = seed;
1358         config.mapping                  = mapping;
1359         config.remap                    = false;
1360
1361         switch (op)
1362         {
1363                 case OP_NONE:
1364                         return config;
1365
1366                 case OP_REMAP:
1367                         config.remap = true;
1368                         return config;
1369
1370                 case OP_FLUSH:
1371                         config.flushMappings = vector<MemoryRange>(1, MemoryRange(mapping.offset, mapping.size));
1372                         return config;
1373
1374                 case OP_SUB_FLUSH:
1375                         DE_ASSERT(mapping.size / 4 > 0);
1376
1377                         config.flushMappings = vector<MemoryRange>(1, MemoryRange(mapping.offset + mapping.size / 4, mapping.size / 2));
1378                         return config;
1379
1380                 case OP_SUB_FLUSH_SEPARATE:
1381                         DE_ASSERT(mapping.size / 2 > 0);
1382
1383                         config.flushMappings.push_back(MemoryRange(mapping.offset + mapping.size /  2, mapping.size - (mapping.size / 2)));
1384                         config.flushMappings.push_back(MemoryRange(mapping.offset, mapping.size / 2));
1385
1386                         return config;
1387
1388                 case OP_SUB_FLUSH_OVERLAPPING:
1389                         DE_ASSERT((mapping.size / 3) > 0);
1390
1391                         config.flushMappings.push_back(MemoryRange(mapping.offset + mapping.size /  3, mapping.size - (mapping.size / 2)));
1392                         config.flushMappings.push_back(MemoryRange(mapping.offset, (2 * mapping.size) / 3));
1393
1394                         return config;
1395
1396                 case OP_INVALIDATE:
1397                         config.flushMappings = vector<MemoryRange>(1, MemoryRange(mapping.offset, mapping.size));
1398                         config.invalidateMappings = vector<MemoryRange>(1, MemoryRange(mapping.offset, mapping.size));
1399                         return config;
1400
1401                 case OP_SUB_INVALIDATE:
1402                         DE_ASSERT(mapping.size / 4 > 0);
1403
1404                         config.flushMappings = vector<MemoryRange>(1, MemoryRange(mapping.offset + mapping.size / 4, mapping.size / 2));
1405                         config.invalidateMappings = vector<MemoryRange>(1, MemoryRange(mapping.offset + mapping.size / 4, mapping.size / 2));
1406                         return config;
1407
1408                 case OP_SUB_INVALIDATE_SEPARATE:
1409                         DE_ASSERT(mapping.size / 2 > 0);
1410
1411                         config.flushMappings.push_back(MemoryRange(mapping.offset + mapping.size /  2, mapping.size - (mapping.size / 2)));
1412                         config.flushMappings.push_back(MemoryRange(mapping.offset, mapping.size / 2));
1413
1414                         config.invalidateMappings.push_back(MemoryRange(mapping.offset + mapping.size /  2, mapping.size - (mapping.size / 2)));
1415                         config.invalidateMappings.push_back(MemoryRange(mapping.offset, mapping.size / 2));
1416
1417                         return config;
1418
1419                 case OP_SUB_INVALIDATE_OVERLAPPING:
1420                         DE_ASSERT((mapping.size / 3) > 0);
1421
1422                         config.flushMappings.push_back(MemoryRange(mapping.offset + mapping.size /  3, mapping.size - (mapping.size / 2)));
1423                         config.flushMappings.push_back(MemoryRange(mapping.offset, (2 * mapping.size) / 3));
1424
1425                         config.invalidateMappings.push_back(MemoryRange(mapping.offset + mapping.size /  3, mapping.size - (mapping.size / 2)));
1426                         config.invalidateMappings.push_back(MemoryRange(mapping.offset, (2 * mapping.size) / 3));
1427
1428                         return config;
1429
1430                 default:
1431                         DE_FATAL("Unknown Op");
1432                         return TestConfig();
1433         }
1434 }
1435
1436 TestConfig fullMappedConfig (VkDeviceSize       allocationSize,
1437                                                          Op                             op,
1438                                                          deUint32               seed)
1439 {
1440         return subMappedConfig(allocationSize, MemoryRange(0, allocationSize), op, seed);
1441 }
1442
1443 } // anonymous
1444
1445 tcu::TestCaseGroup* createMappingTests (tcu::TestContext& testCtx)
1446 {
1447         de::MovePtr<tcu::TestCaseGroup> group (new tcu::TestCaseGroup(testCtx, "mapping", "Memory mapping tests."));
1448
1449         const VkDeviceSize allocationSizes[] =
1450         {
1451                 33, 257, 4087, 8095, 1*1024*1024 + 1
1452         };
1453
1454         const VkDeviceSize offsets[] =
1455         {
1456                 0, 17, 129, 255, 1025, 32*1024+1
1457         };
1458
1459         const VkDeviceSize sizes[] =
1460         {
1461                 31, 255, 1025, 4085, 1*1024*1024 - 1
1462         };
1463
1464         const struct
1465         {
1466                 const Op                        op;
1467                 const char* const       name;
1468         } ops[] =
1469         {
1470                 { OP_NONE,                                              "simple"                                        },
1471                 { OP_REMAP,                                             "remap"                                         },
1472                 { OP_FLUSH,                                             "flush"                                         },
1473                 { OP_SUB_FLUSH,                                 "subflush"                                      },
1474                 { OP_SUB_FLUSH_SEPARATE,                "subflush_separate"                     },
1475                 { OP_SUB_FLUSH_SEPARATE,                "subflush_overlapping"          },
1476
1477                 { OP_INVALIDATE,                                "invalidate"                            },
1478                 { OP_SUB_INVALIDATE,                    "subinvalidate"                         },
1479                 { OP_SUB_INVALIDATE_SEPARATE,   "subinvalidate_separate"        },
1480                 { OP_SUB_INVALIDATE_SEPARATE,   "subinvalidate_overlapping"     }
1481         };
1482
1483         // .full
1484         {
1485                 de::MovePtr<tcu::TestCaseGroup> fullGroup (new tcu::TestCaseGroup(testCtx, "full", "Map memory completely."));
1486
1487                 for (size_t allocationSizeNdx = 0; allocationSizeNdx < DE_LENGTH_OF_ARRAY(allocationSizes); allocationSizeNdx++)
1488                 {
1489                         const VkDeviceSize                              allocationSize          = allocationSizes[allocationSizeNdx];
1490                         de::MovePtr<tcu::TestCaseGroup> allocationSizeGroup     (new tcu::TestCaseGroup(testCtx, de::toString(allocationSize).c_str(), ""));
1491
1492                         for (size_t opNdx = 0; opNdx < DE_LENGTH_OF_ARRAY(ops); opNdx++)
1493                         {
1494                                 const Op                        op              = ops[opNdx].op;
1495                                 const char* const       name    = ops[opNdx].name;
1496                                 const deUint32          seed    = (deUint32)(opNdx * allocationSizeNdx);
1497                                 const TestConfig        config  = fullMappedConfig(allocationSize, op, seed);
1498
1499                                 addFunctionCase(allocationSizeGroup.get(), name, name, testMemoryMapping, config);
1500                         }
1501
1502                         fullGroup->addChild(allocationSizeGroup.release());
1503                 }
1504
1505                 group->addChild(fullGroup.release());
1506         }
1507
1508         // .sub
1509         {
1510                 de::MovePtr<tcu::TestCaseGroup> subGroup (new tcu::TestCaseGroup(testCtx, "sub", "Map part of the memory."));
1511
1512                 for (size_t allocationSizeNdx = 0; allocationSizeNdx < DE_LENGTH_OF_ARRAY(allocationSizes); allocationSizeNdx++)
1513                 {
1514                         const VkDeviceSize                              allocationSize          = allocationSizes[allocationSizeNdx];
1515                         de::MovePtr<tcu::TestCaseGroup> allocationSizeGroup     (new tcu::TestCaseGroup(testCtx, de::toString(allocationSize).c_str(), ""));
1516
1517                         for (size_t offsetNdx = 0; offsetNdx < DE_LENGTH_OF_ARRAY(offsets); offsetNdx++)
1518                         {
1519                                 const VkDeviceSize                              offset                  = offsets[offsetNdx];
1520
1521                                 if (offset >= allocationSize)
1522                                         continue;
1523
1524                                 de::MovePtr<tcu::TestCaseGroup> offsetGroup             (new tcu::TestCaseGroup(testCtx, ("offset_" + de::toString(offset)).c_str(), ""));
1525
1526                                 for (size_t sizeNdx = 0; sizeNdx < DE_LENGTH_OF_ARRAY(sizes); sizeNdx++)
1527                                 {
1528                                         const VkDeviceSize                              size            = sizes[sizeNdx];
1529
1530                                         if (offset + size > allocationSize)
1531                                                 continue;
1532
1533                                         if (offset == 0 && size == allocationSize)
1534                                                 continue;
1535
1536                                         de::MovePtr<tcu::TestCaseGroup> sizeGroup       (new tcu::TestCaseGroup(testCtx, ("size_" + de::toString(size)).c_str(), ""));
1537
1538                                         for (size_t opNdx = 0; opNdx < DE_LENGTH_OF_ARRAY(ops); opNdx++)
1539                                         {
1540                                                 const deUint32          seed    = (deUint32)(opNdx * allocationSizeNdx);
1541                                                 const Op                        op              = ops[opNdx].op;
1542                                                 const char* const       name    = ops[opNdx].name;
1543                                                 const TestConfig        config  = subMappedConfig(allocationSize, MemoryRange(offset, size), op, seed);
1544
1545                                                 addFunctionCase(sizeGroup.get(), name, name, testMemoryMapping, config);
1546                                         }
1547
1548                                         offsetGroup->addChild(sizeGroup.release());
1549                                 }
1550
1551                                 allocationSizeGroup->addChild(offsetGroup.release());
1552                         }
1553
1554                         subGroup->addChild(allocationSizeGroup.release());
1555                 }
1556
1557                 group->addChild(subGroup.release());
1558         }
1559
1560         // .random
1561         {
1562                 de::MovePtr<tcu::TestCaseGroup> randomGroup     (new tcu::TestCaseGroup(testCtx, "random", "Random memory mapping tests."));
1563                 de::Random                                              rng                     (3927960301u);
1564
1565                 for (size_t ndx = 0; ndx < 100; ndx++)
1566                 {
1567                         const deUint32          seed    = rng.getUint32();
1568                         const std::string       name    = de::toString(ndx);
1569
1570                         randomGroup->addChild(new InstanceFactory1<RandomMemoryMappingInstance, deUint32>(testCtx, tcu::NODETYPE_SELF_VALIDATE, de::toString(ndx), "Random case", seed));
1571                 }
1572
1573                 group->addChild(randomGroup.release());
1574         }
1575
1576         return group.release();
1577 }
1578
1579 } // memory
1580 } // vkt