external/vulkancts/framework/vulkan/vkRayTracingUtil.cpp

   1 /*-------------------------------------------------------------------------
   2  * Vulkan CTS Framework
   3  * --------------------
   4  *
   5  * Copyright (c) 2020 The Khronos Group Inc.
   6  *
   7  * Licensed under the Apache License, Version 2.0 (the "License");
   8  * you may not use this file except in compliance with the License.
   9  * You may obtain a copy of the License at
  10  *
  11  *      http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  *
  19  *//*!
  20  * \file
  21  * \brief Utilities for creating commonly used Vulkan objects
  22  *//*--------------------------------------------------------------------*/
  23
  24 #include "vkRayTracingUtil.hpp"
  25
  26 #include "vkRefUtil.hpp"
  27 #include "vkQueryUtil.hpp"
  28 #include "vkObjUtil.hpp"
  29 #include "vkBarrierUtil.hpp"
  30
  31 #include "deStringUtil.hpp"
  32
  33 #include <vector>
  34 #include <string>
  35 #include <thread>
  36
  37 namespace vk
  38 {
  39
  40 struct DeferredThreadParams
  41 {
  42         const DeviceInterface&  vk;
  43         VkDevice                                device;
  44         VkDeferredOperationKHR  deferredOperation;
  45         VkResult                                result;
  46 };
  47
  48 std::string getFormatSimpleName (vk::VkFormat format)
  49 {
  50         constexpr size_t kPrefixLen = 10; // strlen("VK_FORMAT_")
  51         return de::toLower(de::toString(format).substr(kPrefixLen));
  52 }
  53
  54 // Returns true if VK_FORMAT_FEATURE_ACCELERATION_STRUCTURE_VERTEX_BUFFER_BIT_KHR needs to be supported for the given format.
  55 static bool isMandatoryAccelerationStructureVertexBufferFormat (vk::VkFormat format)
  56 {
  57         bool mandatory = false;
  58
  59         switch (format)
  60         {
  61     case VK_FORMAT_R32G32_SFLOAT:
  62     case VK_FORMAT_R32G32B32_SFLOAT:
  63     case VK_FORMAT_R16G16_SFLOAT:
  64     case VK_FORMAT_R16G16B16A16_SFLOAT:
  65     case VK_FORMAT_R16G16_SNORM:
  66     case VK_FORMAT_R16G16B16A16_SNORM:
  67                 mandatory = true;
  68                 break;
  69         default:
  70                 break;
  71         }
  72
  73         return mandatory;
  74 }
  75
  76 void checkAccelerationStructureVertexBufferFormat (const vk::InstanceInterface &vki, vk::VkPhysicalDevice physicalDevice, vk::VkFormat format)
  77 {
  78         const vk::VkFormatProperties formatProperties = getPhysicalDeviceFormatProperties(vki, physicalDevice, format);
  79
  80         if ((formatProperties.bufferFeatures & vk::VK_FORMAT_FEATURE_ACCELERATION_STRUCTURE_VERTEX_BUFFER_BIT_KHR) == 0u)
  81         {
  82                 const std::string errorMsg = "Format not supported for acceleration structure vertex buffers";
  83                 if (isMandatoryAccelerationStructureVertexBufferFormat(format))
  84                         TCU_FAIL(errorMsg);
  85                 TCU_THROW(NotSupportedError, errorMsg);
  86         }
  87 }
  88
  89 std::string getCommonRayGenerationShader (void)
  90 {
  91         return
  92                 "#version 460 core\n"
  93                 "#extension GL_EXT_ray_tracing : require\n"
  94                 "layout(location = 0) rayPayloadEXT vec3 hitValue;\n"
  95                 "layout(set = 0, binding = 1) uniform accelerationStructureEXT topLevelAS;\n"
  96                 "\n"
  97                 "void main()\n"
  98                 "{\n"
  99                 "  uint  rayFlags = 0;\n"
 100                 "  uint  cullMask = 0xFF;\n"
 101                 "  float tmin     = 0.0;\n"
 102                 "  float tmax     = 9.0;\n"
 103                 "  vec3  origin   = vec3((float(gl_LaunchIDEXT.x) + 0.5f) / float(gl_LaunchSizeEXT.x), (float(gl_LaunchIDEXT.y) + 0.5f) / float(gl_LaunchSizeEXT.y), 0.0);\n"
 104                 "  vec3  direct   = vec3(0.0, 0.0, -1.0);\n"
 105                 "  traceRayEXT(topLevelAS, rayFlags, cullMask, 0, 0, 0, origin, tmin, direct, tmax, 0);\n"
 106                 "}\n";
 107 }
 108
 109 RaytracedGeometryBase::RaytracedGeometryBase (VkGeometryTypeKHR geometryType, VkFormat vertexFormat, VkIndexType indexType)
 110         : m_geometryType        (geometryType)
 111         , m_vertexFormat        (vertexFormat)
 112         , m_indexType           (indexType)
 113         , m_geometryFlags       ((VkGeometryFlagsKHR)0u)
 114 {
 115         if (m_geometryType == VK_GEOMETRY_TYPE_AABBS_KHR)
 116                 DE_ASSERT(m_vertexFormat == VK_FORMAT_R32G32B32_SFLOAT);
 117 }
 118
 119 RaytracedGeometryBase::~RaytracedGeometryBase ()
 120 {
 121 }
 122
 123 struct GeometryBuilderParams
 124 {
 125         VkGeometryTypeKHR       geometryType;
 126         bool                            usePadding;
 127 };
 128
 129 template <typename V, typename I>
 130 RaytracedGeometryBase* buildRaytracedGeometry (const GeometryBuilderParams& params)
 131 {
 132         return new RaytracedGeometry<V, I>(params.geometryType, (params.usePadding ? 1u : 0u));
 133 }
 134
 135 de::SharedPtr<RaytracedGeometryBase> makeRaytracedGeometry (VkGeometryTypeKHR geometryType, VkFormat vertexFormat, VkIndexType indexType, bool padVertices)
 136 {
 137         const GeometryBuilderParams builderParams { geometryType, padVertices };
 138
 139         switch (vertexFormat)
 140         {
 141                 case VK_FORMAT_R32G32_SFLOAT:
 142                         switch (indexType)
 143                         {
 144                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::Vec2, deUint16>(builderParams));
 145                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::Vec2, deUint32>(builderParams));
 146                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::Vec2, EmptyIndex>(builderParams));
 147                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 148                         };
 149                 case VK_FORMAT_R32G32B32_SFLOAT:
 150                         switch (indexType)
 151                         {
 152                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::Vec3, deUint16>(builderParams));
 153                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::Vec3, deUint32>(builderParams));
 154                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::Vec3, EmptyIndex>(builderParams));
 155                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 156                         };
 157                 case VK_FORMAT_R32G32B32A32_SFLOAT:
 158                         switch (indexType)
 159                         {
 160                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::Vec4, deUint16>(builderParams));
 161                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::Vec4, deUint32>(builderParams));
 162                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::Vec4, EmptyIndex>(builderParams));
 163                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 164                         };
 165                 case VK_FORMAT_R16G16_SFLOAT:
 166                         switch (indexType)
 167                         {
 168                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec2_16, deUint16>(builderParams));
 169                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec2_16, deUint32>(builderParams));
 170                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec2_16, EmptyIndex>(builderParams));
 171                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 172                         };
 173                 case VK_FORMAT_R16G16B16_SFLOAT:
 174                         switch (indexType)
 175                         {
 176                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec3_16, deUint16>(builderParams));
 177                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec3_16, deUint32>(builderParams));
 178                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec3_16, EmptyIndex>(builderParams));
 179                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 180                         };
 181                 case VK_FORMAT_R16G16B16A16_SFLOAT:
 182                         switch (indexType)
 183                         {
 184                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec4_16, deUint16>(builderParams));
 185                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec4_16, deUint32>(builderParams));
 186                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec4_16, EmptyIndex>(builderParams));
 187                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 188                         };
 189                 case VK_FORMAT_R16G16_SNORM:
 190                         switch (indexType)
 191                         {
 192                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec2_16SNorm, deUint16>(builderParams));
 193                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec2_16SNorm, deUint32>(builderParams));
 194                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec2_16SNorm, EmptyIndex>(builderParams));
 195                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 196                         };
 197                 case VK_FORMAT_R16G16B16_SNORM:
 198                         switch (indexType)
 199                         {
 200                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec3_16SNorm, deUint16>(builderParams));
 201                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec3_16SNorm, deUint32>(builderParams));
 202                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec3_16SNorm, EmptyIndex>(builderParams));
 203                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 204                         };
 205                 case VK_FORMAT_R16G16B16A16_SNORM:
 206                         switch (indexType)
 207                         {
 208                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec4_16SNorm, deUint16>(builderParams));
 209                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec4_16SNorm, deUint32>(builderParams));
 210                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec4_16SNorm, EmptyIndex>(builderParams));
 211                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 212                         };
 213                 case VK_FORMAT_R64G64_SFLOAT:
 214                         switch (indexType)
 215                         {
 216                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::DVec2, deUint16>(builderParams));
 217                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::DVec2, deUint32>(builderParams));
 218                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::DVec2, EmptyIndex>(builderParams));
 219                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 220                         };
 221                 case VK_FORMAT_R64G64B64_SFLOAT:
 222                         switch (indexType)
 223                         {
 224                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::DVec3, deUint16>(builderParams));
 225                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::DVec3, deUint32>(builderParams));
 226                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::DVec3, EmptyIndex>(builderParams));
 227                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 228                         };
 229                 case VK_FORMAT_R64G64B64A64_SFLOAT:
 230                         switch (indexType)
 231                         {
 232                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::DVec4, deUint16>(builderParams));
 233                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::DVec4, deUint32>(builderParams));
 234                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<tcu::DVec4, EmptyIndex>(builderParams));
 235                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 236                         };
 237                 case VK_FORMAT_R8G8_SNORM:
 238                         switch (indexType)
 239                         {
 240                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec2_8SNorm, deUint16>(builderParams));
 241                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec2_8SNorm, deUint32>(builderParams));
 242                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec2_8SNorm, EmptyIndex>(builderParams));
 243                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 244                         };
 245                 case VK_FORMAT_R8G8B8_SNORM:
 246                         switch (indexType)
 247                         {
 248                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec3_8SNorm, deUint16>(builderParams));
 249                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec3_8SNorm, deUint32>(builderParams));
 250                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec3_8SNorm, EmptyIndex>(builderParams));
 251                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 252                         };
 253                 case VK_FORMAT_R8G8B8A8_SNORM:
 254                         switch (indexType)
 255                         {
 256                                 case VK_INDEX_TYPE_UINT16:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec4_8SNorm, deUint16>(builderParams));
 257                                 case VK_INDEX_TYPE_UINT32:              return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec4_8SNorm, deUint32>(builderParams));
 258                                 case VK_INDEX_TYPE_NONE_KHR:    return de::SharedPtr<RaytracedGeometryBase>(buildRaytracedGeometry<Vec4_8SNorm, EmptyIndex>(builderParams));
 259                                 default:                                                TCU_THROW(InternalError, "Wrong index type");
 260                         };
 261                 default:
 262                         TCU_THROW(InternalError, "Wrong vertex format");
 263         };
 264
 265 }
 266
 267 VkDeviceAddress getBufferDeviceAddress ( const DeviceInterface& vk,
 268                                                                                  const VkDevice                 device,
 269                                                                                  const VkBuffer                 buffer,
 270                                                                                  VkDeviceSize                   offset )
 271 {
 272
 273         if (buffer == DE_NULL)
 274                 return 0;
 275
 276         VkBufferDeviceAddressInfo deviceAddressInfo
 277         {
 278                 VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO,           // VkStructureType    sType
 279                 DE_NULL,                                                                                        // const void*        pNext
 280                 buffer                                                                                          // VkBuffer           buffer;
 281         };
 282         return vk.getBufferDeviceAddress(device, &deviceAddressInfo) + offset;
 283 }
 284
 285
 286 static inline VkDeviceOrHostAddressConstKHR makeDeviceOrHostAddressConstKHR (const void* hostAddress)
 287 {
 288         // VS2015: Cannot create as a const due to cannot assign hostAddress due to it is a second field. Only assigning of first field supported.
 289         VkDeviceOrHostAddressConstKHR result;
 290
 291         deMemset(&result, 0, sizeof(result));
 292
 293         result.hostAddress = hostAddress;
 294
 295         return result;
 296 }
 297
 298 static inline VkDeviceOrHostAddressKHR makeDeviceOrHostAddressKHR (void* hostAddress)
 299 {
 300         // VS2015: Cannot create as a const due to cannot assign hostAddress due to it is a second field. Only assigning of first field supported.
 301         VkDeviceOrHostAddressKHR result;
 302
 303         deMemset(&result, 0, sizeof(result));
 304
 305         result.hostAddress = hostAddress;
 306
 307         return result;
 308 }
 309
 310 static inline VkDeviceOrHostAddressConstKHR makeDeviceOrHostAddressConstKHR (const DeviceInterface&     vk,
 311                                                                                                                                           const VkDevice                        device,
 312                                                                                                                                           VkBuffer                                      buffer,
 313                                                                                                                                           VkDeviceSize                          offset)
 314 {
 315         // VS2015: Cannot create as a const due to cannot assign hostAddress due to it is a second field. Only assigning of first field supported.
 316         VkDeviceOrHostAddressConstKHR result;
 317
 318         deMemset(&result, 0, sizeof(result));
 319
 320         VkBufferDeviceAddressInfo bufferDeviceAddressInfo =
 321         {
 322                 VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO_KHR,       // VkStructureType       sType;
 323                 DE_NULL,                                                                                        // const void*           pNext;
 324                 buffer,                                                                                         // VkBuffer                     buffer
 325         };
 326         result.deviceAddress = vk.getBufferDeviceAddress(device, &bufferDeviceAddressInfo) + offset;
 327
 328         return result;
 329 }
 330
 331 static inline VkDeviceOrHostAddressKHR makeDeviceOrHostAddressKHR (const DeviceInterface&       vk,
 332                                                                                                                                    const VkDevice                       device,
 333                                                                                                                                    VkBuffer                                     buffer,
 334                                                                                                                                    VkDeviceSize                         offset)
 335 {
 336         // VS2015: Cannot create as a const due to cannot assign hostAddress due to it is a second field. Only assigning of first field supported.
 337         VkDeviceOrHostAddressKHR result;
 338
 339         deMemset(&result, 0, sizeof(result));
 340
 341         VkBufferDeviceAddressInfo bufferDeviceAddressInfo =
 342         {
 343                 VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO_KHR,       // VkStructureType       sType;
 344                 DE_NULL,                                                                                        // const void*           pNext;
 345                 buffer,                                                                                         // VkBuffer                     buffer
 346         };
 347         result.deviceAddress = vk.getBufferDeviceAddress(device, &bufferDeviceAddressInfo) + offset;
 348
 349         return result;
 350 }
 351
 352 static inline VkAccelerationStructureGeometryDataKHR makeVkAccelerationStructureGeometryDataKHR (const VkAccelerationStructureGeometryTrianglesDataKHR& triangles)
 353 {
 354         VkAccelerationStructureGeometryDataKHR result;
 355
 356         deMemset(&result, 0, sizeof(result));
 357
 358         result.triangles = triangles;
 359
 360         return result;
 361 }
 362
 363 static inline VkAccelerationStructureGeometryDataKHR makeVkAccelerationStructureGeometryDataKHR (const VkAccelerationStructureGeometryAabbsDataKHR& aabbs)
 364 {
 365         VkAccelerationStructureGeometryDataKHR result;
 366
 367         deMemset(&result, 0, sizeof(result));
 368
 369         result.aabbs = aabbs;
 370
 371         return result;
 372 }
 373
 374 static inline VkAccelerationStructureGeometryDataKHR makeVkAccelerationStructureInstancesDataKHR (const VkAccelerationStructureGeometryInstancesDataKHR& instances)
 375 {
 376         VkAccelerationStructureGeometryDataKHR result;
 377
 378         deMemset(&result, 0, sizeof(result));
 379
 380         result.instances = instances;
 381
 382         return result;
 383 }
 384
 385 static inline VkAccelerationStructureInstanceKHR makeVkAccelerationStructureInstanceKHR (const VkTransformMatrixKHR&                    transform,
 386                                                                                                                                                                                  deUint32                                                               instanceCustomIndex,
 387                                                                                                                                                                                  deUint32                                                               mask,
 388                                                                                                                                                                                  deUint32                                                               instanceShaderBindingTableRecordOffset,
 389                                                                                                                                                                                  VkGeometryInstanceFlagsKHR                             flags,
 390                                                                                                                                                                                  deUint64                                                               accelerationStructureReference)
 391 {
 392         VkAccelerationStructureInstanceKHR instance             = { transform, 0, 0, 0, 0, accelerationStructureReference };
 393         instance.instanceCustomIndex                                    = instanceCustomIndex & 0xFFFFFF;
 394         instance.mask                                                                   = mask & 0xFF;
 395         instance.instanceShaderBindingTableRecordOffset = instanceShaderBindingTableRecordOffset & 0xFFFFFF;
 396         instance.flags                                                                  = flags & 0xFF;
 397         return instance;
 398 }
 399
 400 VkResult getRayTracingShaderGroupHandlesKHR (const DeviceInterface&             vk,
 401                                                                                          const VkDevice                         device,
 402                                                                                          const VkPipeline                       pipeline,
 403                                                                                          const deUint32                         firstGroup,
 404                                                                                          const deUint32                         groupCount,
 405                                                                                          const deUintptr                        dataSize,
 406                                                                                          void*                                          pData)
 407 {
 408         return vk.getRayTracingShaderGroupHandlesKHR(device, pipeline, firstGroup, groupCount, dataSize, pData);
 409 }
 410
 411 VkResult getRayTracingShaderGroupHandles (const DeviceInterface&                vk,
 412                                                                                   const VkDevice                                device,
 413                                                                                   const VkPipeline                              pipeline,
 414                                                                                   const deUint32                                firstGroup,
 415                                                                                   const deUint32                                groupCount,
 416                                                                                   const deUintptr                               dataSize,
 417                                                                                   void*                                                 pData)
 418 {
 419         return getRayTracingShaderGroupHandlesKHR(vk, device, pipeline, firstGroup, groupCount, dataSize, pData);
 420 }
 421
 422 VkResult finishDeferredOperation (const DeviceInterface&        vk,
 423                                                                   VkDevice                                      device,
 424                                                                   VkDeferredOperationKHR        deferredOperation)
 425 {
 426         VkResult result = vk.deferredOperationJoinKHR(device, deferredOperation);
 427
 428         while (result == VK_THREAD_IDLE_KHR)
 429         {
 430                 std::this_thread::yield();
 431                 result = vk.deferredOperationJoinKHR(device, deferredOperation);
 432         }
 433
 434         switch( result )
 435         {
 436                 case VK_SUCCESS:
 437                 {
 438                         // Deferred operation has finished. Query its result
 439                         result = vk.getDeferredOperationResultKHR(device, deferredOperation);
 440
 441                         break;
 442                 }
 443
 444                 case VK_THREAD_DONE_KHR:
 445                 {
 446                         // Deferred operation is being wrapped up by another thread
 447                         // wait for that thread to finish
 448                         do
 449                         {
 450                                 std::this_thread::yield();
 451                                 result = vk.getDeferredOperationResultKHR(device, deferredOperation);
 452                         } while (result == VK_NOT_READY);
 453
 454                         break;
 455                 }
 456
 457                 default:
 458                 {
 459                         DE_ASSERT(false);
 460
 461                         break;
 462                 }
 463         }
 464
 465         return result;
 466 }
 467
 468 void finishDeferredOperationThreaded (DeferredThreadParams* deferredThreadParams)
 469 {
 470         deferredThreadParams->result = finishDeferredOperation(deferredThreadParams->vk, deferredThreadParams->device, deferredThreadParams->deferredOperation);
 471 }
 472
 473 void finishDeferredOperation (const DeviceInterface&    vk,
 474                                                           VkDevice                                      device,
 475                                                           VkDeferredOperationKHR        deferredOperation,
 476                                                           const deUint32                        workerThreadCount,
 477                                                           const bool                            operationNotDeferred)
 478 {
 479
 480         if (operationNotDeferred)
 481         {
 482                 // when the operation deferral returns VK_OPERATION_NOT_DEFERRED_KHR,
 483                 // the deferred operation should act as if no command was deferred
 484                 VK_CHECK(vk.getDeferredOperationResultKHR(device, deferredOperation));
 485
 486
 487                 // there is not need to join any threads to the deferred operation,
 488                 // so below can be skipped.
 489                 return;
 490         }
 491
 492         if (workerThreadCount == 0)
 493         {
 494                 VK_CHECK(finishDeferredOperation(vk, device, deferredOperation));
 495         }
 496         else
 497         {
 498                 const deUint32                                                  maxThreadCountSupported = deMinu32(256u, vk.getDeferredOperationMaxConcurrencyKHR(device, deferredOperation));
 499                 const deUint32                                                  requestedThreadCount    = workerThreadCount;
 500                 const deUint32                                                  testThreadCount                 = requestedThreadCount == std::numeric_limits<deUint32>::max() ? maxThreadCountSupported : requestedThreadCount;
 501
 502                 if (maxThreadCountSupported == 0)
 503                         TCU_FAIL("vkGetDeferredOperationMaxConcurrencyKHR must not return 0");
 504
 505                 const DeferredThreadParams                              deferredThreadParams    =
 506                 {
 507                         vk,                                     //  const DeviceInterface&      vk;
 508                         device,                         //  VkDevice                            device;
 509                         deferredOperation,      //  VkDeferredOperationKHR      deferredOperation;
 510                         VK_RESULT_MAX_ENUM,     //  VResult                                     result;
 511                 };
 512                 std::vector<DeferredThreadParams>               threadParams    (testThreadCount, deferredThreadParams);
 513                 std::vector<de::MovePtr<std::thread> >  threads                 (testThreadCount);
 514                 bool                                                                    executionResult = false;
 515
 516                 DE_ASSERT(threads.size() > 0 && threads.size() == testThreadCount);
 517
 518                 for (deUint32 threadNdx = 0; threadNdx < testThreadCount; ++threadNdx)
 519                         threads[threadNdx] = de::MovePtr<std::thread>(new std::thread(finishDeferredOperationThreaded, &threadParams[threadNdx]));
 520
 521                 for (deUint32 threadNdx = 0; threadNdx < testThreadCount; ++threadNdx)
 522                         threads[threadNdx]->join();
 523
 524                 for (deUint32 threadNdx = 0; threadNdx < testThreadCount; ++threadNdx)
 525                         if (threadParams[threadNdx].result == VK_SUCCESS)
 526                                 executionResult = true;
 527
 528                 if (!executionResult)
 529                         TCU_FAIL("Neither reported VK_SUCCESS");
 530         }
 531 }
 532
 533 SerialStorage::SerialStorage (const DeviceInterface&                                                                    vk,
 534                                                           const VkDevice                                                                                        device,
 535                                                           Allocator&                                                                                            allocator,
 536                                                           const VkAccelerationStructureBuildTypeKHR                                     buildType,
 537                                                           const VkDeviceSize                                                                            storageSize)
 538         : m_buildType (buildType)
 539         , m_storageSize(storageSize)
 540 {
 541         const VkBufferCreateInfo        bufferCreateInfo        = makeBufferCreateInfo(storageSize, VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
 542         try
 543         {
 544                 m_buffer = de::MovePtr<BufferWithMemory>(new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::Cached | MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress));
 545         }
 546         catch (const tcu::NotSupportedError&)
 547         {
 548                 // retry without Cached flag
 549                 m_buffer = de::MovePtr<BufferWithMemory>(new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress));
 550         }
 551 }
 552
 553 VkDeviceOrHostAddressKHR SerialStorage::getAddress (const DeviceInterface&                              vk,
 554                                                                                                         const VkDevice                                          device)
 555 {
 556         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
 557                 return makeDeviceOrHostAddressKHR(vk, device, m_buffer->get(), 0);
 558         else
 559                 return makeDeviceOrHostAddressKHR(m_buffer->getAllocation().getHostPtr());
 560 }
 561
 562 VkDeviceOrHostAddressConstKHR SerialStorage::getAddressConst (const DeviceInterface&    vk,
 563                                                                                                                           const VkDevice                        device)
 564 {
 565         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
 566                 return makeDeviceOrHostAddressConstKHR(vk, device, m_buffer->get(), 0);
 567         else
 568                 return makeDeviceOrHostAddressConstKHR(m_buffer->getAllocation().getHostPtr());
 569 }
 570
 571 VkDeviceSize SerialStorage::getStorageSize ()
 572 {
 573         return m_storageSize;
 574 }
 575
 576 deUint64 SerialStorage::getDeserializedSize ()
 577 {
 578         deUint64                result          = 0;
 579         const deUint8*  startPtr        = static_cast<deUint8*>(m_buffer->getAllocation().getHostPtr());
 580
 581         DE_ASSERT(sizeof(result) == DESERIALIZED_SIZE_SIZE);
 582
 583         deMemcpy(&result, startPtr + DESERIALIZED_SIZE_OFFSET, sizeof(result));
 584
 585         return result;
 586 }
 587
 588 BottomLevelAccelerationStructure::~BottomLevelAccelerationStructure ()
 589 {
 590 }
 591
 592 BottomLevelAccelerationStructure::BottomLevelAccelerationStructure ()
 593         : m_structureSize               (0u)
 594         , m_updateScratchSize   (0u)
 595         , m_buildScratchSize    (0u)
 596 {
 597 }
 598
 599 void BottomLevelAccelerationStructure::setGeometryData (const std::vector<tcu::Vec3>&   geometryData,
 600                                                                                                                 const bool                                              triangles,
 601                                                                                                                 const VkGeometryFlagsKHR                geometryFlags)
 602 {
 603         if (triangles)
 604                 DE_ASSERT((geometryData.size() % 3) == 0);
 605         else
 606                 DE_ASSERT((geometryData.size() % 2) == 0);
 607
 608         setGeometryCount(1u);
 609
 610         addGeometry(geometryData, triangles, geometryFlags);
 611 }
 612
 613 void BottomLevelAccelerationStructure::setDefaultGeometryData (const VkShaderStageFlagBits      testStage,
 614                                                                                                                            const VkGeometryFlagsKHR             geometryFlags)
 615 {
 616         bool                                    trianglesData   = false;
 617         float                                   z                               = 0.0f;
 618         std::vector<tcu::Vec3>  geometryData;
 619
 620         switch (testStage)
 621         {
 622                 case VK_SHADER_STAGE_RAYGEN_BIT_KHR:            z = -1.0f; trianglesData = true;        break;
 623                 case VK_SHADER_STAGE_ANY_HIT_BIT_KHR:           z = -1.0f; trianglesData = true;        break;
 624                 case VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR:       z = -1.0f; trianglesData = true;        break;
 625                 case VK_SHADER_STAGE_MISS_BIT_KHR:                      z = -9.9f; trianglesData = true;        break;
 626                 case VK_SHADER_STAGE_INTERSECTION_BIT_KHR:      z = -1.0f; trianglesData = false;       break;
 627                 case VK_SHADER_STAGE_CALLABLE_BIT_KHR:          z = -1.0f; trianglesData = true;        break;
 628                 default:                                                                        TCU_THROW(InternalError, "Unacceptable stage");
 629         }
 630
 631         if (trianglesData)
 632         {
 633                 geometryData.reserve(6);
 634
 635                 geometryData.push_back(tcu::Vec3(-1.0f, -1.0f, z));
 636                 geometryData.push_back(tcu::Vec3(-1.0f, +1.0f, z));
 637                 geometryData.push_back(tcu::Vec3(+1.0f, -1.0f, z));
 638                 geometryData.push_back(tcu::Vec3(-1.0f, +1.0f, z));
 639                 geometryData.push_back(tcu::Vec3(+1.0f, -1.0f, z));
 640                 geometryData.push_back(tcu::Vec3(+1.0f, +1.0f, z));
 641         }
 642         else
 643         {
 644                 geometryData.reserve(2);
 645
 646                 geometryData.push_back(tcu::Vec3(-1.0f, -1.0f, z));
 647                 geometryData.push_back(tcu::Vec3(+1.0f, +1.0f, z));
 648         }
 649
 650         setGeometryCount(1u);
 651
 652         addGeometry(geometryData, trianglesData, geometryFlags);
 653 }
 654
 655 void BottomLevelAccelerationStructure::setGeometryCount (const size_t geometryCount)
 656 {
 657         m_geometriesData.clear();
 658
 659         m_geometriesData.reserve(geometryCount);
 660 }
 661
 662 void BottomLevelAccelerationStructure::addGeometry (de::SharedPtr<RaytracedGeometryBase>&               raytracedGeometry)
 663 {
 664         m_geometriesData.push_back(raytracedGeometry);
 665 }
 666
 667 void BottomLevelAccelerationStructure::addGeometry (const std::vector<tcu::Vec3>&       geometryData,
 668                                                                                                         const bool                                              triangles,
 669                                                                                                         const VkGeometryFlagsKHR                geometryFlags)
 670 {
 671         DE_ASSERT(geometryData.size() > 0);
 672         DE_ASSERT((triangles && geometryData.size() % 3 == 0) || (!triangles && geometryData.size() % 2 == 0));
 673
 674         if (!triangles)
 675                 for (size_t posNdx = 0; posNdx < geometryData.size() / 2; ++posNdx)
 676                 {
 677                         DE_ASSERT(geometryData[2 * posNdx].x() <= geometryData[2 * posNdx + 1].x());
 678                         DE_ASSERT(geometryData[2 * posNdx].y() <= geometryData[2 * posNdx + 1].y());
 679                         DE_ASSERT(geometryData[2 * posNdx].z() <= geometryData[2 * posNdx + 1].z());
 680                 }
 681
 682         de::SharedPtr<RaytracedGeometryBase> geometry = makeRaytracedGeometry(triangles ? VK_GEOMETRY_TYPE_TRIANGLES_KHR : VK_GEOMETRY_TYPE_AABBS_KHR, VK_FORMAT_R32G32B32_SFLOAT, VK_INDEX_TYPE_NONE_KHR);
 683         for (auto it = begin(geometryData), eit = end(geometryData); it != eit; ++it)
 684                 geometry->addVertex(*it);
 685
 686         geometry->setGeometryFlags(geometryFlags);
 687         addGeometry(geometry);
 688 }
 689
 690 VkDeviceSize BottomLevelAccelerationStructure::getStructureSize() const
 691 {
 692         return m_structureSize;
 693 }
 694
 695 BufferWithMemory* createVertexBuffer (const DeviceInterface&                                                                    vk,
 696                                                                           const VkDevice                                                                                        device,
 697                                                                           Allocator&                                                                                            allocator,
 698                                                                           const std::vector<de::SharedPtr<RaytracedGeometryBase>>&      geometriesData)
 699 {
 700         DE_ASSERT(geometriesData.size() != 0);
 701
 702         VkDeviceSize                                    bufferSizeBytes = 0;
 703         for (size_t geometryNdx = 0; geometryNdx < geometriesData.size(); ++geometryNdx)
 704                 bufferSizeBytes += deAlignSize(geometriesData[geometryNdx]->getVertexByteSize(),8);
 705
 706         const VkBufferCreateInfo                bufferCreateInfo        = makeBufferCreateInfo(bufferSizeBytes, VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT_KHR | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
 707         return new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress);
 708 }
 709
 710 void updateVertexBuffer (const DeviceInterface&                                                                         vk,
 711                                                  const VkDevice                                                                                         device,
 712                                                  const std::vector<de::SharedPtr<RaytracedGeometryBase>>&       geometriesData,
 713                                                  BufferWithMemory*                                                                                      vertexBuffer)
 714 {
 715         const Allocation&                               geometryAlloc           = vertexBuffer->getAllocation();
 716         deUint8*                                                bufferStart                     = static_cast<deUint8*>(geometryAlloc.getHostPtr());
 717         VkDeviceSize                                    bufferOffset            = 0;
 718
 719         for (size_t geometryNdx = 0; geometryNdx < geometriesData.size(); ++geometryNdx)
 720         {
 721                 const void*                                     geometryPtr                     = geometriesData[geometryNdx]->getVertexPointer();
 722                 const size_t                            geometryPtrSize         = geometriesData[geometryNdx]->getVertexByteSize();
 723
 724                 deMemcpy(&bufferStart[bufferOffset], geometryPtr, geometryPtrSize);
 725
 726                 bufferOffset += deAlignSize(geometryPtrSize,8);
 727         }
 728
 729         flushMappedMemoryRange(vk, device, geometryAlloc.getMemory(), geometryAlloc.getOffset(), VK_WHOLE_SIZE);
 730 }
 731
 732 BufferWithMemory* createIndexBuffer (const DeviceInterface&                                                                             vk,
 733                                                                          const VkDevice                                                                                         device,
 734                                                                          Allocator&                                                                                                     allocator,
 735                                                                          const std::vector<de::SharedPtr<RaytracedGeometryBase>>&       geometriesData)
 736 {
 737         DE_ASSERT(!geometriesData.empty());
 738
 739         VkDeviceSize                                    bufferSizeBytes = 0;
 740         for (size_t geometryNdx = 0; geometryNdx < geometriesData.size(); ++geometryNdx)
 741                 if(geometriesData[geometryNdx]->getIndexType() != VK_INDEX_TYPE_NONE_KHR)
 742                         bufferSizeBytes += deAlignSize(geometriesData[geometryNdx]->getIndexByteSize(),8);
 743
 744         if (bufferSizeBytes == 0)
 745                 return DE_NULL;
 746
 747         const VkBufferCreateInfo                bufferCreateInfo        = makeBufferCreateInfo(bufferSizeBytes, VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT_KHR | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
 748         return  new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress);
 749 }
 750
 751 void updateIndexBuffer (const DeviceInterface&                                                                          vk,
 752                                                 const VkDevice                                                                                          device,
 753                                                 const std::vector<de::SharedPtr<RaytracedGeometryBase>>&        geometriesData,
 754                                                 BufferWithMemory*                                                                                       indexBuffer)
 755 {
 756         const Allocation&                               indexAlloc                      = indexBuffer->getAllocation();
 757         deUint8*                                                bufferStart                     = static_cast<deUint8*>(indexAlloc.getHostPtr());
 758         VkDeviceSize                                    bufferOffset            = 0;
 759
 760         for (size_t geometryNdx = 0; geometryNdx < geometriesData.size(); ++geometryNdx)
 761         {
 762                 if (geometriesData[geometryNdx]->getIndexType() != VK_INDEX_TYPE_NONE_KHR)
 763                 {
 764                         const void*                                     indexPtr                = geometriesData[geometryNdx]->getIndexPointer();
 765                         const size_t                            indexPtrSize    = geometriesData[geometryNdx]->getIndexByteSize();
 766
 767                         deMemcpy(&bufferStart[bufferOffset], indexPtr, indexPtrSize);
 768
 769                         bufferOffset += deAlignSize(indexPtrSize, 8);
 770                 }
 771         }
 772
 773         flushMappedMemoryRange(vk, device, indexAlloc.getMemory(), indexAlloc.getOffset(), VK_WHOLE_SIZE);
 774 }
 775
 776 class BottomLevelAccelerationStructureKHR : public BottomLevelAccelerationStructure
 777 {
 778 public:
 779         static deUint32                                                                                 getRequiredAllocationCount                                              (void);
 780
 781                                                                                                                         BottomLevelAccelerationStructureKHR                             ();
 782                                                                                                                         BottomLevelAccelerationStructureKHR                             (const BottomLevelAccelerationStructureKHR&             other) = delete;
 783         virtual                                                                                                 ~BottomLevelAccelerationStructureKHR                    ();
 784
 785         void                                                                                                    setBuildType                                                                    (const VkAccelerationStructureBuildTypeKHR              buildType) override;
 786         void                                                                                                    setCreateFlags                                                                  (const VkAccelerationStructureCreateFlagsKHR    createFlags) override;
 787         void                                                                                                    setCreateGeneric                                                                (bool                                                                                   createGeneric) override;
 788         void                                                                                                    setBuildFlags                                                                   (const VkBuildAccelerationStructureFlagsKHR             buildFlags) override;
 789         void                                                                                                    setBuildWithoutGeometries                                               (bool                                                                                   buildWithoutGeometries) override;
 790         void                                                                                                    setBuildWithoutPrimitives                                               (bool                                                                                   buildWithoutPrimitives) override;
 791         void                                                                                                    setDeferredOperation                                                    (const bool                                                                             deferredOperation,
 792                                                                                                                                                                                                                          const deUint32                                                                 workerThreadCount) override;
 793         void                                                                                                    setUseArrayOfPointers                                                   (const bool                                                                             useArrayOfPointers) override;
 794         void                                                                                                    setIndirectBuildParameters                                              (const VkBuffer                                                                 indirectBuffer,
 795                                                                                                                                                                                                                          const VkDeviceSize                                                             indirectBufferOffset,
 796                                                                                                                                                                                                                          const deUint32                                                                 indirectBufferStride) override;
 797         VkBuildAccelerationStructureFlagsKHR                                    getBuildFlags                                                                   () const override;
 798
 799         void                                                                                                    create                                                                                  (const DeviceInterface&                                                 vk,
 800                                                                                                                                                                                                                          const VkDevice                                                                 device,
 801                                                                                                                                                                                                                          Allocator&                                                                             allocator,
 802                                                                                                                                                                                                                          VkDeviceSize                                                                   structureSize,
 803                                                                                                                                                                                                                          VkDeviceAddress                                                                deviceAddress   = 0u ) override;
 804         void                                                                                                    build                                                                                   (const DeviceInterface&                                                 vk,
 805                                                                                                                                                                                                                          const VkDevice                                                                 device,
 806                                                                                                                                                                                                                          const VkCommandBuffer                                                  cmdBuffer) override;
 807         void                                                                                                    copyFrom                                                                                (const DeviceInterface&                                                 vk,
 808                                                                                                                                                                                                                          const VkDevice                                                                 device,
 809                                                                                                                                                                                                                          const VkCommandBuffer                                                  cmdBuffer,
 810                                                                                                                                                                                                                          BottomLevelAccelerationStructure*                              accelerationStructure,
 811                                                                                                                                                                                                                          bool                                                                                   compactCopy) override;
 812
 813         void                                                                                                    serialize                                                                               (const DeviceInterface&                                                 vk,
 814                                                                                                                                                                                                                          const VkDevice                                                                 device,
 815                                                                                                                                                                                                                          const VkCommandBuffer                                                  cmdBuffer,
 816                                                                                                                                                                                                                          SerialStorage*                                                                 storage) override;
 817         void                                                                                                    deserialize                                                                             (const DeviceInterface&                                                 vk,
 818                                                                                                                                                                                                                          const VkDevice                                                                 device,
 819                                                                                                                                                                                                                          const VkCommandBuffer                                                  cmdBuffer,
 820                                                                                                                                                                                                                          SerialStorage*                                                                 storage) override;
 821
 822         const VkAccelerationStructureKHR*                                               getPtr                                                                                  (void) const override;
 823
 824 protected:
 825         VkAccelerationStructureBuildTypeKHR                                             m_buildType;
 826         VkAccelerationStructureCreateFlagsKHR                                   m_createFlags;
 827         bool                                                                                                    m_createGeneric;
 828         VkBuildAccelerationStructureFlagsKHR                                    m_buildFlags;
 829         bool                                                                                                    m_buildWithoutGeometries;
 830         bool                                                                                                    m_buildWithoutPrimitives;
 831         bool                                                                                                    m_deferredOperation;
 832         deUint32                                                                                                m_workerThreadCount;
 833         bool                                                                                                    m_useArrayOfPointers;
 834         de::MovePtr<BufferWithMemory>                                                   m_accelerationStructureBuffer;
 835         de::MovePtr<BufferWithMemory>                                                   m_vertexBuffer;
 836         de::MovePtr<BufferWithMemory>                                                   m_indexBuffer;
 837         de::MovePtr<BufferWithMemory>                                                   m_deviceScratchBuffer;
 838         std::vector<deUint8>                                                                    m_hostScratchBuffer;
 839         Move<VkAccelerationStructureKHR>                                                m_accelerationStructureKHR;
 840         VkBuffer                                                                                                m_indirectBuffer;
 841         VkDeviceSize                                                                                    m_indirectBufferOffset;
 842         deUint32                                                                                                m_indirectBufferStride;
 843
 844         void                                                                                                    prepareGeometries                                                               (const DeviceInterface&                                                                 vk,
 845                                                                                                                                                                                                                          const VkDevice                                                                                 device,
 846                                                                                                                                                                                                                          std::vector<VkAccelerationStructureGeometryKHR>&               accelerationStructureGeometriesKHR,
 847                                                                                                                                                                                                                          std::vector<VkAccelerationStructureGeometryKHR*>&              accelerationStructureGeometriesKHRPointers,
 848                                                                                                                                                                                                                          std::vector<VkAccelerationStructureBuildRangeInfoKHR>& accelerationStructureBuildRangeInfoKHR,
 849                                                                                                                                                                                                                          std::vector<deUint32>&                                                                 maxPrimitiveCounts);
 850 };
 851
 852 deUint32 BottomLevelAccelerationStructureKHR::getRequiredAllocationCount (void)
 853 {
 854         /*
 855                 de::MovePtr<BufferWithMemory>                                                   m_geometryBuffer; // but only when m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR
 856                 de::MovePtr<Allocation>                                                                 m_accelerationStructureAlloc;
 857                 de::MovePtr<BufferWithMemory>                                                   m_deviceScratchBuffer;
 858         */
 859         return 3u;
 860 }
 861
 862 BottomLevelAccelerationStructureKHR::~BottomLevelAccelerationStructureKHR ()
 863 {
 864 }
 865
 866 BottomLevelAccelerationStructureKHR::BottomLevelAccelerationStructureKHR ()
 867         : BottomLevelAccelerationStructure      ()
 868         , m_buildType                                           (VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
 869         , m_createFlags                                         (0u)
 870         , m_createGeneric                                       (false)
 871         , m_buildFlags                                          (0u)
 872         , m_buildWithoutGeometries                      (false)
 873         , m_buildWithoutPrimitives                      (false)
 874         , m_deferredOperation                           (false)
 875         , m_workerThreadCount                           (0)
 876         , m_useArrayOfPointers                          (false)
 877         , m_accelerationStructureBuffer         (DE_NULL)
 878         , m_vertexBuffer                                        (DE_NULL)
 879         , m_indexBuffer                                         (DE_NULL)
 880         , m_deviceScratchBuffer                         (DE_NULL)
 881         , m_accelerationStructureKHR            ()
 882         , m_indirectBuffer                                      (DE_NULL)
 883         , m_indirectBufferOffset                        (0)
 884         , m_indirectBufferStride                        (0)
 885 {
 886 }
 887
 888 void BottomLevelAccelerationStructureKHR::setBuildType (const VkAccelerationStructureBuildTypeKHR       buildType)
 889 {
 890         m_buildType = buildType;
 891 }
 892
 893 void BottomLevelAccelerationStructureKHR::setCreateFlags (const VkAccelerationStructureCreateFlagsKHR   createFlags)
 894 {
 895         m_createFlags = createFlags;
 896 }
 897
 898 void BottomLevelAccelerationStructureKHR::setCreateGeneric (bool createGeneric)
 899 {
 900         m_createGeneric = createGeneric;
 901 }
 902
 903 void BottomLevelAccelerationStructureKHR::setBuildFlags (const VkBuildAccelerationStructureFlagsKHR     buildFlags)
 904 {
 905         m_buildFlags = buildFlags;
 906 }
 907
 908 void BottomLevelAccelerationStructureKHR::setBuildWithoutGeometries (bool buildWithoutGeometries)
 909 {
 910         m_buildWithoutGeometries = buildWithoutGeometries;
 911 }
 912
 913 void BottomLevelAccelerationStructureKHR::setBuildWithoutPrimitives (bool buildWithoutPrimitives)
 914 {
 915         m_buildWithoutPrimitives = buildWithoutPrimitives;
 916 }
 917
 918 void BottomLevelAccelerationStructureKHR::setDeferredOperation (const bool              deferredOperation,
 919                                                                                                                                 const deUint32  workerThreadCount)
 920 {
 921         m_deferredOperation = deferredOperation;
 922         m_workerThreadCount = workerThreadCount;
 923 }
 924
 925 void BottomLevelAccelerationStructureKHR::setUseArrayOfPointers (const bool     useArrayOfPointers)
 926 {
 927         m_useArrayOfPointers = useArrayOfPointers;
 928 }
 929
 930 void BottomLevelAccelerationStructureKHR::setIndirectBuildParameters (const VkBuffer            indirectBuffer,
 931                                                                                                                                           const VkDeviceSize    indirectBufferOffset,
 932                                                                                                                                           const deUint32                indirectBufferStride)
 933 {
 934         m_indirectBuffer                = indirectBuffer;
 935         m_indirectBufferOffset  = indirectBufferOffset;
 936         m_indirectBufferStride  = indirectBufferStride;
 937 }
 938
 939 VkBuildAccelerationStructureFlagsKHR BottomLevelAccelerationStructureKHR::getBuildFlags () const
 940 {
 941         return m_buildFlags;
 942 }
 943
 944 void BottomLevelAccelerationStructureKHR::create (const DeviceInterface&                                vk,
 945                                                                                                   const VkDevice                                                device,
 946                                                                                                   Allocator&                                                    allocator,
 947                                                                                                   VkDeviceSize                                                  structureSize,
 948                                                                                                   VkDeviceAddress                                               deviceAddress)
 949 {
 950         // AS may be built from geometries using vkCmdBuildAccelerationStructuresKHR / vkBuildAccelerationStructuresKHR
 951         // or may be copied/compacted/deserialized from other AS ( in this case AS does not need geometries, but it needs to know its size before creation ).
 952         DE_ASSERT(!m_geometriesData.empty() !=  !(structureSize == 0)); // logical xor
 953
 954         if (structureSize == 0)
 955         {
 956                 std::vector<VkAccelerationStructureGeometryKHR>                 accelerationStructureGeometriesKHR;
 957                 std::vector<VkAccelerationStructureGeometryKHR*>                accelerationStructureGeometriesKHRPointers;
 958                 std::vector<VkAccelerationStructureBuildRangeInfoKHR>   accelerationStructureBuildRangeInfoKHR;
 959                 std::vector<deUint32>                                                                   maxPrimitiveCounts;
 960                 prepareGeometries(vk, device, accelerationStructureGeometriesKHR, accelerationStructureGeometriesKHRPointers, accelerationStructureBuildRangeInfoKHR, maxPrimitiveCounts);
 961
 962                 const VkAccelerationStructureGeometryKHR*                               accelerationStructureGeometriesKHRPointer       = accelerationStructureGeometriesKHR.data();
 963                 const VkAccelerationStructureGeometryKHR* const*                accelerationStructureGeometry                           = accelerationStructureGeometriesKHRPointers.data();
 964
 965                 VkAccelerationStructureBuildGeometryInfoKHR     accelerationStructureBuildGeometryInfoKHR       =
 966                 {
 967                         VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_GEOMETRY_INFO_KHR,                       //  VkStructureType                                                                             sType;
 968                         DE_NULL,                                                                                                                                        //  const void*                                                                                 pNext;
 969                         VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR,                                                        //  VkAccelerationStructureTypeKHR                                              type;
 970                         m_buildFlags,                                                                                                                           //  VkBuildAccelerationStructureFlagsKHR                                flags;
 971                         VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR,                                                         //  VkBuildAccelerationStructureModeKHR                                 mode;
 972                         DE_NULL,                                                                                                                                        //  VkAccelerationStructureKHR                                                  srcAccelerationStructure;
 973                         DE_NULL,                                                                                                                                        //  VkAccelerationStructureKHR                                                  dstAccelerationStructure;
 974                         static_cast<deUint32>(accelerationStructureGeometriesKHR.size()),                       //  deUint32                                                                                    geometryCount;
 975                         m_useArrayOfPointers ? DE_NULL : accelerationStructureGeometriesKHRPointer,     //  const VkAccelerationStructureGeometryKHR*                   pGeometries;
 976                         m_useArrayOfPointers ? accelerationStructureGeometry : DE_NULL,                         //  const VkAccelerationStructureGeometryKHR* const*    ppGeometries;
 977                         makeDeviceOrHostAddressKHR(DE_NULL)                                                                                     //  VkDeviceOrHostAddressKHR                                                    scratchData;
 978                 };
 979                 VkAccelerationStructureBuildSizesInfoKHR sizeInfo =
 980                 {
 981                         VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_SIZES_INFO_KHR,  //  VkStructureType     sType;
 982                         DE_NULL,                                                                                                                //  const void*         pNext;
 983                         0,                                                                                                                              //  VkDeviceSize        accelerationStructureSize;
 984                         0,                                                                                                                              //  VkDeviceSize        updateScratchSize;
 985                         0                                                                                                                               //  VkDeviceSize        buildScratchSize;
 986                 };
 987
 988                 vk.getAccelerationStructureBuildSizesKHR(device, m_buildType, &accelerationStructureBuildGeometryInfoKHR, maxPrimitiveCounts.data(), &sizeInfo);
 989
 990                 m_structureSize         = sizeInfo.accelerationStructureSize;
 991                 m_updateScratchSize     = sizeInfo.updateScratchSize;
 992                 m_buildScratchSize      = sizeInfo.buildScratchSize;
 993         }
 994         else
 995         {
 996                 m_structureSize         = structureSize;
 997                 m_updateScratchSize     = 0u;
 998                 m_buildScratchSize      = 0u;
 999         }
1000
1001         {
1002                 const VkBufferCreateInfo                bufferCreateInfo        = makeBufferCreateInfo(m_structureSize, VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
1003                 try
1004                 {
1005                         m_accelerationStructureBuffer                                           = de::MovePtr<BufferWithMemory>(new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::Cached | MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress));
1006                 }
1007                 catch (const tcu::NotSupportedError&)
1008                 {
1009                         // retry without Cached flag
1010                         m_accelerationStructureBuffer                                           = de::MovePtr<BufferWithMemory>(new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress));
1011                 }
1012         }
1013
1014         {
1015                 const VkAccelerationStructureTypeKHR            structureType                                           = (m_createGeneric
1016                                                                                                                                                                                    ? VK_ACCELERATION_STRUCTURE_TYPE_GENERIC_KHR
1017                                                                                                                                                                                    : VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR);
1018                 const VkAccelerationStructureCreateInfoKHR      accelerationStructureCreateInfoKHR
1019                 {
1020                         VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR,                                               //  VkStructureType                                                                                     sType;
1021                         DE_NULL,                                                                                                                                                //  const void*                                                                                         pNext;
1022                         m_createFlags,                                                                                                                                  //  VkAccelerationStructureCreateFlagsKHR                                       createFlags;
1023                         m_accelerationStructureBuffer->get(),                                                                                   //  VkBuffer                                                                                            buffer;
1024                         0u,                                                                                                                                                             //  VkDeviceSize                                                                                        offset;
1025                         m_structureSize,                                                                                                                                //  VkDeviceSize                                                                                        size;
1026                         structureType,                                                                                                                                  //  VkAccelerationStructureTypeKHR                                                      type;
1027                         deviceAddress                                                                                                                                   //  VkDeviceAddress                                                                                     deviceAddress;
1028                 };
1029
1030                 m_accelerationStructureKHR      = createAccelerationStructureKHR(vk, device, &accelerationStructureCreateInfoKHR, DE_NULL);
1031         }
1032
1033         if (m_buildScratchSize > 0u)
1034         {
1035                 if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1036                 {
1037                         const VkBufferCreateInfo                bufferCreateInfo = makeBufferCreateInfo(m_buildScratchSize, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
1038                         m_deviceScratchBuffer = de::MovePtr<BufferWithMemory>(new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress));
1039                 }
1040                 else
1041                 {
1042                         m_hostScratchBuffer.resize(static_cast<size_t>(m_buildScratchSize));
1043                 }
1044         }
1045
1046         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR && !m_geometriesData.empty())
1047         {
1048                 m_vertexBuffer  = de::MovePtr<BufferWithMemory>(createVertexBuffer(vk, device, allocator, m_geometriesData));
1049                 m_indexBuffer   = de::MovePtr<BufferWithMemory>(createIndexBuffer(vk, device, allocator, m_geometriesData));
1050         }
1051 }
1052
1053 void BottomLevelAccelerationStructureKHR::build (const DeviceInterface&                                         vk,
1054                                                                                                  const VkDevice                                                         device,
1055                                                                                                  const VkCommandBuffer                                          cmdBuffer)
1056 {
1057         DE_ASSERT(!m_geometriesData.empty());
1058         DE_ASSERT(m_accelerationStructureKHR.get() != DE_NULL);
1059         DE_ASSERT(m_buildScratchSize != 0);
1060
1061         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1062         {
1063                 updateVertexBuffer(vk, device, m_geometriesData, m_vertexBuffer.get());
1064                 if(m_indexBuffer.get() != DE_NULL)
1065                         updateIndexBuffer(vk, device, m_geometriesData, m_indexBuffer.get());
1066         }
1067
1068         {
1069                 std::vector<VkAccelerationStructureGeometryKHR>                 accelerationStructureGeometriesKHR;
1070                 std::vector<VkAccelerationStructureGeometryKHR*>                accelerationStructureGeometriesKHRPointers;
1071                 std::vector<VkAccelerationStructureBuildRangeInfoKHR>   accelerationStructureBuildRangeInfoKHR;
1072                 std::vector<deUint32>                                                                   maxPrimitiveCounts;
1073
1074                 prepareGeometries(vk, device, accelerationStructureGeometriesKHR, accelerationStructureGeometriesKHRPointers, accelerationStructureBuildRangeInfoKHR, maxPrimitiveCounts);
1075
1076                 const VkAccelerationStructureGeometryKHR*                       accelerationStructureGeometriesKHRPointer       = accelerationStructureGeometriesKHR.data();
1077                 const VkAccelerationStructureGeometryKHR* const*        accelerationStructureGeometry                           = accelerationStructureGeometriesKHRPointers.data();
1078                 VkDeviceOrHostAddressKHR                                                        scratchData                                                                     = (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1079                                                                                                                                                                                                                 ? makeDeviceOrHostAddressKHR(vk, device, m_deviceScratchBuffer->get(), 0)
1080                                                                                                                                                                                                                 : makeDeviceOrHostAddressKHR(m_hostScratchBuffer.data());
1081                 const deUint32                                                                          geometryCount                                                           = (m_buildWithoutGeometries
1082                                                                                                                                                                                                                 ? 0u
1083                                                                                                                                                                                                                 : static_cast<deUint32>(accelerationStructureGeometriesKHR.size()));
1084
1085                 VkAccelerationStructureBuildGeometryInfoKHR     accelerationStructureBuildGeometryInfoKHR       =
1086                 {
1087                         VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_GEOMETRY_INFO_KHR,                       //  VkStructureType                                                                             sType;
1088                         DE_NULL,                                                                                                                                        //  const void*                                                                                 pNext;
1089                         VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR,                                                        //  VkAccelerationStructureTypeKHR                                              type;
1090                         m_buildFlags,                                                                                                                           //  VkBuildAccelerationStructureFlagsKHR                                flags;
1091                         VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR,                                                         //  VkBuildAccelerationStructureModeKHR                                 mode;
1092                         DE_NULL,                                                                                                                                        //  VkAccelerationStructureKHR                                                  srcAccelerationStructure;
1093                         m_accelerationStructureKHR.get(),                                                                                       //  VkAccelerationStructureKHR                                                  dstAccelerationStructure;
1094                         geometryCount,                                                                                                                          //  deUint32                                                                                    geometryCount;
1095                         m_useArrayOfPointers ? DE_NULL : accelerationStructureGeometriesKHRPointer,     //  const VkAccelerationStructureGeometryKHR*                   pGeometries;
1096                         m_useArrayOfPointers ? accelerationStructureGeometry : DE_NULL,                         //  const VkAccelerationStructureGeometryKHR* const*    ppGeometries;
1097                         scratchData                                                                                                                                     //  VkDeviceOrHostAddressKHR                                                    scratchData;
1098                 };
1099
1100                 VkAccelerationStructureBuildRangeInfoKHR* accelerationStructureBuildRangeInfoKHRPtr     = accelerationStructureBuildRangeInfoKHR.data();
1101
1102                 if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1103                 {
1104                         if (m_indirectBuffer == DE_NULL)
1105                                 vk.cmdBuildAccelerationStructuresKHR(cmdBuffer, 1u, &accelerationStructureBuildGeometryInfoKHR, (const VkAccelerationStructureBuildRangeInfoKHR**)&accelerationStructureBuildRangeInfoKHRPtr);
1106                         else
1107                         {
1108                                 VkDeviceAddress indirectDeviceAddress   = getBufferDeviceAddress(vk, device, m_indirectBuffer, m_indirectBufferOffset);
1109                                 deUint32*               pMaxPrimitiveCounts             = maxPrimitiveCounts.data();
1110                                 vk.cmdBuildAccelerationStructuresIndirectKHR(cmdBuffer, 1u, &accelerationStructureBuildGeometryInfoKHR, &indirectDeviceAddress, &m_indirectBufferStride, &pMaxPrimitiveCounts);
1111                         }
1112                 }
1113                 else if (!m_deferredOperation)
1114                 {
1115                         VK_CHECK(vk.buildAccelerationStructuresKHR(device, DE_NULL, 1u, &accelerationStructureBuildGeometryInfoKHR, (const VkAccelerationStructureBuildRangeInfoKHR**)&accelerationStructureBuildRangeInfoKHRPtr));
1116                 }
1117                 else
1118                 {
1119                         const auto deferredOperationPtr = createDeferredOperationKHR(vk, device);
1120                         const auto deferredOperation    = deferredOperationPtr.get();
1121
1122                         VkResult result = vk.buildAccelerationStructuresKHR(device, deferredOperation, 1u, &accelerationStructureBuildGeometryInfoKHR, (const VkAccelerationStructureBuildRangeInfoKHR**)&accelerationStructureBuildRangeInfoKHRPtr);
1123
1124                         DE_ASSERT(result == VK_OPERATION_DEFERRED_KHR || result == VK_OPERATION_NOT_DEFERRED_KHR || result == VK_SUCCESS);
1125
1126                         finishDeferredOperation(vk, device, deferredOperation, m_workerThreadCount, result == VK_OPERATION_NOT_DEFERRED_KHR);
1127                 }
1128         }
1129
1130         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1131         {
1132                 const VkAccessFlags             accessMasks     = VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR | VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR;
1133                 const VkMemoryBarrier   memBarrier      = makeMemoryBarrier(accessMasks, accessMasks);
1134
1135                 cmdPipelineMemoryBarrier(vk, cmdBuffer, VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, &memBarrier);
1136         }
1137 }
1138
1139 void BottomLevelAccelerationStructureKHR::copyFrom (const DeviceInterface&                                              vk,
1140                                                                                                         const VkDevice                                                          device,
1141                                                                                                         const VkCommandBuffer                                           cmdBuffer,
1142                                                                                                         BottomLevelAccelerationStructure*                       accelerationStructure,
1143                                                                                                         bool                                                                            compactCopy)
1144 {
1145         DE_ASSERT(m_accelerationStructureKHR.get() != DE_NULL);
1146         DE_ASSERT(accelerationStructure != DE_NULL);
1147
1148         VkCopyAccelerationStructureInfoKHR copyAccelerationStructureInfo =
1149         {
1150                 VK_STRUCTURE_TYPE_COPY_ACCELERATION_STRUCTURE_INFO_KHR,                                                                                                                 // VkStructureType                                              sType;
1151                 DE_NULL,                                                                                                                                                                                                                // const void*                                                  pNext;
1152                 *(accelerationStructure->getPtr()),                                                                                                                                                             // VkAccelerationStructureKHR                   src;
1153                 *(getPtr()),                                                                                                                                                                                                    // VkAccelerationStructureKHR                   dst;
1154                 compactCopy ? VK_COPY_ACCELERATION_STRUCTURE_MODE_COMPACT_KHR : VK_COPY_ACCELERATION_STRUCTURE_MODE_CLONE_KHR   // VkCopyAccelerationStructureModeKHR   mode;
1155         };
1156
1157         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1158         {
1159                 vk.cmdCopyAccelerationStructureKHR(cmdBuffer, &copyAccelerationStructureInfo);
1160         }
1161         else if (!m_deferredOperation)
1162         {
1163                 VK_CHECK(vk.copyAccelerationStructureKHR(device, DE_NULL, &copyAccelerationStructureInfo));
1164         }
1165         else
1166         {
1167                 const auto deferredOperationPtr = createDeferredOperationKHR(vk, device);
1168                 const auto deferredOperation    = deferredOperationPtr.get();
1169
1170                 VkResult result = vk.copyAccelerationStructureKHR(device, deferredOperation, &copyAccelerationStructureInfo);
1171
1172                 DE_ASSERT(result == VK_OPERATION_DEFERRED_KHR || result == VK_OPERATION_NOT_DEFERRED_KHR || result == VK_SUCCESS);
1173
1174                 finishDeferredOperation(vk, device, deferredOperation, m_workerThreadCount, result == VK_OPERATION_NOT_DEFERRED_KHR);
1175         }
1176
1177         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1178         {
1179                 const VkAccessFlags             accessMasks     = VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR | VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR;
1180                 const VkMemoryBarrier   memBarrier      = makeMemoryBarrier(accessMasks, accessMasks);
1181
1182                 cmdPipelineMemoryBarrier(vk, cmdBuffer, VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, &memBarrier);
1183         }
1184 }
1185
1186 void BottomLevelAccelerationStructureKHR::serialize (const DeviceInterface&             vk,
1187                                                                                                          const VkDevice                         device,
1188                                                                                                          const VkCommandBuffer          cmdBuffer,
1189                                                                                                          SerialStorage*                         storage)
1190 {
1191         DE_ASSERT(m_accelerationStructureKHR.get() != DE_NULL);
1192         DE_ASSERT(storage != DE_NULL);
1193
1194         const VkCopyAccelerationStructureToMemoryInfoKHR        copyAccelerationStructureInfo   =
1195         {
1196                 VK_STRUCTURE_TYPE_COPY_ACCELERATION_STRUCTURE_TO_MEMORY_INFO_KHR,       // VkStructureType                                              sType;
1197                 DE_NULL,                                                                                                                        // const void*                                                  pNext;
1198                 *(getPtr()),                                                                                                            // VkAccelerationStructureKHR                   src;
1199                 storage->getAddress(vk,device),                                                                         // VkDeviceOrHostAddressKHR                             dst;
1200                 VK_COPY_ACCELERATION_STRUCTURE_MODE_SERIALIZE_KHR                                       // VkCopyAccelerationStructureModeKHR   mode;
1201         };
1202
1203         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1204         {
1205                 vk.cmdCopyAccelerationStructureToMemoryKHR(cmdBuffer, &copyAccelerationStructureInfo);
1206         }
1207         else if (!m_deferredOperation)
1208         {
1209                 VK_CHECK(vk.copyAccelerationStructureToMemoryKHR(device, DE_NULL, &copyAccelerationStructureInfo));
1210         }
1211         else
1212         {
1213                 const auto deferredOperationPtr = createDeferredOperationKHR(vk, device);
1214                 const auto deferredOperation    = deferredOperationPtr.get();
1215
1216                 const VkResult result = vk.copyAccelerationStructureToMemoryKHR(device, deferredOperation, &copyAccelerationStructureInfo);
1217
1218                 DE_ASSERT(result == VK_OPERATION_DEFERRED_KHR || result == VK_OPERATION_NOT_DEFERRED_KHR || result == VK_SUCCESS);
1219
1220                 finishDeferredOperation(vk, device, deferredOperation, m_workerThreadCount, result == VK_OPERATION_NOT_DEFERRED_KHR);
1221         }
1222 }
1223
1224 void BottomLevelAccelerationStructureKHR::deserialize (const DeviceInterface&   vk,
1225                                                                                                            const VkDevice                       device,
1226                                                                                                            const VkCommandBuffer        cmdBuffer,
1227                                                                                                            SerialStorage*                       storage)
1228 {
1229         DE_ASSERT(m_accelerationStructureKHR.get() != DE_NULL);
1230         DE_ASSERT(storage != DE_NULL);
1231
1232         const VkCopyMemoryToAccelerationStructureInfoKHR        copyAccelerationStructureInfo   =
1233         {
1234                 VK_STRUCTURE_TYPE_COPY_MEMORY_TO_ACCELERATION_STRUCTURE_INFO_KHR,       // VkStructureType                                                      sType;
1235                 DE_NULL,                                                                                                                        // const void*                                                          pNext;
1236                 storage->getAddressConst(vk,device),                                                            // VkDeviceOrHostAddressConstKHR                        src;
1237                 *(getPtr()),                                                                                                            // VkAccelerationStructureKHR                           dst;
1238                 VK_COPY_ACCELERATION_STRUCTURE_MODE_DESERIALIZE_KHR                                     // VkCopyAccelerationStructureModeKHR           mode;
1239         };
1240
1241         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1242         {
1243                 vk.cmdCopyMemoryToAccelerationStructureKHR(cmdBuffer, &copyAccelerationStructureInfo);
1244         }
1245         else if (!m_deferredOperation)
1246         {
1247                 VK_CHECK(vk.copyMemoryToAccelerationStructureKHR(device, DE_NULL, &copyAccelerationStructureInfo));
1248         }
1249         else
1250         {
1251                 const auto deferredOperationPtr = createDeferredOperationKHR(vk, device);
1252                 const auto deferredOperation    = deferredOperationPtr.get();
1253
1254                 const VkResult result = vk.copyMemoryToAccelerationStructureKHR(device, deferredOperation, &copyAccelerationStructureInfo);
1255
1256                 DE_ASSERT(result == VK_OPERATION_DEFERRED_KHR || result == VK_OPERATION_NOT_DEFERRED_KHR || result == VK_SUCCESS);
1257
1258                 finishDeferredOperation(vk, device, deferredOperation, m_workerThreadCount, result == VK_OPERATION_NOT_DEFERRED_KHR);
1259         }
1260
1261         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1262         {
1263                 const VkAccessFlags             accessMasks = VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR | VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR;
1264                 const VkMemoryBarrier   memBarrier = makeMemoryBarrier(accessMasks, accessMasks);
1265
1266                 cmdPipelineMemoryBarrier(vk, cmdBuffer, VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, &memBarrier);
1267         }
1268 }
1269
1270 const VkAccelerationStructureKHR* BottomLevelAccelerationStructureKHR::getPtr (void) const
1271 {
1272         return &m_accelerationStructureKHR.get();
1273 }
1274
1275 void BottomLevelAccelerationStructureKHR::prepareGeometries (const DeviceInterface&                                                                             vk,
1276                                                                                                                          const VkDevice                                                                                         device,
1277                                                                                                                          std::vector<VkAccelerationStructureGeometryKHR>&                       accelerationStructureGeometriesKHR,
1278                                                                                                                          std::vector<VkAccelerationStructureGeometryKHR*>&                      accelerationStructureGeometriesKHRPointers,
1279                                                                                                                          std::vector<VkAccelerationStructureBuildRangeInfoKHR>&         accelerationStructureBuildRangeInfoKHR,
1280                                                                                                                          std::vector<deUint32>&                                                                         maxPrimitiveCounts)
1281 {
1282         accelerationStructureGeometriesKHR.resize(m_geometriesData.size());
1283         accelerationStructureGeometriesKHRPointers.resize(m_geometriesData.size());
1284         accelerationStructureBuildRangeInfoKHR.resize(m_geometriesData.size());
1285         maxPrimitiveCounts.resize(m_geometriesData.size());
1286
1287         VkDeviceSize vertexBufferOffset = 0, indexBufferOffset = 0;
1288
1289         for (size_t geometryNdx = 0; geometryNdx < m_geometriesData.size(); ++geometryNdx)
1290         {
1291                 de::SharedPtr<RaytracedGeometryBase>&                                   geometryData = m_geometriesData[geometryNdx];
1292                 VkDeviceOrHostAddressConstKHR                                                   vertexData, indexData;
1293                 if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1294                 {
1295                         if (m_vertexBuffer.get() != DE_NULL)
1296                         {
1297                                 vertexData                      = makeDeviceOrHostAddressConstKHR(vk, device, m_vertexBuffer->get(), vertexBufferOffset);
1298                                 vertexBufferOffset      += deAlignSize(geometryData->getVertexByteSize(), 8);
1299                         }
1300                         else
1301                                 vertexData                      = makeDeviceOrHostAddressConstKHR(DE_NULL);
1302
1303                         if (m_indexBuffer.get() != DE_NULL &&  geometryData->getIndexType() != VK_INDEX_TYPE_NONE_KHR)
1304                         {
1305                                 indexData                       = makeDeviceOrHostAddressConstKHR(vk, device, m_indexBuffer->get(), indexBufferOffset);
1306                                 indexBufferOffset       += deAlignSize(geometryData->getIndexByteSize(), 8);
1307                         }
1308                         else
1309                                 indexData = makeDeviceOrHostAddressConstKHR(DE_NULL);
1310                 }
1311                 else
1312                 {
1313                         vertexData = makeDeviceOrHostAddressConstKHR(geometryData->getVertexPointer());
1314                         if (geometryData->getIndexType() != VK_INDEX_TYPE_NONE_KHR)
1315                                 indexData = makeDeviceOrHostAddressConstKHR(geometryData->getIndexPointer());
1316                         else
1317                                 indexData = makeDeviceOrHostAddressConstKHR(DE_NULL);
1318                 }
1319
1320                 const VkAccelerationStructureGeometryTrianglesDataKHR   accelerationStructureGeometryTrianglesDataKHR =
1321                 {
1322                         VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_TRIANGLES_DATA_KHR,   //  VkStructureType                                     sType;
1323                         DE_NULL,                                                                                                                                //  const void*                                         pNext;
1324                         geometryData->getVertexFormat(),                                                                                //  VkFormat                                            vertexFormat;
1325                         vertexData,                                                                                                                             //  VkDeviceOrHostAddressConstKHR       vertexData;
1326                         geometryData->getVertexStride(),                                                                                //  VkDeviceSize                                        vertexStride;
1327                         static_cast<deUint32>(geometryData->getVertexCount()),                                  //  uint32_t                                            maxVertex;
1328                         geometryData->getIndexType(),                                                                                   //  VkIndexType                                         indexType;
1329                         indexData,                                                                                                                              //  VkDeviceOrHostAddressConstKHR       indexData;
1330                         makeDeviceOrHostAddressConstKHR(DE_NULL),                                                               //  VkDeviceOrHostAddressConstKHR       transformData;
1331                 };
1332
1333                 const VkAccelerationStructureGeometryAabbsDataKHR               accelerationStructureGeometryAabbsDataKHR =
1334                 {
1335                         VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_AABBS_DATA_KHR,       //  VkStructureType                                     sType;
1336                         DE_NULL,                                                                                                                        //  const void*                                         pNext;
1337                         vertexData,                                                                                                                     //  VkDeviceOrHostAddressConstKHR       data;
1338                         geometryData->getAABBStride()                                                                           //  VkDeviceSize                                        stride;
1339                 };
1340                 const VkAccelerationStructureGeometryDataKHR                    geometry = (geometryData->isTrianglesType())
1341                                                                                                                                                  ? makeVkAccelerationStructureGeometryDataKHR(accelerationStructureGeometryTrianglesDataKHR)
1342                                                                                                                                                  : makeVkAccelerationStructureGeometryDataKHR(accelerationStructureGeometryAabbsDataKHR);
1343                 const VkAccelerationStructureGeometryKHR                                accelerationStructureGeometryKHR =
1344                 {
1345                         VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_KHR,  //  VkStructureType                                                     sType;
1346                         DE_NULL,                                                                                                //  const void*                                                         pNext;
1347                         geometryData->getGeometryType(),                                                //  VkGeometryTypeKHR                                           geometryType;
1348                         geometry,                                                                                               //  VkAccelerationStructureGeometryDataKHR      geometry;
1349                         geometryData->getGeometryFlags()                                                //  VkGeometryFlagsKHR                                          flags;
1350                 };
1351
1352                 const deUint32 primitiveCount = (m_buildWithoutPrimitives ? 0u : geometryData->getPrimitiveCount());
1353
1354                 const VkAccelerationStructureBuildRangeInfoKHR                  accelerationStructureBuildRangeInfosKHR =
1355                 {
1356                         primitiveCount, //  deUint32    primitiveCount;
1357                         0,                              //  deUint32    primitiveOffset;
1358                         0,                              //  deUint32    firstVertex;
1359                         0                               //  deUint32    firstTransform;
1360                 };
1361
1362                 accelerationStructureGeometriesKHR[geometryNdx]                 = accelerationStructureGeometryKHR;
1363                 accelerationStructureGeometriesKHRPointers[geometryNdx] = &accelerationStructureGeometriesKHR[geometryNdx];
1364                 accelerationStructureBuildRangeInfoKHR[geometryNdx]             = accelerationStructureBuildRangeInfosKHR;
1365                 maxPrimitiveCounts[geometryNdx]                                                 = geometryData->getPrimitiveCount();
1366         }
1367 }
1368
1369 deUint32 BottomLevelAccelerationStructure::getRequiredAllocationCount (void)
1370 {
1371         return BottomLevelAccelerationStructureKHR::getRequiredAllocationCount();
1372 }
1373
1374 void BottomLevelAccelerationStructure::createAndBuild (const DeviceInterface&   vk,
1375                                                                                                            const VkDevice                       device,
1376                                                                                                            const VkCommandBuffer        cmdBuffer,
1377                                                                                                            Allocator&                           allocator,
1378                                                                                                            VkDeviceAddress                      deviceAddress)
1379 {
1380         create(vk, device, allocator, 0u, deviceAddress);
1381         build(vk, device, cmdBuffer);
1382 }
1383
1384 void BottomLevelAccelerationStructure::createAndCopyFrom (const DeviceInterface&                                vk,
1385                                                                                                                   const VkDevice                                                device,
1386                                                                                                                   const VkCommandBuffer                                 cmdBuffer,
1387                                                                                                                   Allocator&                                                    allocator,
1388                                                                                                                   BottomLevelAccelerationStructure*             accelerationStructure,
1389                                                                                                                   VkDeviceSize                                                  compactCopySize,
1390                                                                                                                   VkDeviceAddress                                               deviceAddress)
1391 {
1392         DE_ASSERT(accelerationStructure != NULL);
1393         VkDeviceSize copiedSize = compactCopySize > 0u ? compactCopySize : accelerationStructure->getStructureSize();
1394         DE_ASSERT(copiedSize != 0u);
1395
1396         create(vk, device, allocator, copiedSize, deviceAddress);
1397         copyFrom(vk, device, cmdBuffer, accelerationStructure, compactCopySize > 0u);
1398 }
1399
1400 void BottomLevelAccelerationStructure::createAndDeserializeFrom (const DeviceInterface& vk,
1401                                                                                                                                  const VkDevice                                                         device,
1402                                                                                                                                  const VkCommandBuffer                                          cmdBuffer,
1403                                                                                                                                  Allocator&                                                                     allocator,
1404                                                                                                                                  SerialStorage*                                                         storage,
1405                                                                                                                                  VkDeviceAddress                                                        deviceAddress )
1406 {
1407         DE_ASSERT(storage != NULL);
1408         DE_ASSERT(storage->getStorageSize() >= SerialStorage::SERIAL_STORAGE_SIZE_MIN);
1409         create(vk, device, allocator, storage->getDeserializedSize(), deviceAddress);
1410         deserialize(vk, device, cmdBuffer, storage);
1411 }
1412
1413 de::MovePtr<BottomLevelAccelerationStructure> makeBottomLevelAccelerationStructure ()
1414 {
1415         return de::MovePtr<BottomLevelAccelerationStructure>(new BottomLevelAccelerationStructureKHR);
1416 }
1417
1418 TopLevelAccelerationStructure::~TopLevelAccelerationStructure ()
1419 {
1420 }
1421
1422 TopLevelAccelerationStructure::TopLevelAccelerationStructure ()
1423         : m_structureSize               (0u)
1424         , m_updateScratchSize   (0u)
1425         , m_buildScratchSize    (0u)
1426 {
1427 }
1428
1429 void TopLevelAccelerationStructure::setInstanceCount (const size_t instanceCount)
1430 {
1431         m_bottomLevelInstances.reserve(instanceCount);
1432         m_instanceData.reserve(instanceCount);
1433 }
1434
1435 void TopLevelAccelerationStructure::addInstance (de::SharedPtr<BottomLevelAccelerationStructure>        bottomLevelStructure,
1436                                                                                                  const VkTransformMatrixKHR&                                            matrix,
1437                                                                                                  deUint32                                                                                       instanceCustomIndex,
1438                                                                                                  deUint32                                                                                       mask,
1439                                                                                                  deUint32                                                                                       instanceShaderBindingTableRecordOffset,
1440                                                                                                  VkGeometryInstanceFlagsKHR                                                     flags)
1441 {
1442         m_bottomLevelInstances.push_back(bottomLevelStructure);
1443         m_instanceData.push_back(InstanceData(matrix, instanceCustomIndex, mask, instanceShaderBindingTableRecordOffset, flags));
1444 }
1445
1446 VkDeviceSize TopLevelAccelerationStructure::getStructureSize () const
1447 {
1448         return m_structureSize;
1449 }
1450
1451 void TopLevelAccelerationStructure::createAndBuild (const DeviceInterface&      vk,
1452                                                                                                         const VkDevice                  device,
1453                                                                                                         const VkCommandBuffer   cmdBuffer,
1454                                                                                                         Allocator&                              allocator,
1455                                                                                                         VkDeviceAddress                 deviceAddress)
1456 {
1457         create(vk, device, allocator, 0u, deviceAddress);
1458         build(vk, device, cmdBuffer);
1459 }
1460
1461 void TopLevelAccelerationStructure::createAndCopyFrom (const DeviceInterface&                           vk,
1462                                                                                                            const VkDevice                                               device,
1463                                                                                                            const VkCommandBuffer                                cmdBuffer,
1464                                                                                                            Allocator&                                                   allocator,
1465                                                                                                            TopLevelAccelerationStructure*               accelerationStructure,
1466                                                                                                            VkDeviceSize                                                 compactCopySize,
1467                                                                                                            VkDeviceAddress                                              deviceAddress)
1468 {
1469         DE_ASSERT(accelerationStructure != NULL);
1470         VkDeviceSize copiedSize = compactCopySize > 0u ? compactCopySize : accelerationStructure->getStructureSize();
1471         DE_ASSERT(copiedSize != 0u);
1472
1473         create(vk, device, allocator, copiedSize, deviceAddress);
1474         copyFrom(vk, device, cmdBuffer, accelerationStructure, compactCopySize > 0u);
1475 }
1476
1477 void TopLevelAccelerationStructure::createAndDeserializeFrom (const DeviceInterface& vk,
1478                                                                                                                           const VkDevice                                                        device,
1479                                                                                                                           const VkCommandBuffer                                         cmdBuffer,
1480                                                                                                                           Allocator&                                                            allocator,
1481                                                                                                                           SerialStorage*                                                        storage,
1482                                                                                                                           VkDeviceAddress                                                       deviceAddress)
1483 {
1484         DE_ASSERT(storage != NULL);
1485         DE_ASSERT(storage->getStorageSize() >= SerialStorage::SERIAL_STORAGE_SIZE_MIN);
1486         create(vk, device, allocator, storage->getDeserializedSize(), deviceAddress);
1487         deserialize(vk, device, cmdBuffer, storage);
1488 }
1489
1490 BufferWithMemory* createInstanceBuffer (const DeviceInterface&                                                                                  vk,
1491                                                                                 const VkDevice                                                                                                  device,
1492                                                                                 Allocator&                                                                                                              allocator,
1493                                                                                 std::vector<de::SharedPtr<BottomLevelAccelerationStructure> >   bottomLevelInstances,
1494                                                                                 std::vector<InstanceData>                                                                               instanceData)
1495 {
1496         DE_ASSERT(bottomLevelInstances.size() != 0);
1497         DE_ASSERT(bottomLevelInstances.size() == instanceData.size());
1498         DE_UNREF(instanceData);
1499
1500         const VkDeviceSize                      bufferSizeBytes         = bottomLevelInstances.size() * sizeof(VkAccelerationStructureInstanceKHR);
1501         const VkBufferCreateInfo        bufferCreateInfo        = makeBufferCreateInfo(bufferSizeBytes, VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT_KHR | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
1502         try
1503         {
1504                 return new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::Cached | MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress);
1505         }
1506         catch (const tcu::NotSupportedError&)
1507         {
1508                 // retry without Cached flag
1509                 return new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress);
1510         }
1511 }
1512
1513 void updateInstanceBuffer (const DeviceInterface&                                                                                       vk,
1514                                                    const VkDevice                                                                                                       device,
1515                                                    std::vector<de::SharedPtr<BottomLevelAccelerationStructure> >        bottomLevelInstances,
1516                                                    std::vector<InstanceData>                                                                            instanceData,
1517                                                    BufferWithMemory*                                                                                            instanceBuffer,
1518                                                    VkAccelerationStructureBuildTypeKHR                                                          buildType,
1519                                                    bool                                                                                                                         inactiveInstances)
1520 {
1521         DE_ASSERT(bottomLevelInstances.size() != 0);
1522         DE_ASSERT(bottomLevelInstances.size() == instanceData.size());
1523
1524         const Allocation&                       instancesAlloc          = instanceBuffer->getAllocation();
1525
1526         deUint8*                                        bufferStart                     = static_cast<deUint8*>(instancesAlloc.getHostPtr());
1527         VkDeviceSize                            bufferOffset            = 0;
1528
1529         for (size_t instanceNdx = 0; instanceNdx < bottomLevelInstances.size(); ++instanceNdx)
1530         {
1531                 const BottomLevelAccelerationStructure&         bottomLevelAccelerationStructure        = *bottomLevelInstances[instanceNdx];
1532                 const VkAccelerationStructureKHR                        accelerationStructureKHR                        = *bottomLevelAccelerationStructure.getPtr();
1533
1534                 // This part needs to be fixed once a new version of the VkAccelerationStructureInstanceKHR will be added to vkStructTypes.inl
1535                 VkDeviceAddress accelerationStructureAddress;
1536                 if (buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1537                 {
1538                         VkAccelerationStructureDeviceAddressInfoKHR asDeviceAddressInfo =
1539                         {
1540                                 VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR,       // VkStructureType                              sType;
1541                                 DE_NULL,                                                                                                                        // const void*                                  pNext;
1542                                 accelerationStructureKHR                                                                                        // VkAccelerationStructureKHR   accelerationStructure;
1543                         };
1544                         accelerationStructureAddress = vk.getAccelerationStructureDeviceAddressKHR(device, &asDeviceAddressInfo);
1545                 }
1546
1547                 deUint64 structureReference;
1548                 if (inactiveInstances)
1549                 {
1550                         // Instances will be marked inactive by making their references VK_NULL_HANDLE or having address zero.
1551                         structureReference = 0ull;
1552                 }
1553                 else
1554                 {
1555                         structureReference      = (buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1556                                                                 ? deUint64(accelerationStructureAddress)
1557                                                                 : deUint64(accelerationStructureKHR.getInternal());
1558                 }
1559
1560                 VkAccelerationStructureInstanceKHR      accelerationStructureInstanceKHR = makeVkAccelerationStructureInstanceKHR
1561                 (
1562                         instanceData[instanceNdx].matrix,                                                                                                       //  VkTransformMatrixKHR                transform;
1563                         instanceData[instanceNdx].instanceCustomIndex,                                                                          //  deUint32                                    instanceCustomIndex:24;
1564                         instanceData[instanceNdx].mask,                                                                                                         //  deUint32                                    mask:8;
1565                         instanceData[instanceNdx].instanceShaderBindingTableRecordOffset,                                       //  deUint32                                    instanceShaderBindingTableRecordOffset:24;
1566                         instanceData[instanceNdx].flags,                                                                                                        //  VkGeometryInstanceFlagsKHR  flags:8;
1567                         structureReference                                                                                                                                      //  deUint64                                    accelerationStructureReference;
1568                 );
1569
1570                 deMemcpy(&bufferStart[bufferOffset], &accelerationStructureInstanceKHR, sizeof(VkAccelerationStructureInstanceKHR));
1571
1572                 bufferOffset += sizeof(VkAccelerationStructureInstanceKHR);
1573         }
1574
1575         flushMappedMemoryRange(vk, device, instancesAlloc.getMemory(), instancesAlloc.getOffset(), VK_WHOLE_SIZE);
1576 }
1577
1578 class TopLevelAccelerationStructureKHR : public TopLevelAccelerationStructure
1579 {
1580 public:
1581         static deUint32                                                                                 getRequiredAllocationCount                                                      (void);
1582
1583                                                                                                                         TopLevelAccelerationStructureKHR                                        ();
1584                                                                                                                         TopLevelAccelerationStructureKHR                                        (const TopLevelAccelerationStructureKHR&                other) = delete;
1585         virtual                                                                                                 ~TopLevelAccelerationStructureKHR                                       ();
1586
1587         void                                                                                                    setBuildType                                                                            (const VkAccelerationStructureBuildTypeKHR              buildType) override;
1588         void                                                                                                    setCreateFlags                                                                          (const VkAccelerationStructureCreateFlagsKHR    createFlags) override;
1589         void                                                                                                    setCreateGeneric                                                                        (bool                                                                                   createGeneric) override;
1590         void                                                                                                    setBuildFlags                                                                           (const VkBuildAccelerationStructureFlagsKHR             buildFlags) override;
1591         void                                                                                                    setBuildWithoutPrimitives                                                       (bool                                                                                   buildWithoutPrimitives) override;
1592         void                                                                                                    setInactiveInstances                                                            (bool                                                                                   inactiveInstances) override;
1593         void                                                                                                    setDeferredOperation                                                            (const bool                                                                             deferredOperation,
1594                                                                                                                                                                                                                                  const deUint32                                                                 workerThreadCount) override;
1595         void                                                                                                    setUseArrayOfPointers                                                           (const bool                                                                             useArrayOfPointers) override;
1596         void                                                                                                    setIndirectBuildParameters                                                      (const VkBuffer                                                                 indirectBuffer,
1597                                                                                                                                                                                                                                  const VkDeviceSize                                                             indirectBufferOffset,
1598                                                                                                                                                                                                                                  const deUint32                                                                 indirectBufferStride) override;
1599         VkBuildAccelerationStructureFlagsKHR                                    getBuildFlags                                                                           () const override;
1600
1601         void                                                                                                    create                                                                                          (const DeviceInterface&                                                 vk,
1602                                                                                                                                                                                                                                  const VkDevice                                                                 device,
1603                                                                                                                                                                                                                                  Allocator&                                                                             allocator,
1604                                                                                                                                                                                                                                  VkDeviceSize                                                                   structureSize,
1605                                                                                                                                                                                                                                  VkDeviceAddress                                                                deviceAddress = 0u ) override;
1606         void                                                                                                    build                                                                                           (const DeviceInterface&                                                 vk,
1607                                                                                                                                                                                                                                  const VkDevice                                                                 device,
1608                                                                                                                                                                                                                                  const VkCommandBuffer                                                  cmdBuffer) override;
1609         void                                                                                                    copyFrom                                                                                        (const DeviceInterface&                                                 vk,
1610                                                                                                                                                                                                                                  const VkDevice                                                                 device,
1611                                                                                                                                                                                                                                  const VkCommandBuffer                                                  cmdBuffer,
1612                                                                                                                                                                                                                                  TopLevelAccelerationStructure*                                 accelerationStructure,
1613                                                                                                                                                                                                                                  bool                                                                                   compactCopy) override;
1614         void                                                                                                    serialize                                                                                       (const DeviceInterface&                                                 vk,
1615                                                                                                                                                                                                                                  const VkDevice                                                                 device,
1616                                                                                                                                                                                                                                  const VkCommandBuffer                                                  cmdBuffer,
1617                                                                                                                                                                                                                                  SerialStorage*                                                                 storage) override;
1618         void                                                                                                    deserialize                                                                                     (const DeviceInterface&                                                 vk,
1619                                                                                                                                                                                                                                  const VkDevice                                                                 device,
1620                                                                                                                                                                                                                                  const VkCommandBuffer                                                  cmdBuffer,
1621                                                                                                                                                                                                                                  SerialStorage*                                                                 storage) override;
1622
1623         const VkAccelerationStructureKHR*                                               getPtr                                                                                          (void) const override;
1624
1625 protected:
1626         VkAccelerationStructureBuildTypeKHR                                             m_buildType;
1627         VkAccelerationStructureCreateFlagsKHR                                   m_createFlags;
1628         bool                                                                                                    m_createGeneric;
1629         VkBuildAccelerationStructureFlagsKHR                                    m_buildFlags;
1630         bool                                                                                                    m_buildWithoutPrimitives;
1631         bool                                                                                                    m_inactiveInstances;
1632         bool                                                                                                    m_deferredOperation;
1633         deUint32                                                                                                m_workerThreadCount;
1634         bool                                                                                                    m_useArrayOfPointers;
1635         de::MovePtr<BufferWithMemory>                                                   m_accelerationStructureBuffer;
1636         de::MovePtr<BufferWithMemory>                                                   m_instanceBuffer;
1637         de::MovePtr<BufferWithMemory>                                                   m_instanceAddressBuffer;
1638         de::MovePtr<BufferWithMemory>                                                   m_deviceScratchBuffer;
1639         std::vector<deUint8>                                                                    m_hostScratchBuffer;
1640         Move<VkAccelerationStructureKHR>                                                m_accelerationStructureKHR;
1641         VkBuffer                                                                                                m_indirectBuffer;
1642         VkDeviceSize                                                                                    m_indirectBufferOffset;
1643         deUint32                                                                                                m_indirectBufferStride;
1644
1645         void                                                                                                    prepareInstances                                                                        (const DeviceInterface&                                                 vk,
1646                                                                                                                                                                                                                                  const VkDevice                                                                 device,
1647                                                                                                                                                                                                                                  VkAccelerationStructureGeometryKHR&                    accelerationStructureGeometryKHR,
1648                                                                                                                                                                                                                                  std::vector<deUint32>&                                                 maxPrimitiveCounts);
1649 };
1650
1651 deUint32 TopLevelAccelerationStructureKHR::getRequiredAllocationCount (void)
1652 {
1653         /*
1654                 de::MovePtr<BufferWithMemory>                                                   m_instanceBuffer;
1655                 de::MovePtr<Allocation>                                                                 m_accelerationStructureAlloc;
1656                 de::MovePtr<BufferWithMemory>                                                   m_deviceScratchBuffer;
1657         */
1658         return 3u;
1659 }
1660
1661 TopLevelAccelerationStructureKHR::TopLevelAccelerationStructureKHR ()
1662         : TopLevelAccelerationStructure ()
1663         , m_buildType                                   (VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1664         , m_createFlags                                 (0u)
1665         , m_createGeneric                               (false)
1666         , m_buildFlags                                  (0u)
1667         , m_buildWithoutPrimitives              (false)
1668         , m_inactiveInstances                   (false)
1669         , m_deferredOperation                   (false)
1670         , m_workerThreadCount                   (0)
1671         , m_useArrayOfPointers                  (false)
1672         , m_accelerationStructureBuffer (DE_NULL)
1673         , m_instanceBuffer                              (DE_NULL)
1674         , m_instanceAddressBuffer               (DE_NULL)
1675         , m_deviceScratchBuffer                 (DE_NULL)
1676         , m_accelerationStructureKHR    ()
1677         , m_indirectBuffer                              (DE_NULL)
1678         , m_indirectBufferOffset                (0)
1679         , m_indirectBufferStride                (0)
1680 {
1681 }
1682
1683 TopLevelAccelerationStructureKHR::~TopLevelAccelerationStructureKHR ()
1684 {
1685 }
1686
1687 void TopLevelAccelerationStructureKHR::setBuildType (const VkAccelerationStructureBuildTypeKHR  buildType)
1688 {
1689         m_buildType = buildType;
1690 }
1691
1692 void TopLevelAccelerationStructureKHR::setCreateFlags (const VkAccelerationStructureCreateFlagsKHR      createFlags)
1693 {
1694         m_createFlags = createFlags;
1695 }
1696
1697 void TopLevelAccelerationStructureKHR::setCreateGeneric (bool createGeneric)
1698 {
1699         m_createGeneric = createGeneric;
1700 }
1701
1702 void TopLevelAccelerationStructureKHR::setInactiveInstances (bool inactiveInstances)
1703 {
1704         m_inactiveInstances = inactiveInstances;
1705 }
1706
1707 void TopLevelAccelerationStructureKHR::setBuildFlags (const VkBuildAccelerationStructureFlagsKHR        buildFlags)
1708 {
1709         m_buildFlags = buildFlags;
1710 }
1711
1712 void TopLevelAccelerationStructureKHR::setBuildWithoutPrimitives (bool buildWithoutPrimitives)
1713 {
1714         m_buildWithoutPrimitives = buildWithoutPrimitives;
1715 }
1716
1717 void TopLevelAccelerationStructureKHR::setDeferredOperation (const bool         deferredOperation,
1718                                                                                                                          const deUint32 workerThreadCount)
1719 {
1720         m_deferredOperation = deferredOperation;
1721         m_workerThreadCount = workerThreadCount;
1722 }
1723
1724 void TopLevelAccelerationStructureKHR::setUseArrayOfPointers (const bool        useArrayOfPointers)
1725 {
1726         m_useArrayOfPointers = useArrayOfPointers;
1727 }
1728
1729 void TopLevelAccelerationStructureKHR::setIndirectBuildParameters (const VkBuffer               indirectBuffer,
1730                                                                                                                                    const VkDeviceSize   indirectBufferOffset,
1731                                                                                                                                    const deUint32               indirectBufferStride)
1732 {
1733         m_indirectBuffer                = indirectBuffer;
1734         m_indirectBufferOffset  = indirectBufferOffset;
1735         m_indirectBufferStride  = indirectBufferStride;
1736 }
1737
1738 VkBuildAccelerationStructureFlagsKHR TopLevelAccelerationStructureKHR::getBuildFlags () const
1739 {
1740         return m_buildFlags;
1741 }
1742
1743 void TopLevelAccelerationStructureKHR::create (const DeviceInterface&                           vk,
1744                                                                                            const VkDevice                                               device,
1745                                                                                            Allocator&                                                   allocator,
1746                                                                                            VkDeviceSize                                                 structureSize,
1747                                                                                            VkDeviceAddress                                              deviceAddress)
1748 {
1749         // AS may be built from geometries using vkCmdBuildAccelerationStructureKHR / vkBuildAccelerationStructureKHR
1750         // or may be copied/compacted/deserialized from other AS ( in this case AS does not need geometries, but it needs to know its size before creation ).
1751         DE_ASSERT(!m_bottomLevelInstances.empty() != !(structureSize == 0)); // logical xor
1752
1753         if (structureSize == 0)
1754         {
1755                 VkAccelerationStructureGeometryKHR              accelerationStructureGeometryKHR;
1756                 std::vector<deUint32>                                   maxPrimitiveCounts;
1757                 prepareInstances(vk, device, accelerationStructureGeometryKHR, maxPrimitiveCounts);
1758
1759                 VkAccelerationStructureBuildGeometryInfoKHR accelerationStructureBuildGeometryInfoKHR           =
1760                 {
1761                         VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_GEOMETRY_INFO_KHR,                                               //  VkStructureType                                                                             sType;
1762                         DE_NULL,                                                                                                                                                                //  const void*                                                                                 pNext;
1763                         VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR,                                                                                   //  VkAccelerationStructureTypeKHR                                              type;
1764                         m_buildFlags,                                                                                                                                                   //  VkBuildAccelerationStructureFlagsKHR                                flags;
1765                         VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR,                                                                                 //  VkBuildAccelerationStructureModeKHR                                 mode;
1766                         DE_NULL,                                                                                                                                                                //  VkAccelerationStructureKHR                                                  srcAccelerationStructure;
1767                         DE_NULL,                                                                                                                                                                //  VkAccelerationStructureKHR                                                  dstAccelerationStructure;
1768                         1u,                                                                                                                                                                             //  deUint32                                                                                    geometryCount;
1769                         &accelerationStructureGeometryKHR,                                                                                                              //  const VkAccelerationStructureGeometryKHR*                   pGeometries;
1770                         DE_NULL,                                                                                                                                                                //  const VkAccelerationStructureGeometryKHR* const*    ppGeometries;
1771                         makeDeviceOrHostAddressKHR(DE_NULL)                                                                                                             //  VkDeviceOrHostAddressKHR                                                    scratchData;
1772                 };
1773
1774                 VkAccelerationStructureBuildSizesInfoKHR        sizeInfo =
1775                 {
1776                         VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_SIZES_INFO_KHR,  //  VkStructureType     sType;
1777                         DE_NULL,                                                                                                                //  const void*         pNext;
1778                         0,                                                                                                                              //  VkDeviceSize        accelerationStructureSize;
1779                         0,                                                                                                                              //  VkDeviceSize        updateScratchSize;
1780                         0                                                                                                                               //  VkDeviceSize        buildScratchSize;
1781                 };
1782
1783                 vk.getAccelerationStructureBuildSizesKHR(device, m_buildType, &accelerationStructureBuildGeometryInfoKHR, maxPrimitiveCounts.data(), &sizeInfo);
1784
1785                 m_structureSize         = sizeInfo.accelerationStructureSize;
1786                 m_updateScratchSize     = sizeInfo.updateScratchSize;
1787                 m_buildScratchSize      = sizeInfo.buildScratchSize;
1788         }
1789         else
1790         {
1791                 m_structureSize         = structureSize;
1792                 m_updateScratchSize     = 0u;
1793                 m_buildScratchSize      = 0u;
1794         }
1795
1796         {
1797                 const VkBufferCreateInfo                bufferCreateInfo = makeBufferCreateInfo(m_structureSize, VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
1798                 try
1799                 {
1800                         m_accelerationStructureBuffer = de::MovePtr<BufferWithMemory>(new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::Cached | MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress));
1801                 }
1802                 catch (const tcu::NotSupportedError&)
1803                 {
1804                         // retry without Cached flag
1805                         m_accelerationStructureBuffer = de::MovePtr<BufferWithMemory>(new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress));
1806                 }
1807         }
1808
1809         {
1810                 const VkAccelerationStructureTypeKHR            structureType                                           = (m_createGeneric
1811                                                                                                                                                                                    ? VK_ACCELERATION_STRUCTURE_TYPE_GENERIC_KHR
1812                                                                                                                                                                                    : VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR);
1813                 const VkAccelerationStructureCreateInfoKHR      accelerationStructureCreateInfoKHR      =
1814                 {
1815                         VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR,       //  VkStructureType                                                                                     sType;
1816                         DE_NULL,                                                                                                        //  const void*                                                                                         pNext;
1817                         m_createFlags,                                                                                          //  VkAccelerationStructureCreateFlagsKHR                                       createFlags;
1818                         m_accelerationStructureBuffer->get(),                                           //  VkBuffer                                                                                            buffer;
1819                         0u,                                                                                                                     //  VkDeviceSize                                                                                        offset;
1820                         m_structureSize,                                                                                        //  VkDeviceSize                                                                                        size;
1821                         structureType,                                                                                          //  VkAccelerationStructureTypeKHR                                                      type;
1822                         deviceAddress                                                                                           //  VkDeviceAddress                                                                                     deviceAddress;
1823                 };
1824
1825                 m_accelerationStructureKHR      = createAccelerationStructureKHR(vk, device, &accelerationStructureCreateInfoKHR, DE_NULL);
1826         }
1827
1828         if (m_buildScratchSize > 0u)
1829         {
1830                 if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1831                 {
1832                         const VkBufferCreateInfo                bufferCreateInfo        = makeBufferCreateInfo(m_buildScratchSize, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
1833                         m_deviceScratchBuffer                                                           = de::MovePtr<BufferWithMemory>(new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress));
1834                 }
1835                 else
1836                 {
1837                         m_hostScratchBuffer.resize(static_cast<size_t>(m_buildScratchSize));
1838                 }
1839         }
1840
1841         if (m_useArrayOfPointers)
1842         {
1843                 const size_t                            pointerSize = (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR) ? sizeof(VkDeviceOrHostAddressConstKHR::deviceAddress) : sizeof(VkDeviceOrHostAddressConstKHR::hostAddress);
1844                 const VkBufferCreateInfo        bufferCreateInfo = makeBufferCreateInfo(static_cast<VkDeviceSize>(m_bottomLevelInstances.size() * pointerSize), VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT_KHR | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT);
1845                 m_instanceAddressBuffer = de::MovePtr<BufferWithMemory>(new BufferWithMemory(vk, device, allocator, bufferCreateInfo, MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress));
1846         }
1847
1848         if(!m_bottomLevelInstances.empty())
1849                 m_instanceBuffer = de::MovePtr<BufferWithMemory>(createInstanceBuffer(vk, device, allocator, m_bottomLevelInstances, m_instanceData));
1850 }
1851
1852 void TopLevelAccelerationStructureKHR::build (const DeviceInterface&    vk,
1853                                                                                           const VkDevice                        device,
1854                                                                                           const VkCommandBuffer         cmdBuffer)
1855 {
1856         DE_ASSERT(!m_bottomLevelInstances.empty());
1857         DE_ASSERT(m_accelerationStructureKHR.get() != DE_NULL);
1858         DE_ASSERT(m_buildScratchSize != 0);
1859
1860         updateInstanceBuffer(vk, device, m_bottomLevelInstances, m_instanceData, m_instanceBuffer.get(), m_buildType, m_inactiveInstances);
1861
1862         VkAccelerationStructureGeometryKHR              accelerationStructureGeometryKHR;
1863         std::vector<deUint32>                                   maxPrimitiveCounts;
1864         prepareInstances(vk, device, accelerationStructureGeometryKHR, maxPrimitiveCounts);
1865
1866         VkDeviceOrHostAddressKHR                                scratchData                                                                             = (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1867                                                                                                                                                                                         ? makeDeviceOrHostAddressKHR(vk, device, m_deviceScratchBuffer->get(), 0)
1868                                                                                                                                                                                         : makeDeviceOrHostAddressKHR(m_hostScratchBuffer.data());
1869
1870         VkAccelerationStructureBuildGeometryInfoKHR accelerationStructureBuildGeometryInfoKHR           =
1871         {
1872                 VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_GEOMETRY_INFO_KHR,                                               //  VkStructureType                                                                             sType;
1873                 DE_NULL,                                                                                                                                                                //  const void*                                                                                 pNext;
1874                 VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR,                                                                                   //  VkAccelerationStructureTypeKHR                                              type;
1875                 m_buildFlags,                                                                                                                                                   //  VkBuildAccelerationStructureFlagsKHR                                flags;
1876                 VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR,                                                                                 //  VkBuildAccelerationStructureModeKHR                                 mode;
1877                 DE_NULL,                                                                                                                                                                //  VkAccelerationStructureKHR                                                  srcAccelerationStructure;
1878                 m_accelerationStructureKHR.get(),                                                                                                               //  VkAccelerationStructureKHR                                                  dstAccelerationStructure;
1879                 1u,                                                                                                                                                                             //  deUint32                                                                                    geometryCount;
1880                 &accelerationStructureGeometryKHR,                                                                                                              //  const VkAccelerationStructureGeometryKHR*                   pGeometries;
1881                 DE_NULL,                                                                                                                                                                //  const VkAccelerationStructureGeometryKHR* const*    ppGeometries;
1882                 scratchData                                                                                                                                                             //  VkDeviceOrHostAddressKHR                                                    scratchData;
1883         };
1884
1885         const deUint32 primitiveCount = (m_buildWithoutPrimitives ? 0u : static_cast<deUint32>(m_bottomLevelInstances.size()));
1886
1887         VkAccelerationStructureBuildRangeInfoKHR accelerationStructureBuildRangeInfoKHR =
1888         {
1889                 primitiveCount, //  deUint32    primitiveCount;
1890                 0,                              //  deUint32    primitiveOffset;
1891                 0,                              //  deUint32    firstVertex;
1892                 0                               //  deUint32    transformOffset;
1893         };
1894         VkAccelerationStructureBuildRangeInfoKHR* accelerationStructureBuildRangeInfoKHRPtr     = &accelerationStructureBuildRangeInfoKHR;
1895
1896         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1897         {
1898                 if (m_indirectBuffer == DE_NULL)
1899                         vk.cmdBuildAccelerationStructuresKHR(cmdBuffer, 1u, &accelerationStructureBuildGeometryInfoKHR, (const VkAccelerationStructureBuildRangeInfoKHR**)&accelerationStructureBuildRangeInfoKHRPtr);
1900                 else
1901                 {
1902                         VkDeviceAddress indirectDeviceAddress = getBufferDeviceAddress(vk, device, m_indirectBuffer, m_indirectBufferOffset);
1903                         deUint32*               pMaxPrimitiveCounts = maxPrimitiveCounts.data();
1904                         vk.cmdBuildAccelerationStructuresIndirectKHR(cmdBuffer, 1u, &accelerationStructureBuildGeometryInfoKHR, &indirectDeviceAddress, &m_indirectBufferStride, &pMaxPrimitiveCounts);
1905                 }
1906         }
1907         else if (!m_deferredOperation)
1908         {
1909                 VK_CHECK(vk.buildAccelerationStructuresKHR(device, DE_NULL, 1u, &accelerationStructureBuildGeometryInfoKHR, (const VkAccelerationStructureBuildRangeInfoKHR**)&accelerationStructureBuildRangeInfoKHRPtr));
1910         }
1911         else
1912         {
1913                 const auto deferredOperationPtr = createDeferredOperationKHR(vk, device);
1914                 const auto deferredOperation    = deferredOperationPtr.get();
1915
1916                 VkResult result = vk.buildAccelerationStructuresKHR(device, deferredOperation, 1u, &accelerationStructureBuildGeometryInfoKHR, (const VkAccelerationStructureBuildRangeInfoKHR**)&accelerationStructureBuildRangeInfoKHRPtr);
1917
1918                 DE_ASSERT(result == VK_OPERATION_DEFERRED_KHR || result == VK_OPERATION_NOT_DEFERRED_KHR || result == VK_SUCCESS);
1919
1920                 finishDeferredOperation(vk, device, deferredOperation, m_workerThreadCount, result == VK_OPERATION_NOT_DEFERRED_KHR);
1921
1922                 accelerationStructureBuildGeometryInfoKHR.pNext = DE_NULL;
1923         }
1924
1925         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1926         {
1927                 const VkAccessFlags             accessMasks     = VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR | VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR;
1928                 const VkMemoryBarrier   memBarrier      = makeMemoryBarrier(accessMasks, accessMasks);
1929
1930                 cmdPipelineMemoryBarrier(vk, cmdBuffer, VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, &memBarrier);
1931         }
1932 }
1933
1934 void TopLevelAccelerationStructureKHR::copyFrom (const DeviceInterface&                         vk,
1935                                                                                                  const VkDevice                                         device,
1936                                                                                                  const VkCommandBuffer                          cmdBuffer,
1937                                                                                                  TopLevelAccelerationStructure*         accelerationStructure,
1938                                                                                                  bool                                                           compactCopy)
1939 {
1940         DE_ASSERT(m_accelerationStructureKHR.get() != DE_NULL);
1941         DE_ASSERT(accelerationStructure != DE_NULL);
1942
1943         VkCopyAccelerationStructureInfoKHR copyAccelerationStructureInfo =
1944         {
1945                 VK_STRUCTURE_TYPE_COPY_ACCELERATION_STRUCTURE_INFO_KHR,                                                                                                                 // VkStructureType                                              sType;
1946                 DE_NULL,                                                                                                                                                                                                                // const void*                                                  pNext;
1947                 *(accelerationStructure->getPtr()),                                                                                                                                                             // VkAccelerationStructureKHR                   src;
1948                 *(getPtr()),                                                                                                                                                                                                    // VkAccelerationStructureKHR                   dst;
1949                 compactCopy ? VK_COPY_ACCELERATION_STRUCTURE_MODE_COMPACT_KHR : VK_COPY_ACCELERATION_STRUCTURE_MODE_CLONE_KHR   // VkCopyAccelerationStructureModeKHR   mode;
1950         };
1951
1952         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1953         {
1954                 vk.cmdCopyAccelerationStructureKHR(cmdBuffer, &copyAccelerationStructureInfo);
1955         }
1956         else if (!m_deferredOperation)
1957         {
1958                 VK_CHECK(vk.copyAccelerationStructureKHR(device, DE_NULL, &copyAccelerationStructureInfo));
1959         }
1960         else
1961         {
1962                 const auto deferredOperationPtr = createDeferredOperationKHR(vk, device);
1963                 const auto deferredOperation    = deferredOperationPtr.get();
1964
1965                 VkResult result = vk.copyAccelerationStructureKHR(device, deferredOperation, &copyAccelerationStructureInfo);
1966
1967                 DE_ASSERT(result == VK_OPERATION_DEFERRED_KHR || result == VK_OPERATION_NOT_DEFERRED_KHR || result == VK_SUCCESS);
1968
1969                 finishDeferredOperation(vk, device, deferredOperation, m_workerThreadCount, result == VK_OPERATION_NOT_DEFERRED_KHR);
1970         }
1971
1972         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
1973         {
1974                 const VkAccessFlags             accessMasks     = VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR | VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR;
1975                 const VkMemoryBarrier   memBarrier      = makeMemoryBarrier(accessMasks, accessMasks);
1976
1977                 cmdPipelineMemoryBarrier(vk, cmdBuffer, VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, &memBarrier);
1978         }
1979
1980 }
1981
1982 void TopLevelAccelerationStructureKHR::serialize (const DeviceInterface&        vk,
1983                                                                                                   const VkDevice                        device,
1984                                                                                                   const VkCommandBuffer         cmdBuffer,
1985                                                                                                   SerialStorage*                        storage)
1986 {
1987         DE_ASSERT(m_accelerationStructureKHR.get() != DE_NULL);
1988         DE_ASSERT(storage != DE_NULL);
1989
1990         const VkCopyAccelerationStructureToMemoryInfoKHR        copyAccelerationStructureInfo   =
1991         {
1992                 VK_STRUCTURE_TYPE_COPY_ACCELERATION_STRUCTURE_TO_MEMORY_INFO_KHR,       // VkStructureType                                              sType;
1993                 DE_NULL,                                                                                                                        // const void*                                                  pNext;
1994                 *(getPtr()),                                                                                                            // VkAccelerationStructureKHR                   src;
1995                 storage->getAddress(vk,device),                                                                         // VkDeviceOrHostAddressKHR                             dst;
1996                 VK_COPY_ACCELERATION_STRUCTURE_MODE_SERIALIZE_KHR                                       // VkCopyAccelerationStructureModeKHR   mode;
1997         };
1998
1999         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
2000         {
2001                 vk.cmdCopyAccelerationStructureToMemoryKHR(cmdBuffer, &copyAccelerationStructureInfo);
2002         }
2003         else if (!m_deferredOperation)
2004         {
2005                 VK_CHECK(vk.copyAccelerationStructureToMemoryKHR(device, DE_NULL, &copyAccelerationStructureInfo));
2006         }
2007         else
2008         {
2009                 const auto deferredOperationPtr = createDeferredOperationKHR(vk, device);
2010                 const auto deferredOperation    = deferredOperationPtr.get();
2011
2012                 const VkResult result = vk.copyAccelerationStructureToMemoryKHR(device, deferredOperation, &copyAccelerationStructureInfo);
2013
2014                 DE_ASSERT(result == VK_OPERATION_DEFERRED_KHR || result == VK_OPERATION_NOT_DEFERRED_KHR || result == VK_SUCCESS);
2015
2016                 finishDeferredOperation(vk, device, deferredOperation, m_workerThreadCount, result == VK_OPERATION_NOT_DEFERRED_KHR);
2017         }
2018 }
2019
2020 void TopLevelAccelerationStructureKHR::deserialize (const DeviceInterface&      vk,
2021                                                                                                         const VkDevice                  device,
2022                                                                                                         const VkCommandBuffer   cmdBuffer,
2023                                                                                                         SerialStorage*                  storage)
2024 {
2025         DE_ASSERT(m_accelerationStructureKHR.get() != DE_NULL);
2026         DE_ASSERT(storage != DE_NULL);
2027
2028         const VkCopyMemoryToAccelerationStructureInfoKHR        copyAccelerationStructureInfo   =
2029         {
2030                 VK_STRUCTURE_TYPE_COPY_MEMORY_TO_ACCELERATION_STRUCTURE_INFO_KHR,       // VkStructureType                                                      sType;
2031                 DE_NULL,                                                                                                                        // const void*                                                          pNext;
2032                 storage->getAddressConst(vk,device),                                                            // VkDeviceOrHostAddressConstKHR                        src;
2033                 *(getPtr()),                                                                                                            // VkAccelerationStructureKHR                           dst;
2034                 VK_COPY_ACCELERATION_STRUCTURE_MODE_DESERIALIZE_KHR                                     // VkCopyAccelerationStructureModeKHR           mode;
2035         };
2036
2037         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
2038         {
2039                 vk.cmdCopyMemoryToAccelerationStructureKHR(cmdBuffer, &copyAccelerationStructureInfo);
2040         }
2041         else if (!m_deferredOperation)
2042         {
2043                 VK_CHECK(vk.copyMemoryToAccelerationStructureKHR(device, DE_NULL, &copyAccelerationStructureInfo));
2044         }
2045         else
2046         {
2047                 const auto deferredOperationPtr = createDeferredOperationKHR(vk, device);
2048                 const auto deferredOperation    = deferredOperationPtr.get();
2049
2050                 const VkResult result = vk.copyMemoryToAccelerationStructureKHR(device, deferredOperation, &copyAccelerationStructureInfo);
2051
2052                 DE_ASSERT(result == VK_OPERATION_DEFERRED_KHR || result == VK_OPERATION_NOT_DEFERRED_KHR || result == VK_SUCCESS);
2053
2054                 finishDeferredOperation(vk, device, deferredOperation, m_workerThreadCount, result == VK_OPERATION_NOT_DEFERRED_KHR);
2055         }
2056
2057         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
2058         {
2059                 const VkAccessFlags             accessMasks = VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR | VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR;
2060                 const VkMemoryBarrier   memBarrier = makeMemoryBarrier(accessMasks, accessMasks);
2061
2062                 cmdPipelineMemoryBarrier(vk, cmdBuffer, VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, &memBarrier);
2063         }
2064 }
2065
2066 const VkAccelerationStructureKHR* TopLevelAccelerationStructureKHR::getPtr (void) const
2067 {
2068         return &m_accelerationStructureKHR.get();
2069 }
2070
2071 void TopLevelAccelerationStructureKHR::prepareInstances (const DeviceInterface&                                                 vk,
2072                                                                                                                  const VkDevice                                                                 device,
2073                                                                                                                  VkAccelerationStructureGeometryKHR&                    accelerationStructureGeometryKHR,
2074                                                                                                                  std::vector<deUint32>&                                                 maxPrimitiveCounts)
2075 {
2076         maxPrimitiveCounts.resize(1);
2077         maxPrimitiveCounts[0] = static_cast<deUint32>(m_bottomLevelInstances.size());
2078
2079         VkDeviceOrHostAddressConstKHR                                                   instancesData;
2080         if (m_buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
2081         {
2082                 if(m_instanceBuffer.get() != DE_NULL)
2083                 {
2084                         if (m_useArrayOfPointers)
2085                         {
2086                                 deUint8*                                                bufferStart                     = static_cast<deUint8*>(m_instanceAddressBuffer->getAllocation().getHostPtr());
2087                                 VkDeviceSize                                    bufferOffset            = 0;
2088                                 VkDeviceOrHostAddressConstKHR   firstInstance           = makeDeviceOrHostAddressConstKHR(vk, device, m_instanceBuffer->get(), 0);
2089                                 for (size_t instanceNdx = 0; instanceNdx < m_bottomLevelInstances.size(); ++instanceNdx)
2090                                 {
2091                                         VkDeviceOrHostAddressConstKHR   currentInstance;
2092                                         currentInstance.deviceAddress   = firstInstance.deviceAddress + instanceNdx * sizeof(VkAccelerationStructureInstanceKHR);
2093
2094                                         deMemcpy(&bufferStart[bufferOffset], &currentInstance, sizeof(VkDeviceOrHostAddressConstKHR::deviceAddress));
2095                                         bufferOffset += sizeof(VkDeviceOrHostAddressConstKHR::deviceAddress);
2096                                 }
2097                                 flushMappedMemoryRange(vk, device, m_instanceAddressBuffer->getAllocation().getMemory(), m_instanceAddressBuffer->getAllocation().getOffset(), VK_WHOLE_SIZE);
2098
2099                                 instancesData = makeDeviceOrHostAddressConstKHR(vk, device, m_instanceAddressBuffer->get(), 0);
2100                         }
2101                         else
2102                                 instancesData = makeDeviceOrHostAddressConstKHR(vk, device, m_instanceBuffer->get(), 0);
2103                 }
2104                 else
2105                         instancesData = makeDeviceOrHostAddressConstKHR(DE_NULL);
2106         }
2107         else
2108         {
2109                 if (m_instanceBuffer.get() != DE_NULL)
2110                 {
2111                         if (m_useArrayOfPointers)
2112                         {
2113                                 deUint8*                                                bufferStart                     = static_cast<deUint8*>(m_instanceAddressBuffer->getAllocation().getHostPtr());
2114                                 VkDeviceSize                                    bufferOffset            = 0;
2115                                 for (size_t instanceNdx = 0; instanceNdx < m_bottomLevelInstances.size(); ++instanceNdx)
2116                                 {
2117                                         VkDeviceOrHostAddressConstKHR   currentInstance;
2118                                         currentInstance.hostAddress     = (deUint8*)m_instanceBuffer->getAllocation().getHostPtr() + instanceNdx * sizeof(VkAccelerationStructureInstanceKHR);
2119
2120                                         deMemcpy(&bufferStart[bufferOffset], &currentInstance, sizeof(VkDeviceOrHostAddressConstKHR::hostAddress));
2121                                         bufferOffset += sizeof(VkDeviceOrHostAddressConstKHR::hostAddress);
2122                                 }
2123                                 instancesData = makeDeviceOrHostAddressConstKHR(m_instanceAddressBuffer->getAllocation().getHostPtr());
2124                         }
2125                         else
2126                                 instancesData = makeDeviceOrHostAddressConstKHR(m_instanceBuffer->getAllocation().getHostPtr());
2127                 }
2128                 else
2129                         instancesData = makeDeviceOrHostAddressConstKHR(DE_NULL);
2130         }
2131
2132         VkAccelerationStructureGeometryInstancesDataKHR accelerationStructureGeometryInstancesDataKHR   =
2133         {
2134                 VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_INSTANCES_DATA_KHR,   //  VkStructureType                                     sType;
2135                 DE_NULL,                                                                                                                                //  const void*                                         pNext;
2136                 (VkBool32)( m_useArrayOfPointers ? DE_TRUE : DE_FALSE ),                                //  VkBool32                                            arrayOfPointers;
2137                 instancesData                                                                                                                   //  VkDeviceOrHostAddressConstKHR       data;
2138         };
2139
2140         accelerationStructureGeometryKHR                                        =
2141         {
2142                 VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_KHR,                                                                          //  VkStructureType                                                     sType;
2143                 DE_NULL,                                                                                                                                                                        //  const void*                                                         pNext;
2144                 VK_GEOMETRY_TYPE_INSTANCES_KHR,                                                                                                                         //  VkGeometryTypeKHR                                           geometryType;
2145                 makeVkAccelerationStructureInstancesDataKHR(accelerationStructureGeometryInstancesDataKHR),     //  VkAccelerationStructureGeometryDataKHR      geometry;
2146                 (VkGeometryFlagsKHR)0u                                                                                                                                          //  VkGeometryFlagsKHR                                          flags;
2147         };
2148 }
2149
2150 deUint32 TopLevelAccelerationStructure::getRequiredAllocationCount (void)
2151 {
2152         return TopLevelAccelerationStructureKHR::getRequiredAllocationCount();
2153 }
2154
2155 de::MovePtr<TopLevelAccelerationStructure> makeTopLevelAccelerationStructure ()
2156 {
2157         return de::MovePtr<TopLevelAccelerationStructure>(new TopLevelAccelerationStructureKHR);
2158 }
2159
2160 bool queryAccelerationStructureSizeKHR (const DeviceInterface&                                                  vk,
2161                                                                                 const VkDevice                                                                  device,
2162                                                                                 const VkCommandBuffer                                                   cmdBuffer,
2163                                                                                 const std::vector<VkAccelerationStructureKHR>&  accelerationStructureHandles,
2164                                                                                 VkAccelerationStructureBuildTypeKHR                             buildType,
2165                                                                                 const VkQueryPool                                                               queryPool,
2166                                                                                 VkQueryType                                                                             queryType,
2167                                                                                 deUint32                                                                                firstQuery,
2168                                                                                 std::vector<VkDeviceSize>&                                              results)
2169 {
2170         DE_ASSERT(queryType == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR || queryType == VK_QUERY_TYPE_ACCELERATION_STRUCTURE_SERIALIZATION_SIZE_KHR);
2171
2172         if (buildType == VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR)
2173         {
2174                 // queryPool must be large enough to contain at least (firstQuery + accelerationStructureHandles.size()) queries
2175                 vk.cmdResetQueryPool(cmdBuffer, queryPool, firstQuery, deUint32(accelerationStructureHandles.size()));
2176                 vk.cmdWriteAccelerationStructuresPropertiesKHR(cmdBuffer, deUint32(accelerationStructureHandles.size()), accelerationStructureHandles.data(), queryType, queryPool, firstQuery);
2177                 // results cannot be retrieved to CPU at the moment - you need to do it using getQueryPoolResults after cmdBuffer is executed. Meanwhile function returns a vector of 0s.
2178                 results.resize(accelerationStructureHandles.size(), 0u);
2179                 return false;
2180         }
2181         // buildType != VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR
2182         results.resize(accelerationStructureHandles.size(), 0u);
2183         vk.writeAccelerationStructuresPropertiesKHR(device, deUint32(accelerationStructureHandles.size()), accelerationStructureHandles.data(), queryType,
2184                                                                                                 sizeof(VkDeviceSize) * accelerationStructureHandles.size(), results.data(), sizeof(VkDeviceSize));
2185         // results will contain proper values
2186         return true;
2187 }
2188
2189 bool queryAccelerationStructureSize (const DeviceInterface&                                                     vk,
2190                                                                          const VkDevice                                                                 device,
2191                                                                          const VkCommandBuffer                                                  cmdBuffer,
2192                                                                          const std::vector<VkAccelerationStructureKHR>& accelerationStructureHandles,
2193                                                                          VkAccelerationStructureBuildTypeKHR                    buildType,
2194                                                                          const VkQueryPool                                                              queryPool,
2195                                                                          VkQueryType                                                                    queryType,
2196                                                                          deUint32                                                                               firstQuery,
2197                                                                          std::vector<VkDeviceSize>&                                             results)
2198 {
2199         return queryAccelerationStructureSizeKHR(vk, device, cmdBuffer, accelerationStructureHandles, buildType, queryPool, queryType, firstQuery, results);
2200 }
2201
2202 RayTracingPipeline::RayTracingPipeline ()
2203         : m_shadersModules                      ()
2204         , m_pipelineLibraries           ()
2205         , m_shaderCreateInfos           ()
2206         , m_shadersGroupCreateInfos     ()
2207         , m_pipelineCreateFlags         (0U)
2208         , m_maxRecursionDepth           (1U)
2209         , m_maxPayloadSize                      (0U)
2210         , m_maxAttributeSize            (0U)
2211         , m_deferredOperation           (false)
2212         , m_workerThreadCount           (0)
2213 {
2214 }
2215
2216 RayTracingPipeline::~RayTracingPipeline ()
2217 {
2218 }
2219
2220 #define CHECKED_ASSIGN_SHADER(SHADER, STAGE)                                            \
2221         if (SHADER == VK_SHADER_UNUSED_KHR)                                                             \
2222                 SHADER = STAGE;                                                                                         \
2223         else                                                                                                                    \
2224                 TCU_THROW(InternalError, "Attempt to reassign shader")
2225
2226 void RayTracingPipeline::addShader (VkShaderStageFlagBits shaderStage, Move<VkShaderModule> shaderModule, deUint32 group, const VkSpecializationInfo* specializationInfo)
2227 {
2228         addShader(shaderStage, makeVkSharedPtr(shaderModule), group, specializationInfo);
2229 }
2230
2231 void RayTracingPipeline::addShader (VkShaderStageFlagBits shaderStage, de::SharedPtr<Move<VkShaderModule>> shaderModule, deUint32 group, const VkSpecializationInfo* specializationInfoPtr)
2232 {
2233         if (group >= m_shadersGroupCreateInfos.size())
2234         {
2235                 for (size_t groupNdx = m_shadersGroupCreateInfos.size(); groupNdx <= group; ++groupNdx)
2236                 {
2237                         VkRayTracingShaderGroupCreateInfoKHR    shaderGroupCreateInfo   =
2238                         {
2239                                 VK_STRUCTURE_TYPE_RAY_TRACING_SHADER_GROUP_CREATE_INFO_KHR,     //  VkStructureType                                     sType;
2240                                 DE_NULL,                                                                                                        //  const void*                                         pNext;
2241                                 VK_RAY_TRACING_SHADER_GROUP_TYPE_MAX_ENUM_KHR,                          //  VkRayTracingShaderGroupTypeKHR      type;
2242                                 VK_SHADER_UNUSED_KHR,                                                                           //  deUint32                                            generalShader;
2243                                 VK_SHADER_UNUSED_KHR,                                                                           //  deUint32                                            closestHitShader;
2244                                 VK_SHADER_UNUSED_KHR,                                                                           //  deUint32                                            anyHitShader;
2245                                 VK_SHADER_UNUSED_KHR,                                                                           //  deUint32                                            intersectionShader;
2246                                 DE_NULL,                                                                                                        //  const void*                                         pShaderGroupCaptureReplayHandle;
2247                         };
2248
2249                         m_shadersGroupCreateInfos.push_back(shaderGroupCreateInfo);
2250                 }
2251         }
2252
2253         const deUint32                                                  shaderStageNdx                  = (deUint32)m_shaderCreateInfos.size();
2254         VkRayTracingShaderGroupCreateInfoKHR&   shaderGroupCreateInfo   = m_shadersGroupCreateInfos[group];
2255
2256         switch (shaderStage)
2257         {
2258                 case VK_SHADER_STAGE_RAYGEN_BIT_KHR:            CHECKED_ASSIGN_SHADER(shaderGroupCreateInfo.generalShader,              shaderStageNdx);        break;
2259                 case VK_SHADER_STAGE_MISS_BIT_KHR:                      CHECKED_ASSIGN_SHADER(shaderGroupCreateInfo.generalShader,              shaderStageNdx);        break;
2260                 case VK_SHADER_STAGE_CALLABLE_BIT_KHR:          CHECKED_ASSIGN_SHADER(shaderGroupCreateInfo.generalShader,              shaderStageNdx);        break;
2261                 case VK_SHADER_STAGE_ANY_HIT_BIT_KHR:           CHECKED_ASSIGN_SHADER(shaderGroupCreateInfo.anyHitShader,               shaderStageNdx);        break;
2262                 case VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR:       CHECKED_ASSIGN_SHADER(shaderGroupCreateInfo.closestHitShader,   shaderStageNdx);        break;
2263                 case VK_SHADER_STAGE_INTERSECTION_BIT_KHR:      CHECKED_ASSIGN_SHADER(shaderGroupCreateInfo.intersectionShader, shaderStageNdx);        break;
2264                 default:                                                                        TCU_THROW(InternalError, "Unacceptable stage");
2265         }
2266
2267         switch (shaderStage)
2268         {
2269                 case VK_SHADER_STAGE_RAYGEN_BIT_KHR:
2270                 case VK_SHADER_STAGE_MISS_BIT_KHR:
2271                 case VK_SHADER_STAGE_CALLABLE_BIT_KHR:
2272                 {
2273                         DE_ASSERT(shaderGroupCreateInfo.type == VK_RAY_TRACING_SHADER_GROUP_TYPE_MAX_ENUM_KHR);
2274                         shaderGroupCreateInfo.type = VK_RAY_TRACING_SHADER_GROUP_TYPE_GENERAL_KHR;
2275
2276                         break;
2277                 }
2278
2279                 case VK_SHADER_STAGE_ANY_HIT_BIT_KHR:
2280                 case VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR:
2281                 case VK_SHADER_STAGE_INTERSECTION_BIT_KHR:
2282                 {
2283                         DE_ASSERT(shaderGroupCreateInfo.type != VK_RAY_TRACING_SHADER_GROUP_TYPE_GENERAL_KHR);
2284                         shaderGroupCreateInfo.type      = (shaderGroupCreateInfo.intersectionShader == VK_SHADER_UNUSED_KHR)
2285                                                                                 ? VK_RAY_TRACING_SHADER_GROUP_TYPE_TRIANGLES_HIT_GROUP_KHR
2286                                                                                 : VK_RAY_TRACING_SHADER_GROUP_TYPE_PROCEDURAL_HIT_GROUP_KHR;
2287
2288                         break;
2289                 }
2290
2291                 default: TCU_THROW(InternalError, "Unacceptable stage");
2292         }
2293
2294         {
2295                 const VkPipelineShaderStageCreateInfo   shaderCreateInfo        =
2296                 {
2297                         VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,    //  VkStructureType                                             sType;
2298                         DE_NULL,                                                                                                //  const void*                                                 pNext;
2299                         (VkPipelineShaderStageCreateFlags)0,                                    //  VkPipelineShaderStageCreateFlags    flags;
2300                         shaderStage,                                                                                    //  VkShaderStageFlagBits                               stage;
2301                         **shaderModule,                                                                                 //  VkShaderModule                                              module;
2302                         "main",                                                                                                 //  const char*                                                 pName;
2303                         specializationInfoPtr,                                                                  //  const VkSpecializationInfo*                 pSpecializationInfo;
2304                 };
2305
2306                 m_shaderCreateInfos.push_back(shaderCreateInfo);
2307         }
2308
2309         m_shadersModules.push_back(shaderModule);
2310 }
2311
2312 void RayTracingPipeline::addLibrary (de::SharedPtr<de::MovePtr<RayTracingPipeline>> pipelineLibrary)
2313 {
2314         m_pipelineLibraries.push_back(pipelineLibrary);
2315 }
2316
2317 Move<VkPipeline> RayTracingPipeline::createPipelineKHR (const DeviceInterface&                                                          vk,
2318                                                                                                                 const VkDevice                                                                          device,
2319                                                                                                                 const VkPipelineLayout                                                          pipelineLayout,
2320                                                                                                                 const std::vector<de::SharedPtr<Move<VkPipeline>>>&     pipelineLibraries)
2321 {
2322         for (size_t groupNdx = 0; groupNdx < m_shadersGroupCreateInfos.size(); ++groupNdx)
2323                 DE_ASSERT(m_shadersGroupCreateInfos[groupNdx].sType == VK_STRUCTURE_TYPE_RAY_TRACING_SHADER_GROUP_CREATE_INFO_KHR);
2324
2325         DE_ASSERT(m_shaderCreateInfos.size() > 0);
2326         DE_ASSERT(m_shadersGroupCreateInfos.size() > 0);
2327
2328         std::vector<VkPipeline>                                                         vkPipelineLibraries;
2329         for (auto it = begin(pipelineLibraries), eit = end(pipelineLibraries); it != eit; ++it)
2330                 vkPipelineLibraries.push_back( it->get()->get() );
2331         VkPipelineLibraryCreateInfoKHR                          librariesCreateInfo     =
2332         {
2333                 VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR,             //  VkStructureType     sType;
2334                 DE_NULL,                                                                                                //  const void*         pNext;
2335                 deUint32(vkPipelineLibraries.size()),                                   //  deUint32            libraryCount;
2336                 dataOrNullPtr(vkPipelineLibraries)                                              //  VkPipeline*         pLibraries;
2337         };
2338         const VkRayTracingPipelineInterfaceCreateInfoKHR        pipelineInterfaceCreateInfo             =
2339         {
2340                 VK_STRUCTURE_TYPE_RAY_TRACING_PIPELINE_INTERFACE_CREATE_INFO_KHR,       //  VkStructureType     sType;
2341                 DE_NULL,                                                                                                                        //  const void*         pNext;
2342                 m_maxPayloadSize,                                                                                                       //  deUint32            maxPayloadSize;
2343                 m_maxAttributeSize                                                                                                      //  deUint32            maxAttributeSize;
2344         };
2345         const bool                                                                                      addPipelineInterfaceCreateInfo  = m_maxPayloadSize != 0 || m_maxAttributeSize != 0;
2346         const VkRayTracingPipelineInterfaceCreateInfoKHR*       pipelineInterfaceCreateInfoPtr  = addPipelineInterfaceCreateInfo ? &pipelineInterfaceCreateInfo : DE_NULL;
2347         const VkPipelineLibraryCreateInfoKHR*                           librariesCreateInfoPtr                  = (vkPipelineLibraries.empty() ? nullptr : &librariesCreateInfo);
2348
2349         Move<VkDeferredOperationKHR>                                            deferredOperation;
2350         if (m_deferredOperation)
2351                 deferredOperation = createDeferredOperationKHR(vk, device);
2352
2353         VkPipelineDynamicStateCreateInfo dynamicStateCreateInfo =
2354         {
2355                 VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,   // VkStructureType                                              sType;
2356                 DE_NULL,                                                                                                // const void*                                                  pNext;
2357                 0,                                                                                                              // VkPipelineDynamicStateCreateFlags    flags;
2358                 static_cast<deUint32>(m_dynamicStates.size() ),                 // deUint32                                                             dynamicStateCount;
2359                 m_dynamicStates.data(),                                                                 // const VkDynamicState*                                pDynamicStates;
2360         };
2361
2362         const VkRayTracingPipelineCreateInfoKHR                         pipelineCreateInfo                              =
2363         {
2364                 VK_STRUCTURE_TYPE_RAY_TRACING_PIPELINE_CREATE_INFO_KHR, //  VkStructureType                                                             sType;
2365                 DE_NULL,                                                                                                //  const void*                                                                 pNext;
2366                 m_pipelineCreateFlags,                                                                  //  VkPipelineCreateFlags                                               flags;
2367                 (deUint32)m_shaderCreateInfos.size(),                                   //  deUint32                                                                    stageCount;
2368                 m_shaderCreateInfos.data(),                                                             //  const VkPipelineShaderStageCreateInfo*              pStages;
2369                 (deUint32)m_shadersGroupCreateInfos.size(),                             //  deUint32                                                                    groupCount;
2370                 m_shadersGroupCreateInfos.data(),                                               //  const VkRayTracingShaderGroupCreateInfoKHR* pGroups;
2371                 m_maxRecursionDepth,                                                                    //  deUint32                                                                    maxRecursionDepth;
2372                 librariesCreateInfoPtr,                                                                 //  VkPipelineLibraryCreateInfoKHR*                             pLibraryInfo;
2373                 pipelineInterfaceCreateInfoPtr,                                                 //  VkRayTracingPipelineInterfaceCreateInfoKHR* pLibraryInterface;
2374                 &dynamicStateCreateInfo,                                                                //  const VkPipelineDynamicStateCreateInfo*             pDynamicState;
2375                 pipelineLayout,                                                                                 //  VkPipelineLayout                                                    layout;
2376                 (VkPipeline)DE_NULL,                                                                    //  VkPipeline                                                                  basePipelineHandle;
2377                 0,                                                                                                              //  deInt32                                                                             basePipelineIndex;
2378         };
2379         VkPipeline                                                                                      object                                                  = DE_NULL;
2380         VkResult                                                                                        result                                                  = vk.createRayTracingPipelinesKHR(device, deferredOperation.get(), DE_NULL, 1u, &pipelineCreateInfo, DE_NULL, &object);
2381         Move<VkPipeline>                                                                        pipeline                                                (check<VkPipeline>(object), Deleter<VkPipeline>(vk, device, DE_NULL));
2382
2383         if (m_deferredOperation)
2384         {
2385                 DE_ASSERT(result == VK_OPERATION_DEFERRED_KHR || result == VK_OPERATION_NOT_DEFERRED_KHR || result == VK_SUCCESS);
2386
2387                 finishDeferredOperation(vk, device, deferredOperation.get(), m_workerThreadCount, result == VK_OPERATION_NOT_DEFERRED_KHR);
2388         }
2389
2390         return pipeline;
2391 }
2392
2393
2394 Move<VkPipeline> RayTracingPipeline::createPipeline (const DeviceInterface&                                                                     vk,
2395                                                                                                          const VkDevice                                                                                 device,
2396                                                                                                          const VkPipelineLayout                                                                 pipelineLayout,
2397                                                                                                          const std::vector<de::SharedPtr<Move<VkPipeline>>>&    pipelineLibraries)
2398 {
2399         return createPipelineKHR(vk, device, pipelineLayout, pipelineLibraries);
2400 }
2401
2402 std::vector<de::SharedPtr<Move<VkPipeline>>> RayTracingPipeline::createPipelineWithLibraries (const DeviceInterface&                    vk,
2403                                                                                                                                                                                                 const VkDevice                                  device,
2404                                                                                                                                                                                                 const VkPipelineLayout                  pipelineLayout)
2405 {
2406         for (size_t groupNdx = 0; groupNdx < m_shadersGroupCreateInfos.size(); ++groupNdx)
2407                 DE_ASSERT(m_shadersGroupCreateInfos[groupNdx].sType == VK_STRUCTURE_TYPE_RAY_TRACING_SHADER_GROUP_CREATE_INFO_KHR);
2408
2409         DE_ASSERT(m_shaderCreateInfos.size() > 0);
2410         DE_ASSERT(m_shadersGroupCreateInfos.size() > 0);
2411
2412         std::vector<de::SharedPtr<Move<VkPipeline>>> result, allLibraries, firstLibraries;
2413         for(auto it=begin(m_pipelineLibraries), eit=end(m_pipelineLibraries); it!=eit; ++it)
2414         {
2415                 auto childLibraries = (*it)->get()->createPipelineWithLibraries(vk, device, pipelineLayout);
2416                 DE_ASSERT(childLibraries.size() > 0);
2417                 firstLibraries.push_back(childLibraries[0]);
2418                 std::copy(begin(childLibraries), end(childLibraries), std::back_inserter(allLibraries));
2419         }
2420         result.push_back(makeVkSharedPtr(createPipeline(vk, device, pipelineLayout, firstLibraries)));
2421         std::copy(begin(allLibraries), end(allLibraries), std::back_inserter(result));
2422         return result;
2423 }
2424
2425 de::MovePtr<BufferWithMemory> RayTracingPipeline::createShaderBindingTable (const DeviceInterface&              vk,
2426                                                                                                                                                         const VkDevice                          device,
2427                                                                                                                                                         const VkPipeline                        pipeline,
2428                                                                                                                                                         Allocator&                                      allocator,
2429                                                                                                                                                         const deUint32&                         shaderGroupHandleSize,
2430                                                                                                                                                         const deUint32                          shaderGroupBaseAlignment,
2431                                                                                                                                                         const deUint32&                         firstGroup,
2432                                                                                                                                                         const deUint32&                         groupCount,
2433                                                                                                                                                         const VkBufferCreateFlags&      additionalBufferCreateFlags,
2434                                                                                                                                                         const VkBufferUsageFlags&       additionalBufferUsageFlags,
2435                                                                                                                                                         const MemoryRequirement&        additionalMemoryRequirement,
2436                                                                                                                                                         const VkDeviceAddress&          opaqueCaptureAddress,
2437                                                                                                                                                         const deUint32                          shaderBindingTableOffset,
2438                                                                                                                                                         const deUint32                          shaderRecordSize,
2439                                                                                                                                                         const void**                            shaderGroupDataPtrPerGroup)
2440 {
2441         DE_ASSERT(shaderGroupBaseAlignment != 0u);
2442         DE_ASSERT((shaderBindingTableOffset % shaderGroupBaseAlignment) == 0);
2443         DE_UNREF(shaderGroupBaseAlignment);
2444
2445         const deUint32                                                  sbtSize                                                 = shaderBindingTableOffset + groupCount * deAlign32(shaderGroupHandleSize + shaderRecordSize, shaderGroupHandleSize);
2446         const VkBufferUsageFlags                                sbtFlags                                                = VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_BINDING_TABLE_BIT_KHR | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | additionalBufferUsageFlags;
2447         VkBufferCreateInfo                                              sbtCreateInfo                                   = makeBufferCreateInfo(sbtSize, sbtFlags);
2448         sbtCreateInfo.flags                                                                                                             |= additionalBufferCreateFlags;
2449         VkBufferOpaqueCaptureAddressCreateInfo  sbtCaptureAddressInfo                   =
2450         {
2451                 VK_STRUCTURE_TYPE_BUFFER_OPAQUE_CAPTURE_ADDRESS_CREATE_INFO,    // VkStructureType      sType;
2452                 DE_NULL,                                                                                                                // const void*          pNext;
2453                 deUint64(opaqueCaptureAddress)                                                                  // deUint64                     opaqueCaptureAddress;
2454         };
2455
2456         if (opaqueCaptureAddress != 0u)
2457         {
2458                 sbtCreateInfo.pNext = &sbtCaptureAddressInfo;
2459                 sbtCreateInfo.flags |= VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT;
2460         }
2461         const MemoryRequirement                 sbtMemRequirements                                              = MemoryRequirement::HostVisible | MemoryRequirement::Coherent | MemoryRequirement::DeviceAddress | additionalMemoryRequirement;
2462         de::MovePtr<BufferWithMemory>   sbtBuffer                                                               = de::MovePtr<BufferWithMemory>(new BufferWithMemory(vk, device, allocator, sbtCreateInfo, sbtMemRequirements));
2463         vk::Allocation&                                 sbtAlloc                                                                = sbtBuffer->getAllocation();
2464
2465         // collect shader group handles
2466         std::vector<deUint8>                    shaderHandles                                                   (groupCount * shaderGroupHandleSize);
2467         VK_CHECK(getRayTracingShaderGroupHandles(vk, device, pipeline, firstGroup, groupCount, groupCount * shaderGroupHandleSize, shaderHandles.data()));
2468
2469         // reserve place for ShaderRecordKHR after each shader handle ( ShaderRecordKHR size might be 0 ). Also take alignment into consideration
2470         deUint8* shaderBegin = (deUint8*)sbtAlloc.getHostPtr() + shaderBindingTableOffset;
2471         for (deUint32 idx = 0; idx < groupCount; ++idx)
2472         {
2473                 deUint8* shaderSrcPos   = shaderHandles.data() + idx * shaderGroupHandleSize;
2474                 deUint8* shaderDstPos   = shaderBegin + idx * deAlign32(shaderGroupHandleSize + shaderRecordSize, shaderGroupHandleSize);
2475                 deMemcpy(shaderDstPos, shaderSrcPos, shaderGroupHandleSize);
2476
2477                 if (shaderGroupDataPtrPerGroup          != nullptr &&
2478                         shaderGroupDataPtrPerGroup[idx] != nullptr)
2479                 {
2480                         DE_ASSERT(sbtSize >= static_cast<deUint32>(shaderDstPos - shaderBegin) + shaderGroupHandleSize);
2481
2482                         deMemcpy(       shaderDstPos + shaderGroupHandleSize,
2483                                                 shaderGroupDataPtrPerGroup[idx],
2484                                                 shaderRecordSize);
2485                 }
2486         }
2487
2488         flushMappedMemoryRange(vk, device, sbtAlloc.getMemory(), sbtAlloc.getOffset(), VK_WHOLE_SIZE);
2489
2490         return sbtBuffer;
2491 }
2492
2493 void RayTracingPipeline::setCreateFlags (const VkPipelineCreateFlags& pipelineCreateFlags)
2494 {
2495         m_pipelineCreateFlags = pipelineCreateFlags;
2496 }
2497
2498 void RayTracingPipeline::setMaxRecursionDepth (const deUint32& maxRecursionDepth)
2499 {
2500         m_maxRecursionDepth = maxRecursionDepth;
2501 }
2502
2503 void RayTracingPipeline::setMaxPayloadSize (const deUint32& maxPayloadSize)
2504 {
2505         m_maxPayloadSize = maxPayloadSize;
2506 }
2507
2508 void RayTracingPipeline::setMaxAttributeSize (const deUint32& maxAttributeSize)
2509 {
2510         m_maxAttributeSize = maxAttributeSize;
2511 }
2512
2513 void RayTracingPipeline::setDeferredOperation (const bool               deferredOperation,
2514                                                                                            const deUint32       workerThreadCount)
2515 {
2516         m_deferredOperation = deferredOperation;
2517         m_workerThreadCount = workerThreadCount;
2518 }
2519
2520 void RayTracingPipeline::addDynamicState(const VkDynamicState& dynamicState)
2521 {
2522         m_dynamicStates.push_back(dynamicState);
2523 }
2524
2525 class RayTracingPropertiesKHR : public RayTracingProperties
2526 {
2527 public:
2528                                                         RayTracingPropertiesKHR                                         () = delete;
2529                                                         RayTracingPropertiesKHR                                         (const InstanceInterface&       vki,
2530                                                                                                                                                  const VkPhysicalDevice         physicalDevice);
2531         virtual                                 ~RayTracingPropertiesKHR                                        ();
2532
2533         virtual deUint32                getShaderGroupHandleSize                                        (void)  { return m_rayTracingPipelineProperties.shaderGroupHandleSize;                                          };
2534         virtual deUint32                getMaxRecursionDepth                                            (void)  { return m_rayTracingPipelineProperties.maxRayRecursionDepth;                                           };
2535         virtual deUint32                getMaxShaderGroupStride                                         (void)  { return m_rayTracingPipelineProperties.maxShaderGroupStride;                                           };
2536         virtual deUint32                getShaderGroupBaseAlignment                                     (void)  { return m_rayTracingPipelineProperties.shaderGroupBaseAlignment;                                       };
2537         virtual deUint64                getMaxGeometryCount                                                     (void)  { return m_accelerationStructureProperties.maxGeometryCount;                                            };
2538         virtual deUint64                getMaxInstanceCount                                                     (void)  { return m_accelerationStructureProperties.maxInstanceCount;                                            };
2539         virtual deUint64                getMaxPrimitiveCount                                            (void)  { return m_accelerationStructureProperties.maxPrimitiveCount;                                           };
2540         virtual deUint32                getMaxDescriptorSetAccelerationStructures       (void)  { return m_accelerationStructureProperties.maxDescriptorSetAccelerationStructures;      };
2541         deUint32                                getMaxRayDispatchInvocationCount                        (void)  { return m_rayTracingPipelineProperties.maxRayDispatchInvocationCount;                          };
2542         deUint32                                getMaxRayHitAttributeSize                                       (void)  { return m_rayTracingPipelineProperties.maxRayHitAttributeSize;                                         };
2543
2544 protected:
2545         VkPhysicalDeviceAccelerationStructurePropertiesKHR      m_accelerationStructureProperties;
2546         VkPhysicalDeviceRayTracingPipelinePropertiesKHR         m_rayTracingPipelineProperties;
2547 };
2548
2549 RayTracingPropertiesKHR::~RayTracingPropertiesKHR ()
2550 {
2551 }
2552
2553 RayTracingPropertiesKHR::RayTracingPropertiesKHR (const InstanceInterface&      vki,
2554                                                                                                   const VkPhysicalDevice        physicalDevice)
2555         : RayTracingProperties  (vki, physicalDevice)
2556 {
2557         m_accelerationStructureProperties       = getPhysicalDeviceExtensionProperties(vki, physicalDevice);
2558         m_rayTracingPipelineProperties          = getPhysicalDeviceExtensionProperties(vki, physicalDevice);
2559 }
2560
2561 de::MovePtr<RayTracingProperties> makeRayTracingProperties (const InstanceInterface&    vki,
2562                                                                                                                         const VkPhysicalDevice          physicalDevice)
2563 {
2564         return de::MovePtr<RayTracingProperties>(new RayTracingPropertiesKHR(vki, physicalDevice));
2565 }
2566
2567 static inline void cmdTraceRaysKHR (const DeviceInterface&                                      vk,
2568                                                                         VkCommandBuffer                                                 commandBuffer,
2569                                                                         const VkStridedDeviceAddressRegionKHR*  raygenShaderBindingTableRegion,
2570                                                                         const VkStridedDeviceAddressRegionKHR*  missShaderBindingTableRegion,
2571                                                                         const VkStridedDeviceAddressRegionKHR*  hitShaderBindingTableRegion,
2572                                                                         const VkStridedDeviceAddressRegionKHR*  callableShaderBindingTableRegion,
2573                                                                         deUint32                                                                width,
2574                                                                         deUint32                                                                height,
2575                                                                         deUint32                                                                depth)
2576 {
2577         return vk.cmdTraceRaysKHR(commandBuffer,
2578                                                           raygenShaderBindingTableRegion,
2579                                                           missShaderBindingTableRegion,
2580                                                           hitShaderBindingTableRegion,
2581                                                           callableShaderBindingTableRegion,
2582                                                           width,
2583                                                           height,
2584                                                           depth);
2585 }
2586
2587
2588 void cmdTraceRays (const DeviceInterface&                                       vk,
2589                                    VkCommandBuffer                                                      commandBuffer,
2590                                    const VkStridedDeviceAddressRegionKHR*       raygenShaderBindingTableRegion,
2591                                    const VkStridedDeviceAddressRegionKHR*       missShaderBindingTableRegion,
2592                                    const VkStridedDeviceAddressRegionKHR*       hitShaderBindingTableRegion,
2593                                    const VkStridedDeviceAddressRegionKHR*       callableShaderBindingTableRegion,
2594                                    deUint32                                                                     width,
2595                                    deUint32                                                                     height,
2596                                    deUint32                                                                     depth)
2597 {
2598         DE_ASSERT(raygenShaderBindingTableRegion        != DE_NULL);
2599         DE_ASSERT(missShaderBindingTableRegion          != DE_NULL);
2600         DE_ASSERT(hitShaderBindingTableRegion           != DE_NULL);
2601         DE_ASSERT(callableShaderBindingTableRegion      != DE_NULL);
2602
2603         return cmdTraceRaysKHR(vk,
2604                                                    commandBuffer,
2605                                                    raygenShaderBindingTableRegion,
2606                                                    missShaderBindingTableRegion,
2607                                                    hitShaderBindingTableRegion,
2608                                                    callableShaderBindingTableRegion,
2609                                                    width,
2610                                                    height,
2611                                                    depth);
2612 }
2613
2614 static inline void cmdTraceRaysIndirectKHR (const DeviceInterface&                                      vk,
2615                                                                                         VkCommandBuffer                                                 commandBuffer,
2616                                                                                         const VkStridedDeviceAddressRegionKHR*  raygenShaderBindingTableRegion,
2617                                                                                         const VkStridedDeviceAddressRegionKHR*  missShaderBindingTableRegion,
2618                                                                                         const VkStridedDeviceAddressRegionKHR*  hitShaderBindingTableRegion,
2619                                                                                         const VkStridedDeviceAddressRegionKHR*  callableShaderBindingTableRegion,
2620                                                                                         VkDeviceAddress                                                 indirectDeviceAddress )
2621 {
2622         DE_ASSERT(raygenShaderBindingTableRegion        != DE_NULL);
2623         DE_ASSERT(missShaderBindingTableRegion          != DE_NULL);
2624         DE_ASSERT(hitShaderBindingTableRegion           != DE_NULL);
2625         DE_ASSERT(callableShaderBindingTableRegion      != DE_NULL);
2626         DE_ASSERT(indirectDeviceAddress                         != 0);
2627
2628         return vk.cmdTraceRaysIndirectKHR(commandBuffer,
2629                                                                           raygenShaderBindingTableRegion,
2630                                                                           missShaderBindingTableRegion,
2631                                                                           hitShaderBindingTableRegion,
2632                                                                           callableShaderBindingTableRegion,
2633                                                                           indirectDeviceAddress);
2634 }
2635
2636 void cmdTraceRaysIndirect (const DeviceInterface&                                       vk,
2637                                                    VkCommandBuffer                                                      commandBuffer,
2638                                                    const VkStridedDeviceAddressRegionKHR*       raygenShaderBindingTableRegion,
2639                                                    const VkStridedDeviceAddressRegionKHR*       missShaderBindingTableRegion,
2640                                                    const VkStridedDeviceAddressRegionKHR*       hitShaderBindingTableRegion,
2641                                                    const VkStridedDeviceAddressRegionKHR*       callableShaderBindingTableRegion,
2642                                                    VkDeviceAddress                                                      indirectDeviceAddress)
2643 {
2644         return cmdTraceRaysIndirectKHR(vk,
2645                                                                    commandBuffer,
2646                                                                    raygenShaderBindingTableRegion,
2647                                                                    missShaderBindingTableRegion,
2648                                                                    hitShaderBindingTableRegion,
2649                                                                    callableShaderBindingTableRegion,
2650                                                                    indirectDeviceAddress);
2651 }
2652
2653 } // vk