external/vulkancts/modules/vulkan/device_group/vktDeviceGroupRendering.cpp

   1 /*------------------------------------------------------------------------
   2 * Vulkan Conformance Tests
   3 * ------------------------
   4 *
   5 * Copyright (c) 2017 The Khronos Group Inc.
   6 * Copyright (c) 2017 Nvidia Corporation
   7 *
   8 * Licensed under the Apache License, Version 2.0 (the "License");
   9 * you may not use this file except in compliance with the License.
  10 * You may obtain a copy of the License at
  11 *
  12 *               http://www.apache.org/licenses/LICENSE-2.0
  13 *
  14 * Unless required by applicable law or agreed to in writing, software
  15 * distributed under the License is distributed on an "AS IS" BASIS,
  16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 * See the License for the specific language governing permissions and
  18 * limitations under the License.
  19 *
  20 *//*!
  21 * \file
  22 * \brief Device Group Tests
  23 *//*--------------------------------------------------------------------*/
  24
  25 #include "vktDeviceGroupTests.hpp"
  26
  27 #include "vkDefs.hpp"
  28 #include "vkDeviceUtil.hpp"
  29 #include "vkImageUtil.hpp"
  30 #include "vkMemUtil.hpp"
  31 #include "vkPlatform.hpp"
  32 #include "vkPrograms.hpp"
  33 #include "vkQueryUtil.hpp"
  34 #include "vkRef.hpp"
  35 #include "vkRefUtil.hpp"
  36 #include "vkStrUtil.hpp"
  37 #include "vkTypeUtil.hpp"
  38 #include "vktTestCase.hpp"
  39 #include "vktTestCaseUtil.hpp"
  40 #include "vktTestGroupUtil.hpp"
  41
  42 #include "tcuDefs.hpp"
  43 #include "tcuFormatUtil.hpp"
  44 #include "tcuImageCompare.hpp"
  45 #include "tcuResource.hpp"
  46 #include "tcuTestCase.hpp"
  47 #include "tcuTestLog.hpp"
  48 #include "tcuCommandLine.hpp"
  49 #include "tcuTextureUtil.hpp"
  50 #include "tcuImageIO.hpp"
  51
  52 #include "rrRenderer.hpp"
  53
  54 namespace vkt
  55 {
  56 namespace DeviceGroup
  57 {
  58 namespace
  59 {
  60
  61 using namespace vk;
  62 using std::string;
  63 using std::vector;
  64 using tcu::TestLog;
  65 using de::UniquePtr;
  66
  67 //Device group test modes
  68 enum TestModeType
  69 {
  70         TEST_MODE_SFR                   = 1 << 0,                       //!< Split frame remdering
  71         TEST_MODE_AFR                   = 1 << 1,                       //!< Alternate frame rendering
  72         TEST_MODE_HOSTMEMORY    = 1 << 2,                       //!< Use host memory for rendertarget
  73         TEST_MODE_DEDICATED             = 1 << 3,                       //!< Use dedicated allocations
  74         TEST_MODE_PEER_FETCH    = 1 << 4,                       //!< Peer vertex attributes from peer memroy
  75         TEST_MODE_TESSELLATION  = 1 << 5,                       //!< Generate a tessellated sphere instead of triangle
  76         TEST_MODE_LINEFILL              = 1 << 6,                       //!< Draw polygon edges as line segments
  77 };
  78
  79 class RefVertexShader : public rr::VertexShader
  80 {
  81 public:
  82         RefVertexShader (void)
  83                 : rr::VertexShader(1, 0)
  84         {
  85                 m_inputs[0].type = rr::GENERICVECTYPE_FLOAT;
  86         }
  87         virtual ~RefVertexShader(void) {}
  88
  89         void shadeVertices (const rr::VertexAttrib* inputs, rr::VertexPacket* const* packets, const int numPackets) const
  90         {
  91                 for (int packetNdx = 0; packetNdx < numPackets; ++packetNdx)
  92                 {
  93                         packets[packetNdx]->position = rr::readVertexAttribFloat(inputs[0],
  94                                 packets[packetNdx]->instanceNdx,
  95                                 packets[packetNdx]->vertexNdx);
  96                 }
  97         }
  98 };
  99
 100 class RefFragmentShader : public rr::FragmentShader
 101 {
 102 public:
 103         RefFragmentShader (void)
 104                 : rr::FragmentShader(0, 1)
 105         {
 106                 m_outputs[0].type = rr::GENERICVECTYPE_FLOAT;
 107         }
 108
 109         virtual ~RefFragmentShader(void) {}
 110
 111         void shadeFragments (rr::FragmentPacket*, const int numPackets, const rr::FragmentShadingContext& context) const
 112         {
 113                 for (int packetNdx = 0; packetNdx < numPackets; ++packetNdx)
 114                 {
 115                         for (int fragNdx = 0; fragNdx < rr::NUM_FRAGMENTS_PER_PACKET; ++fragNdx)
 116                         {
 117                                 rr::writeFragmentOutput(context, packetNdx, fragNdx, 0, tcu::Vec4(1.0f, 1.0f, 0.0f, 1.0f));
 118                         }
 119                 }
 120         }
 121 };
 122
 123 void renderReferenceTriangle (const tcu::PixelBufferAccess& dst, const tcu::Vec4(&vertices)[3])
 124 {
 125         const RefVertexShader                                   vertShader;
 126         const RefFragmentShader                                 fragShader;
 127         const rr::Program                                               program(&vertShader, &fragShader);
 128         const rr::MultisamplePixelBufferAccess  colorBuffer = rr::MultisamplePixelBufferAccess::fromSinglesampleAccess(dst);
 129         const rr::RenderTarget                                  renderTarget(colorBuffer);
 130         const rr::RenderState                                   renderState((rr::ViewportState(colorBuffer)));
 131         const rr::Renderer                                              renderer;
 132         const rr::VertexAttrib                                  vertexAttribs[] =
 133         {
 134                 rr::VertexAttrib(rr::VERTEXATTRIBTYPE_FLOAT, 4, sizeof(tcu::Vec4), 0, vertices[0].getPtr())
 135         };
 136         renderer.draw(rr::DrawCommand(renderState,
 137                 renderTarget,
 138                 program,
 139                 DE_LENGTH_OF_ARRAY(vertexAttribs),
 140                 &vertexAttribs[0],
 141                 rr::PrimitiveList(rr::PRIMITIVETYPE_TRIANGLES, DE_LENGTH_OF_ARRAY(vertices), 0)));
 142 }
 143
 144 class DeviceGroupTestInstance : public TestInstance
 145 {
 146 public:
 147         DeviceGroupTestInstance(Context& context, deUint32 mode);
 148         ~DeviceGroupTestInstance(void) {}
 149 private:
 150                         void                                            init                                    (void);
 151                         deUint32                                        getMemoryIndex                  (deUint32 memoryTypeBits, deUint32 memoryPropertyFlag);
 152                         void                                            getDeviceLayers                 (vector<string>& enabledLayers);
 153                         bool                                            isPeerFetchAllowed              (deUint32 memoryTypeIndex, deUint32 firstdeviceID, deUint32 seconddeviceID);
 154         virtual tcu::TestStatus                         iterate                                 (void);
 155
 156                         Move<VkDevice>                          m_deviceGroup;
 157                         deUint32                                        m_physicalDeviceCount;
 158                         VkQueue                                         m_deviceGroupQueue;
 159                         vector<VkPhysicalDevice>        m_physicalDevices;
 160
 161                         deUint32                                        m_testMode;
 162                         bool                                            m_useHostMemory;
 163                         bool                                            m_useDedicated;
 164                         bool                                            m_usePeerFetch;
 165                         bool                                            m_subsetAllocation;
 166                         bool                                            m_fillModeNonSolid;
 167                         bool                                            m_drawTessellatedSphere;
 168 };
 169
 170 DeviceGroupTestInstance::DeviceGroupTestInstance (Context& context, const deUint32 mode)
 171         : TestInstance                          (context)
 172         , m_physicalDeviceCount         (0)
 173         , m_deviceGroupQueue            (DE_NULL)
 174         , m_testMode                            (mode)
 175         , m_useHostMemory                       (m_testMode & TEST_MODE_HOSTMEMORY)
 176         , m_useDedicated                        (m_testMode & TEST_MODE_DEDICATED)
 177         , m_usePeerFetch                        (m_testMode & TEST_MODE_PEER_FETCH)
 178         , m_subsetAllocation            (true)
 179         , m_fillModeNonSolid            (m_testMode & TEST_MODE_LINEFILL)
 180         , m_drawTessellatedSphere       (m_testMode & TEST_MODE_TESSELLATION)
 181 {
 182         init();
 183 }
 184
 185 deUint32 DeviceGroupTestInstance::getMemoryIndex (const deUint32 memoryTypeBits, const deUint32 memoryPropertyFlag)
 186 {
 187         const VkPhysicalDeviceMemoryProperties deviceMemProps = getPhysicalDeviceMemoryProperties(m_context.getInstanceInterface(), m_context.getPhysicalDevice());
 188         for (deUint32 memoryTypeNdx = 0; memoryTypeNdx < deviceMemProps.memoryTypeCount; memoryTypeNdx++)
 189         {
 190                 if ((memoryTypeBits & (1u << memoryTypeNdx)) != 0 &&
 191                         (deviceMemProps.memoryTypes[memoryTypeNdx].propertyFlags & memoryPropertyFlag) == memoryPropertyFlag)
 192                         return memoryTypeNdx;
 193         }
 194         TCU_THROW(NotSupportedError, "No compatible memory type found");
 195 }
 196
 197 bool DeviceGroupTestInstance::isPeerFetchAllowed (deUint32 memoryTypeIndex, deUint32 firstdeviceID, deUint32 seconddeviceID)
 198 {
 199         VkPeerMemoryFeatureFlags                                peerMemFeatures1;
 200         VkPeerMemoryFeatureFlags                                peerMemFeatures2;
 201         const DeviceDriver                                              vk                                              (m_context.getInstanceInterface(), *m_deviceGroup);
 202         const VkPhysicalDeviceMemoryProperties  deviceMemProps1                 = getPhysicalDeviceMemoryProperties(m_context.getInstanceInterface(), m_physicalDevices[firstdeviceID]);
 203         const VkPhysicalDeviceMemoryProperties  deviceMemProps2                 = getPhysicalDeviceMemoryProperties(m_context.getInstanceInterface(), m_physicalDevices[seconddeviceID]);
 204         vk.getDeviceGroupPeerMemoryFeatures(*m_deviceGroup, deviceMemProps2.memoryTypes[memoryTypeIndex].heapIndex, firstdeviceID, seconddeviceID, &peerMemFeatures1);
 205         vk.getDeviceGroupPeerMemoryFeatures(*m_deviceGroup, deviceMemProps1.memoryTypes[memoryTypeIndex].heapIndex, seconddeviceID, firstdeviceID, &peerMemFeatures2);
 206         return (peerMemFeatures1 & VK_PEER_MEMORY_FEATURE_GENERIC_SRC_BIT) && (peerMemFeatures2 & VK_PEER_MEMORY_FEATURE_GENERIC_SRC_BIT);
 207 }
 208
 209 void DeviceGroupTestInstance::getDeviceLayers (vector<string>& enabledLayers)
 210 {
 211         const tcu::CommandLine& cmdLine = m_context.getTestContext().getCommandLine();
 212         if (cmdLine.isValidationEnabled())
 213         {
 214                 const vector<VkLayerProperties> layerProperties = enumerateDeviceLayerProperties(m_context.getInstanceInterface(), m_context.getPhysicalDevice());
 215
 216                 static const char*      s_magicLayer = "VK_LAYER_LUNARG_standard_validation";
 217                 static const char*      s_defaultLayers[] =
 218                 {
 219                         "VK_LAYER_GOOGLE_threading",
 220                         "VK_LAYER_LUNARG_parameter_validation",
 221                         "VK_LAYER_LUNARG_device_limits",
 222                         "VK_LAYER_LUNARG_object_tracker",
 223                         "VK_LAYER_LUNARG_image",
 224                         "VK_LAYER_LUNARG_core_validation",
 225                         "VK_LAYER_LUNARG_swapchain",
 226                         "VK_LAYER_GOOGLE_unique_objects",
 227                 };
 228
 229                 if (isLayerSupported(layerProperties, RequiredLayer(s_magicLayer)))
 230                         enabledLayers.push_back(s_magicLayer);
 231                 else
 232                 {
 233                         for (deUint32 ndx = 0; ndx < DE_LENGTH_OF_ARRAY(s_defaultLayers); ++ndx)
 234                         {
 235                                 if (isLayerSupported(layerProperties, RequiredLayer(s_defaultLayers[ndx])))
 236                                         enabledLayers.push_back(s_defaultLayers[ndx]);
 237                         }
 238                 }
 239                 if (enabledLayers.empty())
 240                         TCU_THROW(NotSupportedError, "No device validation layers found");
 241         }
 242 }
 243
 244 void DeviceGroupTestInstance::init (void)
 245 {
 246         if (!isInstanceExtensionSupported(m_context.getUsedApiVersion(), m_context.getInstanceExtensions(), "VK_KHR_device_group_creation"))
 247                 TCU_THROW(NotSupportedError, "Device Group tests are not supported, no device group extension present.");
 248
 249         const InstanceInterface&                instanceInterface       = m_context.getInstanceInterface();
 250         const deUint32                                  queueFamilyIndex        = m_context.getUniversalQueueFamilyIndex();
 251         const deUint32                                  queueIndex                      = 0;
 252         const float                                             queuePriority           = 1.0f;
 253         vector<const char*>                             extensionPtrs;
 254         de::MovePtr<vk::DeviceDriver>   deviceDriver;
 255         vector<const char*>                             layerPtrs;
 256         vector<string>                                  deviceExtensions;
 257         vector<string>                                  enabledLayers;
 258
 259         if (!isDeviceExtensionSupported(m_context.getUsedApiVersion(), m_context.getDeviceExtensions(), "VK_KHR_device_group"))
 260                 TCU_THROW(NotSupportedError, "Missing extension: VK_KHR_device_group");
 261
 262         if (!isCoreDeviceExtension(m_context.getUsedApiVersion(), "VK_KHR_device_group"))
 263                 deviceExtensions.push_back("VK_KHR_device_group");
 264
 265         if(m_useDedicated)
 266         {
 267                 if (!isDeviceExtensionSupported(m_context.getUsedApiVersion(), m_context.getDeviceExtensions(), "VK_KHR_dedicated_allocation"))
 268                         TCU_THROW(NotSupportedError, "Missing extension: VK_KHR_dedicated_allocation");
 269
 270                 if (!isCoreDeviceExtension(m_context.getUsedApiVersion(), "VK_KHR_dedicated_allocation"))
 271                         deviceExtensions.push_back("VK_KHR_dedicated_allocation");
 272         }
 273
 274         {
 275                 const tcu::CommandLine&                                                         cmdLine = m_context.getTestContext().getCommandLine();
 276                 const vector<VkPhysicalDeviceGroupProperties>           properties = enumeratePhysicalDeviceGroups(instanceInterface, m_context.getInstance());
 277                 if ((size_t)cmdLine.getVKDeviceGroupId() > properties.size())
 278                         TCU_THROW(TestError, "Invalid device group index.");
 279
 280                 m_physicalDeviceCount = properties[cmdLine.getVKDeviceGroupId() - 1].physicalDeviceCount;
 281                 for (deUint32 idx = 0; idx < m_physicalDeviceCount; idx++)
 282                 {
 283                         m_physicalDevices.push_back(properties[cmdLine.getVKDeviceGroupId() - 1].physicalDevices[idx]);
 284                 }
 285
 286                 if (m_usePeerFetch && m_physicalDeviceCount < 2)
 287                         TCU_THROW(NotSupportedError, "Peer fetching needs more than 1 physical device.");
 288
 289                 if (!(m_testMode & TEST_MODE_AFR) || (m_physicalDeviceCount > 1))
 290                 {
 291                         if (!de::contains(m_context.getDeviceExtensions().begin(), m_context.getDeviceExtensions().end(), std::string("VK_KHR_bind_memory2")))
 292                                 TCU_THROW(NotSupportedError, "Missing extension: VK_KHR_bind_memory2");
 293                         deviceExtensions.push_back("VK_KHR_bind_memory2");
 294                 }
 295
 296                 const VkDeviceQueueCreateInfo                                           deviceQueueCreateInfo =
 297                 {
 298                         VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,     //type
 299                         DE_NULL,                                                                        //pNext
 300                         (VkDeviceQueueCreateFlags)0u,                           //flags
 301                         queueFamilyIndex,                                                       //queueFamilyIndex;
 302                         1u,                                                                                     //queueCount;
 303                         &queuePriority,                                                         //pQueuePriorities;
 304                 };
 305                 const VkDeviceGroupDeviceCreateInfo             deviceGroupInfo =
 306                 {
 307                         VK_STRUCTURE_TYPE_DEVICE_GROUP_DEVICE_CREATE_INFO,                                      //stype
 308                         DE_NULL,                                                                                                                        //pNext
 309                         properties[cmdLine.getVKDeviceGroupId() - 1].physicalDeviceCount,       //physicalDeviceCount
 310                         properties[cmdLine.getVKDeviceGroupId() - 1].physicalDevices            //physicalDevices
 311                 };
 312
 313                 VkPhysicalDevice                        physicalDevice                  = properties[cmdLine.getVKDeviceGroupId() - 1].physicalDevices[(size_t)(cmdLine.getVKDeviceId() - 1)];
 314                 VkPhysicalDeviceFeatures        enabledDeviceFeatures   = getPhysicalDeviceFeatures(instanceInterface, physicalDevice);
 315                 m_subsetAllocation                                                                      = properties[cmdLine.getVKDeviceGroupId() - 1].subsetAllocation;
 316
 317                 if (m_drawTessellatedSphere & static_cast<bool>(!enabledDeviceFeatures.tessellationShader))
 318                         TCU_THROW(NotSupportedError, "Tessellation is not supported.");
 319
 320                 if (m_fillModeNonSolid & static_cast<bool>(!enabledDeviceFeatures.fillModeNonSolid))
 321                         TCU_THROW(NotSupportedError, "Line polygon mode is not supported.");
 322
 323                 extensionPtrs.resize(deviceExtensions.size());
 324                 for (size_t ndx = 0; ndx < deviceExtensions.size(); ++ndx)
 325                         extensionPtrs[ndx] = deviceExtensions[ndx].c_str();
 326
 327                 // Get Layers
 328                 getDeviceLayers(enabledLayers);
 329                 layerPtrs.resize(enabledLayers.size());
 330                 for (size_t ndx = 0; ndx < enabledLayers.size(); ++ndx)
 331                         layerPtrs[ndx] = enabledLayers[ndx].c_str();
 332
 333                 const VkDeviceCreateInfo        deviceCreateInfo =
 334                 {
 335                         VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,                                   //sType;
 336                         &deviceGroupInfo,                                                                               //pNext;
 337                         (VkDeviceCreateFlags)0u,                                                                //flags
 338                         1,                                                                                                              //queueRecordCount;
 339                         &deviceQueueCreateInfo,                                                                 //pRequestedQueues;
 340                         (deUint32)layerPtrs.size(),                                                             //layerCount;
 341                         (layerPtrs.empty() ? DE_NULL : &layerPtrs[0]),                  //ppEnabledLayerNames;
 342                         (deUint32)extensionPtrs.size(),                                                 //extensionCount;
 343                         (extensionPtrs.empty() ? DE_NULL : &extensionPtrs[0]),  //ppEnabledExtensionNames;
 344                         &enabledDeviceFeatures,                                                                 //pEnabledFeatures;
 345                 };
 346                 m_deviceGroup = createDevice(instanceInterface, physicalDevice, &deviceCreateInfo);
 347         }
 348
 349         deviceDriver = de::MovePtr<vk::DeviceDriver>(new vk::DeviceDriver(instanceInterface, *m_deviceGroup));
 350         m_deviceGroupQueue = getDeviceQueue(*deviceDriver, *m_deviceGroup, queueFamilyIndex, queueIndex);
 351 }
 352
 353 tcu::TestStatus DeviceGroupTestInstance::iterate (void)
 354 {
 355         const InstanceInterface&        vki                                             (m_context.getInstanceInterface());
 356         const DeviceDriver                      vk                                              (vki, *m_deviceGroup);
 357         const deUint32                          queueFamilyIndex                = m_context.getUniversalQueueFamilyIndex();
 358         const tcu::UVec2                        renderSize                              (256, 256);
 359         const VkFormat                          colorFormat                             = VK_FORMAT_R8G8B8A8_UNORM;
 360         const tcu::Vec4                         clearColor                              (0.125f, 0.25f, 0.75f, 1.0f);
 361         const tcu::Vec4                         drawColor                               (1.0f, 1.0f, 0.0f, 1.0f);
 362         const float                                     tessLevel                               = 16.0f;
 363         SimpleAllocator                         memAlloc                                (vk, *m_deviceGroup, getPhysicalDeviceMemoryProperties(m_context.getInstanceInterface(), m_context.getPhysicalDevice()));
 364         bool                                            iterateResultSuccess    = false;
 365         const tcu::Vec4                         sphereVertices[]                =
 366         {
 367                 tcu::Vec4(0.0f, 0.0f, 1.0f, 1.0f),
 368                 tcu::Vec4(0.0f, 1.0f, 0.0f, 1.0f),
 369                 tcu::Vec4(1.0f, 0.0f, 0.0f, 1.0f),
 370                 tcu::Vec4(0.0f, 0.0f, -1.0f, 1.0f),
 371                 tcu::Vec4(0.0f, -1.0f, 0.0f, 1.0f),
 372                 tcu::Vec4(-1.0f, 0.0f, 0.0f, 1.0f),
 373         };
 374         const deUint32                          sphereIndices[]                 = {0, 1, 2, 2, 1, 3, 3, 1, 5, 5, 1, 0, 0, 2, 4, 2, 3, 4, 3, 5, 4, 5, 0, 4};
 375         const tcu::Vec4                         triVertices[]                   =
 376         {
 377                 tcu::Vec4(-0.5f, -0.5f, 0.0f, 1.0f),
 378                 tcu::Vec4(+0.5f, -0.5f, 0.0f, 1.0f),
 379                 tcu::Vec4(0.0f, +0.5f, 0.0f, 1.0f)
 380         };
 381         const deUint32                          triIndices[]                    = {0, 1, 2};
 382         const tcu::Vec4 *                       vertices                                = m_drawTessellatedSphere ? &sphereVertices[0] : &triVertices[0];
 383         const deUint32 *                        indices                                 = m_drawTessellatedSphere ? &sphereIndices[0] : &triIndices[0];
 384         const deUint32                          verticesSize                    = m_drawTessellatedSphere ? deUint32(sizeof(sphereVertices)) : deUint32(sizeof(triVertices));
 385         const deUint32                          numIndices                              = m_drawTessellatedSphere ? deUint32(sizeof(sphereIndices)/sizeof(sphereIndices[0])) : deUint32(sizeof(triIndices)/sizeof(triIndices[0]));
 386         const deUint32                          indicesSize                             = m_drawTessellatedSphere ? deUint32(sizeof(sphereIndices)) : deUint32(sizeof(triIndices));
 387
 388         // Loop through all physical devices in the device group
 389         for (deUint32 physDevID = 0; physDevID < m_physicalDeviceCount; physDevID++)
 390         {
 391                 const deUint32                                  firstDeviceID           = physDevID;
 392                 const deUint32                                  secondDeviceID          = (firstDeviceID + 1 ) % m_physicalDeviceCount;
 393                 vector<deUint32>                                deviceIndices             (m_physicalDeviceCount);
 394
 395                 // Set broadcast on memory allocation
 396                 const deUint32                                  allocDeviceMask         = m_subsetAllocation ? (1 << firstDeviceID) | (1 << secondDeviceID) : (1 << m_physicalDeviceCount) - 1;
 397
 398                 for (deUint32 i = 0; i < m_physicalDeviceCount; i++)
 399                         deviceIndices[i] = i;
 400                 deviceIndices[firstDeviceID] = secondDeviceID;
 401                 deviceIndices[secondDeviceID] = firstDeviceID;
 402
 403                 VkMemoryRequirements                    memReqs                         =
 404                 {
 405                         0,                                                      // VkDeviceSize         size
 406                         0,                                                      // VkDeviceSize         alignment
 407                         0,                                                      // uint32_t                     memoryTypeBits
 408                 };
 409                 deUint32                                                memoryTypeNdx           = 0;
 410                 de::MovePtr<Allocation>                 stagingVertexBufferMemory;
 411                 de::MovePtr<Allocation>                 stagingIndexBufferMemory;
 412                 de::MovePtr<Allocation>                 stagingUniformBufferMemory;
 413                 de::MovePtr<Allocation>                 stagingSboBufferMemory;
 414
 415                 vk::Move<vk::VkDeviceMemory>    vertexBufferMemory;
 416                 vk::Move<vk::VkDeviceMemory>    indexBufferMemory;
 417                 vk::Move<vk::VkDeviceMemory>    uniformBufferMemory;
 418                 vk::Move<vk::VkDeviceMemory>    sboBufferMemory;
 419                 vk::Move<vk::VkDeviceMemory>    imageMemory;
 420
 421                 Move<VkRenderPass>                              renderPass;
 422                 Move<VkImage>                                   renderImage;
 423                 Move<VkImage>                                   readImage;
 424
 425                 Move<VkDescriptorSetLayout>             descriptorSetLayout;
 426                 Move<VkDescriptorPool>                  descriptorPool;
 427                 Move<VkDescriptorSet>                   descriptorSet;
 428
 429                 Move<VkBuffer>                                  stagingVertexBuffer;
 430                 Move<VkBuffer>                                  stagingUniformBuffer;
 431                 Move<VkBuffer>                                  stagingIndexBuffer;
 432                 Move<VkBuffer>                                  stagingSboBuffer;
 433
 434                 Move<VkBuffer>                                  vertexBuffer;
 435                 Move<VkBuffer>                                  indexBuffer;
 436                 Move<VkBuffer>                                  uniformBuffer;
 437                 Move<VkBuffer>                                  sboBuffer;
 438
 439                 Move<VkPipeline>                                pipeline;
 440                 Move<VkPipelineLayout>                  pipelineLayout;
 441
 442                 Move<VkImageView>                               colorAttView;
 443                 Move<VkFramebuffer>                             framebuffer;
 444                 Move<VkCommandPool>                             cmdPool;
 445                 Move<VkCommandBuffer>                   cmdBuffer;
 446
 447                 VkMemoryDedicatedAllocateInfo   dedicatedAllocInfo =
 448                 {
 449                                 VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,               // sType
 450                                 DE_NULL,                                                                                                // pNext
 451                                 DE_NULL,                                                                                                // image
 452                                 DE_NULL                                                                                                 // buffer
 453                 };
 454
 455                 VkMemoryAllocateFlagsInfo               allocDeviceMaskInfo =
 456                 {
 457                         VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO,           // sType
 458                         m_useDedicated ? &dedicatedAllocInfo : DE_NULL,         // pNext
 459                         VK_MEMORY_ALLOCATE_DEVICE_MASK_BIT,                                     // flags
 460                         allocDeviceMask,                                                                        // deviceMask
 461                 };
 462
 463                 VkMemoryAllocateInfo            allocInfo =
 464                 {
 465                         VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,                 // sType
 466                         &allocDeviceMaskInfo,                                                   // pNext
 467                         0u,                                                                                             // allocationSize
 468                         0u,                                                                                             // memoryTypeIndex
 469                 };
 470
 471                 VkDeviceGroupSubmitInfo         deviceGroupSubmitInfo =
 472                 {
 473                         VK_STRUCTURE_TYPE_DEVICE_GROUP_SUBMIT_INFO,             // sType
 474                         DE_NULL,                                                                                // pNext
 475                         0u,                                                                                             // waitSemaphoreCount
 476                         DE_NULL,                                                                                // pWaitSemaphoreDeviceIndices
 477                         0u,                                                                                             // commandBufferCount
 478                         DE_NULL,                                                                                // pCommandBufferDeviceMasks
 479                         0u,                                                                                             // signalSemaphoreCount
 480                         DE_NULL,                                                                                // pSignalSemaphoreDeviceIndices
 481                 };
 482
 483                 // create vertex buffers
 484                 {
 485                         const VkBufferCreateInfo        stagingVertexBufferParams =
 486                         {
 487                                 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,                                                                   // sType
 488                                 DE_NULL,                                                                                                                                // pNext
 489                                 0u,                                                                                                                                             // flags
 490                                 (VkDeviceSize)verticesSize,                                                                                             // size
 491                                 VK_BUFFER_USAGE_TRANSFER_SRC_BIT,                                                                               // usage
 492                                 VK_SHARING_MODE_EXCLUSIVE,                                                                                              // sharingMode
 493                                 1u,                                                                                                                                             // queueFamilyIndexCount
 494                                 &queueFamilyIndex,                                                                                                              // pQueueFamilyIndices
 495                         };
 496                         stagingVertexBuffer = createBuffer(vk, *m_deviceGroup, &stagingVertexBufferParams);
 497                         stagingVertexBufferMemory = memAlloc.allocate(getBufferMemoryRequirements(vk, *m_deviceGroup, *stagingVertexBuffer), MemoryRequirement::HostVisible);
 498                         VK_CHECK(vk.bindBufferMemory(*m_deviceGroup, *stagingVertexBuffer, stagingVertexBufferMemory->getMemory(), stagingVertexBufferMemory->getOffset()));
 499
 500                         const VkMappedMemoryRange       range   =
 501                         {
 502                                 VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,  // sType
 503                                 DE_NULL,                                                                // pNext
 504                                 stagingVertexBufferMemory->getMemory(), // memory
 505                                 0u,                                                                             // offset
 506                                 (VkDeviceSize)verticesSize,                             // size
 507                         };
 508                         void*   vertexBufPtr    = stagingVertexBufferMemory->getHostPtr();
 509                         deMemcpy(vertexBufPtr, &vertices[0], verticesSize);
 510                         VK_CHECK(vk.flushMappedMemoryRanges(*m_deviceGroup, 1u, &range));
 511                 }
 512
 513                 {
 514                         const VkBufferCreateInfo        vertexBufferParams =
 515                         {
 516                                 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,                                                                   // sType
 517                                 DE_NULL,                                                                                                                                // pNext
 518                                 0u,                                                                                                                                             // flags
 519                                 (VkDeviceSize)verticesSize,                                                                                             // size
 520                                 VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,   // usage
 521                                 VK_SHARING_MODE_EXCLUSIVE,                                                                                              // sharingMode
 522                                 1u,                                                                                                                                             // queueFamilyIndexCount
 523                                 &queueFamilyIndex,                                                                                                              // pQueueFamilyIndices
 524                         };
 525                         vertexBuffer = createBuffer(vk, *m_deviceGroup, &vertexBufferParams);
 526
 527                         memReqs = getBufferMemoryRequirements(vk, *m_deviceGroup, vertexBuffer.get());
 528                         memoryTypeNdx = getMemoryIndex(memReqs.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
 529
 530                         dedicatedAllocInfo.buffer = vertexBuffer.get();
 531                         allocInfo.allocationSize = memReqs.size;
 532                         allocInfo.memoryTypeIndex = memoryTypeNdx;
 533                         vertexBufferMemory = allocateMemory(vk, *m_deviceGroup, &allocInfo);
 534
 535                         if (m_usePeerFetch && !isPeerFetchAllowed(memoryTypeNdx, firstDeviceID, secondDeviceID))
 536                                 TCU_THROW(NotSupportedError, "Peer fetch is not supported.");
 537
 538                         // Bind vertex buffer
 539                         if (m_usePeerFetch)
 540                         {
 541                                 VkBindBufferMemoryDeviceGroupInfo       devGroupBindInfo =
 542                                 {
 543                                         VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_DEVICE_GROUP_INFO,         // sType
 544                                         DE_NULL,                                                                                                        // pNext
 545                                         m_physicalDeviceCount,                                                                          // deviceIndexCount
 546                                         &deviceIndices[0],                                                                                      // pDeviceIndices
 547                                 };
 548
 549                                 VkBindBufferMemoryInfo                          bindInfo =
 550                                 {
 551                                         VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,                                      // sType
 552                                         &devGroupBindInfo,                                                                                      // pNext
 553                                         vertexBuffer.get(),                                                                                     // buffer
 554                                         vertexBufferMemory.get(),                                                                       // memory
 555                                         0u,                                                                                                                     // memoryOffset
 556                                 };
 557                                 VK_CHECK(vk.bindBufferMemory2(*m_deviceGroup, 1, &bindInfo));
 558                         }
 559                         else
 560                                 VK_CHECK(vk.bindBufferMemory(*m_deviceGroup, *vertexBuffer, vertexBufferMemory.get(), 0));
 561                 }
 562
 563                 // create index buffers
 564                 {
 565                         const VkBufferCreateInfo        stagingIndexBufferParams =
 566                         {
 567                                 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,                                                                   // sType
 568                                 DE_NULL,                                                                                                                                // pNext
 569                                 0u,                                                                                                                                             // flags
 570                                 (VkDeviceSize)indicesSize,                                                                                              // size
 571                                 VK_BUFFER_USAGE_TRANSFER_SRC_BIT,                                                                               // usage
 572                                 VK_SHARING_MODE_EXCLUSIVE,                                                                                              // sharingMode
 573                                 1u,                                                                                                                                             // queueFamilyIndexCount
 574                                 &queueFamilyIndex,                                                                                                              // pQueueFamilyIndices
 575                         };
 576                         stagingIndexBuffer = createBuffer(vk, *m_deviceGroup, &stagingIndexBufferParams);
 577                         stagingIndexBufferMemory = memAlloc.allocate(getBufferMemoryRequirements(vk, *m_deviceGroup, *stagingIndexBuffer), MemoryRequirement::HostVisible);
 578                         VK_CHECK(vk.bindBufferMemory(*m_deviceGroup, *stagingIndexBuffer, stagingIndexBufferMemory->getMemory(), stagingIndexBufferMemory->getOffset()));
 579
 580                         const VkMappedMemoryRange       range   =
 581                         {
 582                                 VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,  // sType
 583                                 DE_NULL,                                                                // pNext
 584                                 stagingIndexBufferMemory->getMemory(),  // memory
 585                                 0u,                                                                             // offset
 586                                 (VkDeviceSize)indicesSize,                              // size
 587                         };
 588                         void*   indexBufPtr     = stagingIndexBufferMemory->getHostPtr();
 589                         deMemcpy(indexBufPtr, &indices[0], indicesSize);
 590                         VK_CHECK(vk.flushMappedMemoryRanges(*m_deviceGroup, 1u, &range));
 591                 }
 592
 593                 {
 594                         const VkBufferCreateInfo        indexBufferParams =
 595                         {
 596                                 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,                                                                   // sType
 597                                 DE_NULL,                                                                                                                                // pNext
 598                                 0u,                                                                                                                                             // flags
 599                                 (VkDeviceSize)indicesSize,                                                                                              // size
 600                                 VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,    // usage
 601                                 VK_SHARING_MODE_EXCLUSIVE,                                                                                              // sharingMode
 602                                 1u,                                                                                                                                             // queueFamilyIndexCount
 603                                 &queueFamilyIndex,                                                                                                              // pQueueFamilyIndices
 604                         };
 605                         indexBuffer = createBuffer(vk, *m_deviceGroup, &indexBufferParams);
 606
 607                         memReqs = getBufferMemoryRequirements(vk, *m_deviceGroup, indexBuffer.get());
 608                         memoryTypeNdx = getMemoryIndex(memReqs.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
 609
 610                         dedicatedAllocInfo.buffer = indexBuffer.get();
 611                         allocInfo.allocationSize = memReqs.size;
 612                         allocInfo.memoryTypeIndex = memoryTypeNdx;
 613                         indexBufferMemory = allocateMemory(vk, *m_deviceGroup, &allocInfo);
 614
 615                         if (m_usePeerFetch && !isPeerFetchAllowed(memoryTypeNdx, firstDeviceID, secondDeviceID))
 616                                 TCU_THROW(NotSupportedError, "Peer fetch is not supported.");
 617
 618                         // Bind index buffer
 619                         if (m_usePeerFetch)
 620                         {
 621                                 VkBindBufferMemoryDeviceGroupInfo       devGroupBindInfo =
 622                                 {
 623                                         VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_DEVICE_GROUP_INFO,         // sType
 624                                         DE_NULL,                                                                                                        // pNext
 625                                         m_physicalDeviceCount,                                                                          // deviceIndexCount
 626                                         &deviceIndices[0],                                                                                      // pDeviceIndices
 627                                 };
 628
 629                                 VkBindBufferMemoryInfo                          bindInfo =
 630                                 {
 631                                         VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,                                      // sType
 632                                         &devGroupBindInfo,                                                                                      // pNext
 633                                         indexBuffer.get(),                                                                                      // buffer
 634                                         indexBufferMemory.get(),                                                                        // memory
 635                                         0u,                                                                                                                     // memoryOffset
 636                                 };
 637                                 VK_CHECK(vk.bindBufferMemory2(*m_deviceGroup, 1, &bindInfo));
 638                         }
 639                         else
 640                                 VK_CHECK(vk.bindBufferMemory(*m_deviceGroup, *indexBuffer, indexBufferMemory.get(), 0));
 641                 }
 642
 643                 // create uniform buffers
 644                 {
 645                         const VkBufferCreateInfo        stagingUniformBufferParams =
 646                         {
 647                                 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,                                                                   // sType
 648                                 DE_NULL,                                                                                                                                // pNext
 649                                 0u,                                                                                                                                             // flags
 650                                 (VkDeviceSize)sizeof(drawColor),                                                                                                // size
 651                                 VK_BUFFER_USAGE_TRANSFER_SRC_BIT,                                                                               // usage
 652                                 VK_SHARING_MODE_EXCLUSIVE,                                                                                              // sharingMode
 653                                 1u,                                                                                                                                             // queueFamilyIndexCount
 654                                 &queueFamilyIndex,                                                                                                              // pQueueFamilyIndices
 655                         };
 656                         stagingUniformBuffer = createBuffer(vk, *m_deviceGroup, &stagingUniformBufferParams);
 657                         stagingUniformBufferMemory = memAlloc.allocate(getBufferMemoryRequirements(vk, *m_deviceGroup, *stagingUniformBuffer), MemoryRequirement::HostVisible);
 658                         VK_CHECK(vk.bindBufferMemory(*m_deviceGroup, *stagingUniformBuffer, stagingUniformBufferMemory->getMemory(), stagingUniformBufferMemory->getOffset()));
 659
 660                         const VkMappedMemoryRange       range   =
 661                         {
 662                                 VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,  // sType
 663                                 DE_NULL,                                                                // pNext
 664                                 stagingUniformBufferMemory->getMemory(),// memory
 665                                 0u,                                                                             // offset
 666                                 (VkDeviceSize)sizeof(drawColor),                // size
 667                         };
 668                         void*   uniformBufPtr   = stagingUniformBufferMemory->getHostPtr();
 669                         deMemcpy(uniformBufPtr, &drawColor[0], sizeof(drawColor));
 670                         VK_CHECK(vk.flushMappedMemoryRanges(*m_deviceGroup, 1u, &range));
 671                 }
 672
 673                 {
 674                         const VkBufferCreateInfo        uniformBufferParams =
 675                         {
 676                                 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,                                                                   // sType
 677                                 DE_NULL,                                                                                                                                // pNext
 678                                 0u,                                                                                                                                             // flags
 679                                 (VkDeviceSize)sizeof(drawColor),                                                                                // size
 680                                 VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,  // usage
 681                                 VK_SHARING_MODE_EXCLUSIVE,                                                                                              // sharingMode
 682                                 1u,                                                                                                                                             // queueFamilyIndexCount
 683                                 &queueFamilyIndex,                                                                                                              // pQueueFamilyIndices
 684                         };
 685                         uniformBuffer = createBuffer(vk, *m_deviceGroup, &uniformBufferParams);
 686
 687                         memReqs = getBufferMemoryRequirements(vk, *m_deviceGroup, uniformBuffer.get());
 688                         memoryTypeNdx = getMemoryIndex(memReqs.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
 689
 690                         dedicatedAllocInfo.buffer = uniformBuffer.get();
 691                         allocInfo.allocationSize = memReqs.size;
 692                         allocInfo.memoryTypeIndex = memoryTypeNdx;
 693                         uniformBufferMemory = allocateMemory(vk, *m_deviceGroup, &allocInfo);
 694
 695                         if (m_usePeerFetch && !isPeerFetchAllowed(memoryTypeNdx, firstDeviceID, secondDeviceID))
 696                                 TCU_THROW(NotSupportedError, "Peer fetch is not supported.");
 697
 698                         if (m_usePeerFetch)
 699                         {
 700                                 VkBindBufferMemoryDeviceGroupInfo       devGroupBindInfo =
 701                                 {
 702                                         VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_DEVICE_GROUP_INFO,         // sType
 703                                         DE_NULL,                                                                                                        // pNext
 704                                         m_physicalDeviceCount,                                                                          // deviceIndexCount
 705                                         &deviceIndices[0],                                                                                      // pDeviceIndices
 706                                 };
 707
 708                                 VkBindBufferMemoryInfo                          bindInfo =
 709                                 {
 710                                         VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,                                      // sType
 711                                         &devGroupBindInfo,                                                                                      // pNext
 712                                         uniformBuffer.get(),                                                                            // buffer
 713                                         uniformBufferMemory.get(),                                                                      // memory
 714                                         0u,                                                                                                                     // memoryOffset
 715                                 };
 716                                 VK_CHECK(vk.bindBufferMemory2(*m_deviceGroup, 1, &bindInfo));
 717                         }
 718                         else
 719                                 VK_CHECK(vk.bindBufferMemory(*m_deviceGroup, uniformBuffer.get(), uniformBufferMemory.get(), 0));
 720                 }
 721
 722                 // create SBO buffers
 723                 {
 724                         const VkBufferCreateInfo        stagingSboBufferParams =
 725                         {
 726                                 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,                                                                   // sType
 727                                 DE_NULL,                                                                                                                                // pNext
 728                                 0u,                                                                                                                                             // flags
 729                                 (VkDeviceSize)sizeof(tessLevel),                                                                                // size
 730                                 VK_BUFFER_USAGE_TRANSFER_SRC_BIT,                                                                               // usage
 731                                 VK_SHARING_MODE_EXCLUSIVE,                                                                                              // sharingMode
 732                                 1u,                                                                                                                                             // queueFamilyIndexCount
 733                                 &queueFamilyIndex,                                                                                                              // pQueueFamilyIndices
 734                         };
 735                         stagingSboBuffer = createBuffer(vk, *m_deviceGroup, &stagingSboBufferParams);
 736                         stagingSboBufferMemory = memAlloc.allocate(getBufferMemoryRequirements(vk, *m_deviceGroup, *stagingSboBuffer), MemoryRequirement::HostVisible);
 737                         VK_CHECK(vk.bindBufferMemory(*m_deviceGroup, *stagingSboBuffer, stagingSboBufferMemory->getMemory(), stagingSboBufferMemory->getOffset()));
 738
 739                         const VkMappedMemoryRange       range   =
 740                         {
 741                                 VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,  // sType
 742                                 DE_NULL,                                                                // pNext
 743                                 stagingSboBufferMemory->getMemory(),    // memory
 744                                 0u,                                                                             // offset
 745                                 (VkDeviceSize)sizeof(tessLevel),                // size
 746                         };
 747                         void*   sboBufPtr       = stagingSboBufferMemory->getHostPtr();
 748                         deMemcpy(sboBufPtr, &tessLevel, sizeof(tessLevel));
 749                         VK_CHECK(vk.flushMappedMemoryRanges(*m_deviceGroup, 1u, &range));
 750                 }
 751
 752                 {
 753                         const VkBufferCreateInfo        sboBufferParams =
 754                         {
 755                                 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,                                                                   // sType
 756                                 DE_NULL,                                                                                                                                // pNext
 757                                 0u,                                                                                                                                             // flags
 758                                 (VkDeviceSize)sizeof(tessLevel),                                                                                // size
 759                                 VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,  // usage
 760                                 VK_SHARING_MODE_EXCLUSIVE,                                                                                              // sharingMode
 761                                 1u,                                                                                                                                             // queueFamilyIndexCount
 762                                 &queueFamilyIndex,                                                                                                              // pQueueFamilyIndices
 763                         };
 764                         sboBuffer = createBuffer(vk, *m_deviceGroup, &sboBufferParams);
 765
 766                         memReqs = getBufferMemoryRequirements(vk, *m_deviceGroup, sboBuffer.get());
 767                         memoryTypeNdx = getMemoryIndex(memReqs.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
 768
 769                         dedicatedAllocInfo.buffer = sboBuffer.get();
 770                         allocInfo.allocationSize = memReqs.size;
 771                         allocInfo.memoryTypeIndex = memoryTypeNdx;
 772                         sboBufferMemory = allocateMemory(vk, *m_deviceGroup, &allocInfo);
 773
 774                         if (m_usePeerFetch && !isPeerFetchAllowed(memoryTypeNdx, firstDeviceID, secondDeviceID))
 775                                 TCU_THROW(NotSupportedError, "Peer fetch is not supported.");
 776
 777                         if (m_usePeerFetch)
 778                         {
 779                                 VkBindBufferMemoryDeviceGroupInfo       devGroupBindInfo =
 780                                 {
 781                                         VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_DEVICE_GROUP_INFO,         // sType
 782                                         DE_NULL,                                                                                                        // pNext
 783                                         m_physicalDeviceCount,                                                                          // deviceIndexCount
 784                                         &deviceIndices[0],                                                                                      // pDeviceIndices
 785                                 };
 786
 787                                 VkBindBufferMemoryInfo                          bindInfo =
 788                                 {
 789                                         VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,                                      // sType
 790                                         &devGroupBindInfo,                                                                                      // pNext
 791                                         sboBuffer.get(),                                                                                        // buffer
 792                                         sboBufferMemory.get(),                                                                          // memory
 793                                         0u,                                                                                                                     // memoryOffset
 794                                 };
 795                                 VK_CHECK(vk.bindBufferMemory2(*m_deviceGroup, 1, &bindInfo));
 796                         }
 797                         else
 798                                 VK_CHECK(vk.bindBufferMemory(*m_deviceGroup, sboBuffer.get(), sboBufferMemory.get(), 0));
 799                 }
 800
 801                 // Create image resources
 802                 // Use a consistent usage flag because of memory aliasing
 803                 VkImageUsageFlags imageUsageFlag = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
 804                 {
 805                         // Check for SFR support
 806                         VkImageFormatProperties properties;
 807                         if ((m_testMode & TEST_MODE_SFR) && vki.getPhysicalDeviceImageFormatProperties(m_context.getPhysicalDevice(),
 808                                 colorFormat,                                                                                                                    // format
 809                                 VK_IMAGE_TYPE_2D,                                                                                                               // type
 810                                 VK_IMAGE_TILING_OPTIMAL,                                                                                                // tiling
 811                                 VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT,  // usage
 812                                 VK_IMAGE_CREATE_BIND_SFR_BIT,                                                                                   // flags
 813                                 &properties) != VK_SUCCESS)                                                                                             // properties
 814                         {
 815                                 TCU_THROW(NotSupportedError, "Format not supported for SFR");
 816                         }
 817
 818                         VkImageCreateFlags      imageCreateFlags = VK_IMAGE_CREATE_ALIAS_BIT;   // The image objects alias same memory
 819                         if (m_testMode & TEST_MODE_SFR)
 820                         {
 821                                 imageCreateFlags |= VK_IMAGE_CREATE_BIND_SFR_BIT;
 822                         }
 823
 824                         const VkImageCreateInfo         imageParams =
 825                         {
 826                                 VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,                                                                    // sType
 827                                 DE_NULL,                                                                                                                                // pNext
 828                                 imageCreateFlags,                                                                                                               // flags
 829                                 VK_IMAGE_TYPE_2D,                                                                                                               // imageType
 830                                 colorFormat,                                                                                                                    // format
 831                                 { renderSize.x(), renderSize.y(), 1 },                                                                  // extent
 832                                 1u,                                                                                                                                             // mipLevels
 833                                 1u,                                                                                                                                             // arraySize
 834                                 VK_SAMPLE_COUNT_1_BIT,                                                                                                  // samples
 835                                 VK_IMAGE_TILING_OPTIMAL,                                                                                                // tiling
 836                                 imageUsageFlag,                                                                                                                 // usage
 837                                 VK_SHARING_MODE_EXCLUSIVE,                                                                                              // sharingMode
 838                                 1u,                                                                                                                                             // queueFamilyIndexCount
 839                                 &queueFamilyIndex,                                                                                                              // pQueueFamilyIndices
 840                                 VK_IMAGE_LAYOUT_UNDEFINED,                                                                                              // initialLayout
 841                         };
 842
 843                         renderImage = createImage(vk, *m_deviceGroup, &imageParams);
 844                         readImage = createImage(vk, *m_deviceGroup, &imageParams);
 845
 846                         dedicatedAllocInfo.image = *renderImage;
 847                         dedicatedAllocInfo.buffer = DE_NULL;
 848                         memReqs = getImageMemoryRequirements(vk, *m_deviceGroup, renderImage.get());
 849                         memoryTypeNdx = getMemoryIndex(memReqs.memoryTypeBits, m_useHostMemory ? 0 : VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
 850                         allocInfo.allocationSize = memReqs.size;
 851                         allocInfo.memoryTypeIndex = memoryTypeNdx;
 852                         imageMemory = allocateMemory(vk, *m_deviceGroup, &allocInfo);
 853                 }
 854
 855                 if (m_testMode & TEST_MODE_SFR)
 856                 {
 857                         if (m_usePeerFetch && !isPeerFetchAllowed(memoryTypeNdx, firstDeviceID, secondDeviceID))
 858                                 TCU_THROW(NotSupportedError, "Peer texture reads is not supported.");
 859
 860                         VkRect2D zeroRect = {
 861                                 {
 862                                         0,      //      VkOffset2D.x
 863                                         0,      //      VkOffset2D.x
 864                                 },
 865                                 {
 866                                         0,      //      VkExtent2D.x
 867                                         0,      //      VkExtent2D.x
 868                                 }
 869                         };
 870                         vector<VkRect2D> sfrRects;
 871                         for (deUint32 i = 0; i < m_physicalDeviceCount*m_physicalDeviceCount; i++)
 872                                 sfrRects.push_back(zeroRect);
 873
 874                         if (m_physicalDeviceCount == 1u)
 875                         {
 876                                 sfrRects[0].extent.width        = (deInt32)renderSize.x();
 877                                 sfrRects[0].extent.height       = (deInt32)renderSize.y();
 878                         }
 879                         else
 880                         {
 881                                 // Split into 2 vertical halves
 882                                 sfrRects[firstDeviceID * m_physicalDeviceCount + firstDeviceID].extent.width    = (deInt32)renderSize.x() / 2;
 883                                 sfrRects[firstDeviceID * m_physicalDeviceCount + firstDeviceID].extent.height   = (deInt32)renderSize.y();
 884                                 sfrRects[firstDeviceID * m_physicalDeviceCount + secondDeviceID]                                = sfrRects[firstDeviceID * m_physicalDeviceCount + firstDeviceID];
 885                                 sfrRects[firstDeviceID * m_physicalDeviceCount + secondDeviceID].offset.x               = (deInt32)renderSize.x() / 2;
 886                                 sfrRects[secondDeviceID * m_physicalDeviceCount + firstDeviceID]                                = sfrRects[firstDeviceID * m_physicalDeviceCount + firstDeviceID];
 887                                 sfrRects[secondDeviceID * m_physicalDeviceCount + secondDeviceID]                               = sfrRects[firstDeviceID * m_physicalDeviceCount + secondDeviceID];
 888                         }
 889
 890                         VkBindImageMemoryDeviceGroupInfo        devGroupBindInfo =
 891                         {
 892                                 VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_DEVICE_GROUP_INFO,          // sType
 893                                 DE_NULL,                                                                                                        // pNext
 894                                 0u,                                                                                                                     // deviceIndexCount
 895                                 DE_NULL,                                                                                                        // pDeviceIndices
 896                                 m_physicalDeviceCount*m_physicalDeviceCount,                            // SFRRectCount
 897                                 &sfrRects[0],                                                                                           // pSFRRects
 898                         };
 899
 900                         VkBindImageMemoryInfo                           bindInfo =
 901                         {
 902                                 VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO,                                       // sType
 903                                 &devGroupBindInfo,                                                                                      // pNext
 904                                 *renderImage,                                                                                           // image
 905                                 imageMemory.get(),                                                                                      // memory
 906                                 0u,                                                                                                                     // memoryOffset
 907                         };
 908                         VK_CHECK(vk.bindImageMemory2(*m_deviceGroup, 1, &bindInfo));
 909                 }
 910                 else
 911                         VK_CHECK(vk.bindImageMemory(*m_deviceGroup, *renderImage, imageMemory.get(), 0));
 912
 913                 VK_CHECK(vk.bindImageMemory(*m_deviceGroup, *readImage, imageMemory.get(), 0));
 914
 915                 // Create renderpass
 916                 {
 917                         const VkAttachmentDescription                   colorAttDesc =
 918                         {
 919                                 0u,                                                                                             // flags
 920                                 colorFormat,                                                                    // format
 921                                 VK_SAMPLE_COUNT_1_BIT,                                                  // samples
 922                                 VK_ATTACHMENT_LOAD_OP_CLEAR,                                    // loadOp
 923                                 VK_ATTACHMENT_STORE_OP_STORE,                                   // storeOp
 924                                 VK_ATTACHMENT_LOAD_OP_DONT_CARE,                                // stencilLoadOp
 925                                 VK_ATTACHMENT_STORE_OP_DONT_CARE,                               // stencilStoreOp
 926                                 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,               // initialLayout
 927                                 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,               // finalLayout
 928                         };
 929                         const VkAttachmentReference                             colorAttRef =
 930                         {
 931                                 0u,                                                                                             // attachment
 932                                 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,               // layout
 933                         };
 934                         const VkSubpassDescription                              subpassDesc =
 935                         {
 936                                 (VkSubpassDescriptionFlags)0u,                                  // flags
 937                                 VK_PIPELINE_BIND_POINT_GRAPHICS,                                // pipelineBindPoint
 938                                 0u,                                                                                             // inputAttachmentCount
 939                                 DE_NULL,                                                                                // pInputAttachments
 940                                 1u,                                                                                             // colorAttachmentCount
 941                                 &colorAttRef,                                                                   // pColorAttachments
 942                                 DE_NULL,                                                                                // pResolveAttachments
 943                                 DE_NULL,                                                                                // depthStencilAttachment
 944                                 0u,                                                                                             // preserveAttachmentCount
 945                                 DE_NULL,                                                                                // pPreserveAttachments
 946                         };
 947                         const VkRenderPassCreateInfo                    renderPassParams =
 948                         {
 949                                 VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,              // sType
 950                                 DE_NULL,                                                                                // pNext
 951                                 0u,                                                                                             // flags
 952                                 1u,                                                                                             // attachmentCount
 953                                 &colorAttDesc,                                                                  // pAttachments
 954                                 1u,                                                                                             // subpassCount
 955                                 &subpassDesc,                                                                   // pSubpasses
 956                                 0u,                                                                                             // dependencyCount
 957                                 DE_NULL,                                                                                // pDependencies
 958                         };
 959                         renderPass = createRenderPass(vk, *m_deviceGroup, &renderPassParams);
 960                 }
 961
 962                 // Create descriptors
 963                 {
 964                         vector<VkDescriptorSetLayoutBinding>    layoutBindings;
 965                         vector<VkDescriptorPoolSize>                    descriptorTypes;
 966                         vector<VkWriteDescriptorSet>                    writeDescritporSets;
 967
 968                         const VkDescriptorSetLayoutBinding layoutBindingUBO =
 969                         {
 970                                 0u,                                                                                     // deUint32                             binding;
 971                                 VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,                      // VkDescriptorType             descriptorType;
 972                                 1u,                                                                                     // deUint32                             descriptorCount;
 973                                 VK_SHADER_STAGE_FRAGMENT_BIT,                           // VkShaderStageFlags   stageFlags;
 974                                 DE_NULL                                                                         // const VkSampler*             pImmutableSamplers;
 975                         };
 976                         const VkDescriptorSetLayoutBinding layoutBindingSBO =
 977                         {
 978                                 1u,                                                                                     // deUint32                             binding;
 979                                 VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,                      // VkDescriptorType             descriptorType;
 980                                 1u,                                                                                     // deUint32                             descriptorCount;
 981                                 VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT,       // VkShaderStageFlags   stageFlags;
 982                                 DE_NULL                                                                         // const VkSampler*             pImmutableSamplers;
 983                         };
 984
 985                         layoutBindings.push_back(layoutBindingUBO);
 986                         if (m_drawTessellatedSphere)
 987                                 layoutBindings.push_back(layoutBindingSBO);
 988
 989                         const VkDescriptorSetLayoutCreateInfo   descriptorLayoutParams =
 990                         {
 991                                 VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,    // VkStructureType                                              sType;
 992                                 DE_NULL,                                                                                                // cost void*                                                   pNext;
 993                                 (VkDescriptorSetLayoutCreateFlags)0,                                    // VkDescriptorSetLayoutCreateFlags             flags
 994                                 deUint32(layoutBindings.size()),                                                // deUint32                                                             count;
 995                                 layoutBindings.data()                                                                   // const VkDescriptorSetLayoutBinding   pBinding;
 996                         };
 997                         descriptorSetLayout = createDescriptorSetLayout(vk, *m_deviceGroup, &descriptorLayoutParams);
 998
 999                         const VkDescriptorPoolSize descriptorTypeUBO =
1000                         {
1001                                 VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,              // VkDescriptorType             type;
1002                                 1                                                                               // deUint32                             count;
1003                         };
1004                         const VkDescriptorPoolSize descriptorTypeSBO =
1005                         {
1006                                 VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,              // VkDescriptorType             type;
1007                                 1                                                                               // deUint32                             count;
1008                         };
1009                         descriptorTypes.push_back(descriptorTypeUBO);
1010                         if (m_drawTessellatedSphere)
1011                                 descriptorTypes.push_back(descriptorTypeSBO);
1012
1013                         const VkDescriptorPoolCreateInfo descriptorPoolParams =
1014                         {
1015                                 VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,          // VkStructureType                                      sType;
1016                                 DE_NULL,                                                                                        // void*                                                        pNext;
1017                                 VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT,      // VkDescriptorPoolCreateFlags          flags;
1018                                 1u,                                                                                                     // deUint32                                                     maxSets;
1019                                 deUint32(descriptorTypes.size()),                                       // deUint32                                                     count;
1020                                 descriptorTypes.data()                                                          // const VkDescriptorTypeCount*         pTypeCount
1021                         };
1022                         descriptorPool = createDescriptorPool(vk, *m_deviceGroup, &descriptorPoolParams);
1023
1024                         const VkDescriptorSetAllocateInfo descriptorSetParams =
1025                         {
1026                                 VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
1027                                 DE_NULL,
1028                                 *descriptorPool,
1029                                 1u,
1030                                 &descriptorSetLayout.get(),
1031                         };
1032                         descriptorSet = allocateDescriptorSet(vk, *m_deviceGroup, &descriptorSetParams);
1033
1034                         const VkDescriptorBufferInfo uboDescriptorInfo =
1035                         {
1036                                 uniformBuffer.get(),
1037                                 0,
1038                                 (VkDeviceSize)sizeof(drawColor)
1039                         };
1040                         const VkDescriptorBufferInfo sboDescriptorInfo =
1041                         {
1042                                 sboBuffer.get(),
1043                                 0,
1044                                 (VkDeviceSize)sizeof(tessLevel)
1045                         };
1046                         const VkWriteDescriptorSet writeDescritporSetUBO =
1047                         {
1048                                 VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,         // VkStructureType                      sType;
1049                                 DE_NULL,                                                                        // const void*                          pNext;
1050                                 *descriptorSet,                                                         // VkDescriptorSet                      destSet;
1051                                 0,                                                                                      // deUint32                                     destBinding;
1052                                 0,                                                                                      // deUint32                                     destArrayElement;
1053                                 1u,                                                                                     // deUint32                                     count;
1054                                 VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,                      // VkDescriptorType                     descriptorType;
1055                                 (const VkDescriptorImageInfo*)DE_NULL,          // VkDescriptorImageInfo*       pImageInfo;
1056                                 &uboDescriptorInfo,                                                     // VkDescriptorBufferInfo*      pBufferInfo;
1057                                 (const VkBufferView*)DE_NULL                            // VkBufferView*                        pTexelBufferView;
1058                         };
1059
1060                         const VkWriteDescriptorSet writeDescritporSetSBO =
1061                         {
1062                                 VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,         // VkStructureType                      sType;
1063                                 DE_NULL,                                                                        // const void*                          pNext;
1064                                 *descriptorSet,                                                         // VkDescriptorSet                      destSet;
1065                                 1,                                                                                      // deUint32                                     destBinding;
1066                                 0,                                                                                      // deUint32                                     destArrayElement;
1067                                 1u,                                                                                     // deUint32                                     count;
1068                                 VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,                      // VkDescriptorType                     descriptorType;
1069                                 (const VkDescriptorImageInfo*)DE_NULL,          // VkDescriptorImageInfo*       pImageInfo;
1070                                 &sboDescriptorInfo,                                                     // VkDescriptorBufferInfo*      pBufferInfo;
1071                                 (const VkBufferView*)DE_NULL                            // VkBufferView*                        pTexelBufferView;
1072                         };
1073                         writeDescritporSets.push_back(writeDescritporSetUBO);
1074                         if (m_drawTessellatedSphere)
1075                                 writeDescritporSets.push_back(writeDescritporSetSBO);
1076
1077                         vk.updateDescriptorSets(*m_deviceGroup, deUint32(writeDescritporSets.size()), writeDescritporSets.data(), 0u, DE_NULL);
1078                 }
1079
1080                 // Create Pipeline
1081                 {
1082                         vector<VkPipelineShaderStageCreateInfo> shaderStageParams;
1083                         Move<VkShaderModule>                                    vertShaderModule;
1084                         Move<VkShaderModule>                                    tcssShaderModule;
1085                         Move<VkShaderModule>                                    tessShaderModule;
1086                         Move<VkShaderModule>                                    fragShaderModule;
1087
1088                         const VkDescriptorSetLayout descset = descriptorSetLayout.get();
1089                         const VkPipelineLayoutCreateInfo        pipelineLayoutParams =
1090                         {
1091                                 VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,                  // sType
1092                                 DE_NULL,                                                                                                // pNext
1093                                 (vk::VkPipelineLayoutCreateFlags)0,                                             // flags
1094                                 1u,                                                                                                             // setLayoutCount
1095                                 &descset,                                                                                               // pSetLayouts
1096                                 0u,                                                                                                             // pushConstantRangeCount
1097                                 DE_NULL,                                                                                                // pPushConstantRanges
1098                         };
1099                         pipelineLayout = createPipelineLayout(vk, *m_deviceGroup, &pipelineLayoutParams);
1100
1101                         // Shaders
1102                         vertShaderModule = createShaderModule(vk, *m_deviceGroup, m_context.getBinaryCollection().get("vert"), 0);
1103                         fragShaderModule = createShaderModule(vk, *m_deviceGroup, m_context.getBinaryCollection().get("frag"), 0);
1104
1105                         const VkSpecializationInfo                              emptyShaderSpecParams =
1106                         {
1107                                 0u,                                                                                                                     // mapEntryCount
1108                                 DE_NULL,                                                                                                        // pMap
1109                                 0,                                                                                                                      // dataSize
1110                                 DE_NULL,                                                                                                        // pData
1111                         };
1112                         const VkPipelineShaderStageCreateInfo   vertexShaderStageParams =
1113                         {
1114                                 VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,    // sType
1115                                 DE_NULL,                                                                                                // pNext
1116                                 0u,                                                                                                             // flags
1117                                 VK_SHADER_STAGE_VERTEX_BIT,                                                             // stage
1118                                 *vertShaderModule,                                                                              // module
1119                                 "main",                                                                                                 // pName
1120                                 &emptyShaderSpecParams,                                                                 // pSpecializationInfo
1121                         };
1122                         shaderStageParams.push_back(vertexShaderStageParams);
1123
1124                         if (m_drawTessellatedSphere)
1125                         {
1126                                 tcssShaderModule = createShaderModule(vk, *m_deviceGroup, m_context.getBinaryCollection().get("tesc"), 0);
1127                                 tessShaderModule = createShaderModule(vk, *m_deviceGroup, m_context.getBinaryCollection().get("tese"), 0);
1128
1129                                 const VkPipelineShaderStageCreateInfo   tessControlShaderStageParams =
1130                                 {
1131                                         VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,    // sType
1132                                         DE_NULL,                                                                                                // pNext
1133                                         0u,                                                                                                             // flags
1134                                         VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT,                               // stage
1135                                         *tcssShaderModule,                                                                              // module
1136                                         "main",                                                                                                 // pName
1137                                         &emptyShaderSpecParams,                                                                 // pSpecializationInfo
1138                                 };
1139                                 const VkPipelineShaderStageCreateInfo   tessEvalShaderStageParams =
1140                                 {
1141                                         VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,    // sType
1142                                         DE_NULL,                                                                                                // pNext
1143                                         0u,                                                                                                             // flags
1144                                         VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT,                    // stage
1145                                         *tessShaderModule,                                                                              // module
1146                                         "main",                                                                                                 // pName
1147                                         &emptyShaderSpecParams,                                                                 // pSpecializationInfo
1148                                 };
1149
1150                                 shaderStageParams.push_back(tessControlShaderStageParams);
1151                                 shaderStageParams.push_back(tessEvalShaderStageParams);
1152                         }
1153
1154                         const VkPipelineShaderStageCreateInfo   fragmentShaderStageParams =
1155                         {
1156                                 VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,    // sType
1157                                 DE_NULL,                                                                                                // pNext
1158                                 0u,                                                                                                             // flags
1159                                 VK_SHADER_STAGE_FRAGMENT_BIT,                                                   // stage
1160                                 *fragShaderModule,                                                                              // module
1161                                 "main",                                                                                                 // pName
1162                                 &emptyShaderSpecParams,                                                                 // pSpecializationInfo
1163                         };
1164                         shaderStageParams.push_back(fragmentShaderStageParams);
1165
1166                         const VkPipelineDepthStencilStateCreateInfo     depthStencilParams =
1167                         {
1168                                 VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,     // sType
1169                                 DE_NULL,                                                                                                        // pNext
1170                                 0u,                                                                                                                     // flags
1171                                 DE_FALSE,                                                                                                       // depthTestEnable
1172                                 DE_FALSE,                                                                                                       // depthWriteEnable
1173                                 VK_COMPARE_OP_ALWAYS,                                                                           // depthCompareOp
1174                                 DE_FALSE,                                                                                                       // depthBoundsTestEnable
1175                                 DE_FALSE,                                                                                                       // stencilTestEnable
1176                                 {
1177                                         VK_STENCIL_OP_KEEP,                                                                             // failOp
1178                                         VK_STENCIL_OP_KEEP,                                                                             // passOp
1179                                         VK_STENCIL_OP_KEEP,                                                                             // depthFailOp
1180                                         VK_COMPARE_OP_ALWAYS,                                                                   // compareOp
1181                                         0u,                                                                                                             // compareMask
1182                                         0u,                                                                                                             // writeMask
1183                                         0u,                                                                                                             // reference
1184                                 },                                                                                                                      // front
1185                                 {
1186                                         VK_STENCIL_OP_KEEP,                                                                             // failOp
1187                                         VK_STENCIL_OP_KEEP,                                                                             // passOp
1188                                         VK_STENCIL_OP_KEEP,                                                                             // depthFailOp
1189                                         VK_COMPARE_OP_ALWAYS,                                                                   // compareOp
1190                                         0u,                                                                                                             // compareMask
1191                                         0u,                                                                                                             // writeMask
1192                                         0u,                                                                                                             // reference
1193                                 },                                                                                                                      // back;
1194                                 0.0f,                                                                                                           // minDepthBounds;
1195                                 1.0f,                                                                                                           // maxDepthBounds;
1196                         };
1197                         const VkViewport        viewport0 =
1198                         {
1199                                 0.0f,                                                                                                           // x
1200                                 0.0f,                                                                                                           // y
1201                                 (float)renderSize.x(),                                                                          // width
1202                                 (float)renderSize.y(),                                                                          // height
1203                                 0.0f,                                                                                                           // minDepth
1204                                 1.0f,                                                                                                           // maxDepth
1205                         };
1206                         const VkRect2D          scissor0 =
1207                         {
1208                                 {
1209                                         0u,                                                                                                             // x
1210                                         0u,                                                                                                             // y
1211                                 },                                                                                                                      // offset
1212                                 {
1213                                         renderSize.x(),                                                                                 // width
1214                                         renderSize.y(),                                                                                 // height
1215                                 },                                                                                                                      // extent;
1216                         };
1217                         const VkPipelineViewportStateCreateInfo         viewportParams =
1218                         {
1219                                 VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,          // sType
1220                                 DE_NULL,                                                                                                        // pNext
1221                                 0u,                                                                                                                     // flags
1222                                 1u,                                                                                                                     // viewportCount
1223                                 &viewport0,                                                                                                     // pViewports
1224                                 1u,                                                                                                                     // scissorCount
1225                                 &scissor0                                                                                                       // pScissors
1226                         };
1227                         const VkSampleMask                                                      sampleMask = ~0u;
1228                         const VkPipelineMultisampleStateCreateInfo      multisampleParams =
1229                         {
1230                                 VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,       // sType
1231                                 DE_NULL,                                                                                                        // pNext
1232                                 0u,                                                                                                                     // flags
1233                                 VK_SAMPLE_COUNT_1_BIT,                                                                          // rasterizationSamples
1234                                 VK_FALSE,                                                                                                       // sampleShadingEnable
1235                                 0.0f,                                                                                                           // minSampleShading
1236                                 &sampleMask,                                                                                            // sampleMask
1237                                 VK_FALSE,                                                                                                       // alphaToCoverageEnable
1238                                 VK_FALSE,                                                                                                       // alphaToOneEnable
1239                         };
1240                         const VkPipelineRasterizationStateCreateInfo    rasterParams =
1241                         {
1242                                 VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,                     // sType
1243                                 DE_NULL,                                                                                                                        // pNext
1244                                 0u,                                                                                                                                     // flags
1245                                 VK_TRUE,                                                                                                                        // depthClampEnable
1246                                 VK_FALSE,                                                                                                                       // rasterizerDiscardEnable
1247                                 m_fillModeNonSolid ? VK_POLYGON_MODE_LINE : VK_POLYGON_MODE_FILL,       // polygonMode
1248                                 VK_CULL_MODE_NONE,                                                                                                      // cullMode
1249                                 VK_FRONT_FACE_COUNTER_CLOCKWISE,                                                                        // frontFace
1250                                 VK_FALSE,                                                                                                                       // depthBiasEnable
1251                                 0.0f,                                                                                                                           // depthBiasConstantFactor
1252                                 0.0f,                                                                                                                           // depthBiasClamp
1253                                 0.0f,                                                                                                                           // depthBiasSlopeFactor
1254                                 1.0f,                                                                                                                           // lineWidth
1255                         };
1256                         const VkPipelineInputAssemblyStateCreateInfo    inputAssemblyParams =
1257                         {
1258                                 VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,    // sType
1259                                 DE_NULL,                                                                                                                // pNext
1260                                 0u,                                                                                                                             // flags
1261                                 m_drawTessellatedSphere ? VK_PRIMITIVE_TOPOLOGY_PATCH_LIST : VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST,       // topology
1262                                 DE_FALSE,                                                                                                               // primitiveRestartEnable
1263                         };
1264                         const VkVertexInputBindingDescription           vertexBinding0 =
1265                         {
1266                                 0u,                                                                                                             // binding
1267                                 (deUint32)sizeof(tcu::Vec4),                                                    // stride
1268                                 VK_VERTEX_INPUT_RATE_VERTEX,                                                    // inputRate
1269                         };
1270                         const VkVertexInputAttributeDescription         vertexAttrib0 =
1271                         {
1272                                 0u,                                                                                                             // location
1273                                 0u,                                                                                                             // binding
1274                                 VK_FORMAT_R32G32B32A32_SFLOAT,                                                  // format
1275                                 0u,                                                                                                             // offset
1276                         };
1277                         const VkPipelineVertexInputStateCreateInfo      vertexInputStateParams =
1278                         {
1279                                 VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,      // sType
1280                                 DE_NULL,                                                                                                        // pNext
1281                                 0u,                                                                                                                     // flags
1282                                 1u,                                                                                                                     // vertexBindingDescriptionCount
1283                                 &vertexBinding0,                                                                                        // pVertexBindingDescriptions
1284                                 1u,                                                                                                                     // vertexAttributeDescriptionCount
1285                                 &vertexAttrib0,                                                                                         // pVertexAttributeDescriptions
1286                         };
1287                         const VkPipelineColorBlendAttachmentState       attBlendParams =
1288                         {
1289                                 VK_FALSE,                                                                                                       // blendEnable
1290                                 VK_BLEND_FACTOR_ONE,                                                                            // srcColorBlendFactor
1291                                 VK_BLEND_FACTOR_ZERO,                                                                           // dstColorBlendFactor
1292                                 VK_BLEND_OP_ADD,                                                                                        // colorBlendOp
1293                                 VK_BLEND_FACTOR_ONE,                                                                            // srcAlphaBlendFactor
1294                                 VK_BLEND_FACTOR_ZERO,                                                                           // dstAlphaBlendFactor
1295                                 VK_BLEND_OP_ADD,                                                                                        // alphaBlendOp
1296                                 (VK_COLOR_COMPONENT_R_BIT |
1297                                  VK_COLOR_COMPONENT_G_BIT |
1298                                  VK_COLOR_COMPONENT_B_BIT |
1299                                  VK_COLOR_COMPONENT_A_BIT),                                                                     // colorWriteMask
1300                         };
1301                         const VkPipelineColorBlendStateCreateInfo       blendParams =
1302                         {
1303                                 VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,       // sType
1304                                 DE_NULL,                                                                                                        // pNext
1305                                 0u,                                                                                                                     // flags
1306                                 DE_FALSE,                                                                                                       // logicOpEnable
1307                                 VK_LOGIC_OP_COPY,                                                                                       // logicOp
1308                                 1u,                                                                                                                     // attachmentCount
1309                                 &attBlendParams,                                                                                        // pAttachments
1310                                 { 0.0f, 0.0f, 0.0f, 0.0f },                                                                     // blendConstants[4]
1311                         };
1312
1313                         const VkPipelineTessellationStateCreateInfo tessState =
1314                         {
1315                                 VK_STRUCTURE_TYPE_PIPELINE_TESSELLATION_STATE_CREATE_INFO,      // sType
1316                                 DE_NULL,                                                                                                        // pNext
1317                                 0u,                                                                                                                     // flags
1318                                 3u,                                                                                                                     // patchControlPoints
1319                         };
1320                         const VkGraphicsPipelineCreateInfo              pipelineParams =
1321                         {
1322                                 VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,                // sType
1323                                 DE_NULL,                                                                                                // pNext
1324                                 0u,                                                                                                             // flags
1325                                 deUint32(shaderStageParams.size()),                                             // stageCount
1326                                 shaderStageParams.data(),                                                               // pStages
1327                                 &vertexInputStateParams,                                                                // pVertexInputState
1328                                 &inputAssemblyParams,                                                                   // pInputAssemblyState
1329                                 m_drawTessellatedSphere ? &tessState : DE_NULL,                 // pTessellationState
1330                                 &viewportParams,                                                                                // pViewportState
1331                                 &rasterParams,                                                                                  // pRasterizationState
1332                                 &multisampleParams,                                                                             // pMultisampleState
1333                                 &depthStencilParams,                                                                    // pDepthStencilState
1334                                 &blendParams,                                                                                   // pColorBlendState
1335                                 (const VkPipelineDynamicStateCreateInfo*)DE_NULL,               // pDynamicState
1336                                 *pipelineLayout,                                                                                // layout
1337                                 *renderPass,                                                                                    // renderPass
1338                                 0u,                                                                                                             // subpass
1339                                 DE_NULL,                                                                                                // basePipelineHandle
1340                                 0u,                                                                                                             // basePipelineIndex
1341                         };
1342                         pipeline = createGraphicsPipeline(vk, *m_deviceGroup, DE_NULL, &pipelineParams);
1343                 }
1344
1345                 // Create Framebuffer
1346                 {
1347                         const VkImageViewCreateInfo                             colorAttViewParams =
1348                         {
1349                                 VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,               // sType
1350                                 DE_NULL,                                                                                // pNext
1351                                 0u,                                                                                             // flags
1352                                 *renderImage,                                                                   // image
1353                                 VK_IMAGE_VIEW_TYPE_2D,                                                  // viewType
1354                                 colorFormat,                                                                    // format
1355                                 {
1356                                         VK_COMPONENT_SWIZZLE_R,
1357                                         VK_COMPONENT_SWIZZLE_G,
1358                                         VK_COMPONENT_SWIZZLE_B,
1359                                         VK_COMPONENT_SWIZZLE_A
1360                                 },                                                                                              // components
1361                                 {
1362                                         VK_IMAGE_ASPECT_COLOR_BIT,                                      // aspectMask
1363                                         0u,                                                                                     // baseMipLevel
1364                                         1u,                                                                                     // levelCount
1365                                         0u,                                                                                     // baseArrayLayer
1366                                         1u,                                                                                     // layerCount
1367                                 },                                                                                              // subresourceRange
1368                         };
1369                         colorAttView = createImageView(vk, *m_deviceGroup, &colorAttViewParams);
1370
1371                         const VkFramebufferCreateInfo                   framebufferParams =
1372                         {
1373                                 VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,                              // sType
1374                                 DE_NULL,                                                                                                // pNext
1375                                 0u,                                                                                                             // flags
1376                                 *renderPass,                                                                                    // renderPass
1377                                 1u,                                                                                                             // attachmentCount
1378                                 &*colorAttView,                                                                                 // pAttachments
1379                                 renderSize.x(),                                                                                 // width
1380                                 renderSize.y(),                                                                                 // height
1381                                 1u,                                                                                                             // layers
1382                         };
1383                         framebuffer = createFramebuffer(vk, *m_deviceGroup, &framebufferParams);
1384                 }
1385
1386                 // Create Command buffer
1387                 {
1388                         const VkCommandPoolCreateInfo                   cmdPoolParams =
1389                         {
1390                                 VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,                                     // sType
1391                                 DE_NULL,                                                                                                        // pNext
1392                                 VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,                        // flags
1393                                 queueFamilyIndex,                                                                                       // queueFamilyIndex
1394                         };
1395                         cmdPool = createCommandPool(vk, *m_deviceGroup, &cmdPoolParams);
1396
1397                         const VkCommandBufferAllocateInfo               cmdBufParams =
1398                         {
1399                                 VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,                 // sType
1400                                 DE_NULL,                                                                                                // pNext
1401                                 *cmdPool,                                                                                               // pool
1402                                 VK_COMMAND_BUFFER_LEVEL_PRIMARY,                                                // level
1403                                 1u,                                                                                                             // bufferCount
1404                         };
1405                         cmdBuffer = allocateCommandBuffer(vk, *m_deviceGroup, &cmdBufParams);
1406                 }
1407
1408                 // Begin recording
1409                 VkCommandBufferBeginInfo                                cmdBufBeginParams =
1410                 {
1411                         VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,                    // sType
1412                         DE_NULL,                                                                                                // pNext
1413                         VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,                    // flags
1414                         (const VkCommandBufferInheritanceInfo*)DE_NULL,
1415                 };
1416                 VK_CHECK(vk.beginCommandBuffer(*cmdBuffer, &cmdBufBeginParams));
1417
1418                 // Prepare render target for rendering
1419                 {
1420                         const VkMemoryBarrier           vertFlushBarrier =
1421                         {
1422                                 VK_STRUCTURE_TYPE_MEMORY_BARRIER,                       // sType
1423                                 DE_NULL,                                                                        // pNext
1424                                 VK_ACCESS_HOST_WRITE_BIT,                                       // srcAccessMask
1425                                 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT,            // dstAccessMask
1426                         };
1427                         const VkImageMemoryBarrier      colorAttBarrier =
1428                         {
1429                                 VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,         // sType
1430                                 DE_NULL,                                                                        // pNext
1431                                 0u,                                                                                     // srcAccessMask
1432                                 (VK_ACCESS_COLOR_ATTACHMENT_READ_BIT |
1433                                 VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT),          // dstAccessMask
1434                                 VK_IMAGE_LAYOUT_UNDEFINED,                                      // oldLayout
1435                                 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,       // newLayout
1436                                 queueFamilyIndex,                                                       // srcQueueFamilyIndex
1437                                 queueFamilyIndex,                                                       // dstQueueFamilyIndex
1438                                 *renderImage,                                                           // image
1439                                 {
1440                                         VK_IMAGE_ASPECT_COLOR_BIT,                              // aspectMask
1441                                         0u,                                                                             // baseMipLevel
1442                                         1u,                                                                             // levelCount
1443                                         0u,                                                                             // baseArrayLayer
1444                                         1u,                                                                             // layerCount
1445                                 }                                                                                       // subresourceRange
1446                         };
1447                         vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, (VkDependencyFlags)0, 1, &vertFlushBarrier, 0, (const VkBufferMemoryBarrier*)DE_NULL, 1, &colorAttBarrier);
1448                 }
1449
1450                 // Update buffers
1451                 {
1452                         const VkBufferMemoryBarrier             stagingVertexBufferUpdateBarrier =
1453                         {
1454                                 VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,        // VkStructureType      sType;
1455                                 DE_NULL,                                                                        // const void*          pNext;
1456                                 VK_ACCESS_HOST_WRITE_BIT,                                       // VkAccessFlags        srcAccessMask;
1457                                 VK_ACCESS_TRANSFER_READ_BIT,                            // VkAccessFlags        dstAccessMask;
1458                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     srcQueueFamilyIndex;
1459                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     dstQueueFamilyIndex;
1460                                 stagingVertexBuffer.get(),                                      // VkBuffer                     buffer;
1461                                 0u,                                                                                     // VkDeviceSize         offset;
1462                                 verticesSize                                                            // VkDeviceSize         size;
1463                         };
1464
1465                         const VkBufferMemoryBarrier             vertexBufferUpdateBarrier =
1466                         {
1467                                 VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,        // VkStructureType      sType;
1468                                 DE_NULL,                                                                        // const void*          pNext;
1469                                 VK_ACCESS_TRANSFER_WRITE_BIT,                           // VkAccessFlags        srcAccessMask;
1470                                 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT,            // VkAccessFlags        dstAccessMask;
1471                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     srcQueueFamilyIndex;
1472                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     dstQueueFamilyIndex;
1473                                 vertexBuffer.get(),                                                     // VkBuffer                     buffer;
1474                                 0u,                                                                                     // VkDeviceSize         offset;
1475                                 verticesSize                                                            // VkDeviceSize         size;
1476                         };
1477
1478                         const VkBufferMemoryBarrier             stagingIndexBufferUpdateBarrier =
1479                         {
1480                                 VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,        // VkStructureType      sType;
1481                                 DE_NULL,                                                                        // const void*          pNext;
1482                                 VK_ACCESS_HOST_WRITE_BIT,                                       // VkAccessFlags        srcAccessMask;
1483                                 VK_ACCESS_TRANSFER_READ_BIT,                            // VkAccessFlags        dstAccessMask;
1484                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     srcQueueFamilyIndex;
1485                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     dstQueueFamilyIndex;
1486                                 stagingIndexBuffer.get(),                                       // VkBuffer                     buffer;
1487                                 0u,                                                                                     // VkDeviceSize         offset;
1488                                 indicesSize                                                                     // VkDeviceSize         size;
1489                         };
1490
1491                         const VkBufferMemoryBarrier             indexBufferUpdateBarrier =
1492                         {
1493                                 VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,        // VkStructureType      sType;
1494                                 DE_NULL,                                                                        // const void*          pNext;
1495                                 VK_ACCESS_TRANSFER_WRITE_BIT,                           // VkAccessFlags        srcAccessMask;
1496                                 VK_ACCESS_INDEX_READ_BIT,                                       // VkAccessFlags        dstAccessMask;
1497                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     srcQueueFamilyIndex;
1498                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     dstQueueFamilyIndex;
1499                                 indexBuffer.get(),                                                      // VkBuffer                     buffer;
1500                                 0u,                                                                                     // VkDeviceSize         offset;
1501                                 indicesSize                                                                     // VkDeviceSize         size;
1502                         };
1503
1504                         const VkBufferMemoryBarrier             stagingUboBufferUpdateBarrier =
1505                         {
1506                                 VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,        // VkStructureType      sType;
1507                                 DE_NULL,                                                                        // const void*          pNext;
1508                                 VK_ACCESS_HOST_WRITE_BIT,                                       // VkAccessFlags        srcAccessMask;
1509                                 VK_ACCESS_TRANSFER_READ_BIT,                            // VkAccessFlags        dstAccessMask;
1510                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     srcQueueFamilyIndex;
1511                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     dstQueueFamilyIndex;
1512                                 stagingUniformBuffer.get(),                                     // VkBuffer                     buffer;
1513                                 0u,                                                                                     // VkDeviceSize         offset;
1514                                 indicesSize                                                                     // VkDeviceSize         size;
1515                         };
1516
1517                         const VkBufferMemoryBarrier             uboUpdateBarrier =
1518                         {
1519                                 VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,        // VkStructureType      sType;
1520                                 DE_NULL,                                                                        // const void*          pNext;
1521                                 VK_ACCESS_TRANSFER_WRITE_BIT,                           // VkAccessFlags        srcAccessMask;
1522                                 VK_ACCESS_UNIFORM_READ_BIT,                                     // VkAccessFlags        dstAccessMask;
1523                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     srcQueueFamilyIndex;
1524                                 VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     dstQueueFamilyIndex;
1525                                 uniformBuffer.get(),                                            // VkBuffer                     buffer;
1526                                 0u,                                                                                     // VkDeviceSize         offset;
1527                                 sizeof(drawColor)                                                       // VkDeviceSize         size;
1528                         };
1529
1530
1531                         vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 1, &stagingVertexBufferUpdateBarrier, 0, (const VkImageMemoryBarrier*)DE_NULL);
1532                         VkBufferCopy    vertexBufferCopy        = { 0u, 0u, verticesSize };
1533                         vk.cmdCopyBuffer(*cmdBuffer, stagingVertexBuffer.get(), vertexBuffer.get(), 1u, &vertexBufferCopy);
1534                         vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 1, &vertexBufferUpdateBarrier, 0, (const VkImageMemoryBarrier*)DE_NULL);
1535
1536                         vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 1, &stagingIndexBufferUpdateBarrier, 0, (const VkImageMemoryBarrier*)DE_NULL);
1537                         VkBufferCopy    indexBufferCopy = { 0u, 0u, indicesSize };
1538                         vk.cmdCopyBuffer(*cmdBuffer, stagingIndexBuffer.get(), indexBuffer.get(), 1u, &indexBufferCopy);
1539                         vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 1, &indexBufferUpdateBarrier, 0, (const VkImageMemoryBarrier*)DE_NULL);
1540
1541                         vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 1, &stagingUboBufferUpdateBarrier, 0, (const VkImageMemoryBarrier*)DE_NULL);
1542                         VkBufferCopy    uboBufferCopy   = { 0u, 0u, sizeof(drawColor) };
1543                         vk.cmdCopyBuffer(*cmdBuffer, stagingUniformBuffer.get(), uniformBuffer.get(), 1u, &uboBufferCopy);
1544                         vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 1, &uboUpdateBarrier, 0, (const VkImageMemoryBarrier*)DE_NULL);
1545
1546                         if (m_drawTessellatedSphere)
1547                         {
1548                                 const VkBufferMemoryBarrier             stagingsboUpdateBarrier =
1549                                 {
1550                                         VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,        // VkStructureType      sType;
1551                                         DE_NULL,                                                                        // const void*          pNext;
1552                                         VK_ACCESS_HOST_WRITE_BIT,                                       // VkAccessFlags        srcAccessMask;
1553                                         VK_ACCESS_TRANSFER_READ_BIT,                            // VkAccessFlags        dstAccessMask;
1554                                         VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     srcQueueFamilyIndex;
1555                                         VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     dstQueueFamilyIndex;
1556                                         stagingSboBuffer.get(),                                         // VkBuffer                     buffer;
1557                                         0u,                                                                                     // VkDeviceSize         offset;
1558                                         sizeof(tessLevel)                                                       // VkDeviceSize         size;
1559                                 };
1560
1561                                 const VkBufferMemoryBarrier             sboUpdateBarrier =
1562                                 {
1563                                         VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,        // VkStructureType      sType;
1564                                         DE_NULL,                                                                        // const void*          pNext;
1565                                         VK_ACCESS_TRANSFER_WRITE_BIT,                           // VkAccessFlags        srcAccessMask;
1566                                         VK_ACCESS_SHADER_READ_BIT,                                      // VkAccessFlags        dstAccessMask;
1567                                         VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     srcQueueFamilyIndex;
1568                                         VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                     dstQueueFamilyIndex;
1569                                         sboBuffer.get(),                                                        // VkBuffer                     buffer;
1570                                         0u,                                                                                     // VkDeviceSize         offset;
1571                                         sizeof(tessLevel)                                                       // VkDeviceSize         size;
1572                                 };
1573
1574                                 vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 1, &stagingsboUpdateBarrier, 0, (const VkImageMemoryBarrier*)DE_NULL);
1575                                 VkBufferCopy    sboBufferCopy   = { 0u, 0u, sizeof(tessLevel) };
1576                                 vk.cmdCopyBuffer(*cmdBuffer, stagingSboBuffer.get(), sboBuffer.get(), 1u, &sboBufferCopy);
1577                                 vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 1, &sboUpdateBarrier, 0, (const VkImageMemoryBarrier*)DE_NULL);
1578                         }
1579
1580                         vk.cmdBindPipeline(*cmdBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline);
1581                         vk.cmdBindDescriptorSets(*cmdBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, *pipelineLayout, 0u, 1, &*descriptorSet, 0u, DE_NULL);
1582                         {
1583                                 const VkDeviceSize bindingOffset = 0;
1584                                 vk.cmdBindVertexBuffers(*cmdBuffer, 0u, 1u, &vertexBuffer.get(), &bindingOffset);
1585                                 vk.cmdBindIndexBuffer(*cmdBuffer, *indexBuffer, 0, VK_INDEX_TYPE_UINT32);
1586                         }
1587                 }
1588
1589                 // Begin renderpass
1590                 {
1591                         const VkClearValue clearValue = makeClearValueColorF32(
1592                                 clearColor[0],
1593                                 clearColor[1],
1594                                 clearColor[2],
1595                                 clearColor[3]);
1596
1597                         VkRect2D zeroRect = { { 0, 0, },{ 0, 0, } };
1598                         vector<VkRect2D> renderAreas;
1599                         for (deUint32 i = 0; i < m_physicalDeviceCount; i++)
1600                                 renderAreas.push_back(zeroRect);
1601
1602                         // Render completely if there is only 1 device
1603                         if (m_physicalDeviceCount == 1u)
1604                         {
1605                                 renderAreas[0].extent.width = (deInt32)renderSize.x();
1606                                 renderAreas[0].extent.height = (deInt32)renderSize.y();
1607                         }
1608                         else
1609                         {
1610                                 // Split into 2 vertical halves
1611                                 renderAreas[firstDeviceID].extent.width         = (deInt32)renderSize.x() / 2;
1612                                 renderAreas[firstDeviceID].extent.height        = (deInt32)renderSize.y();
1613                                 renderAreas[secondDeviceID]                                     = renderAreas[firstDeviceID];
1614                                 renderAreas[secondDeviceID].offset.x            = (deInt32)renderSize.x() / 2;
1615                         }
1616
1617                         const VkDeviceGroupRenderPassBeginInfo deviceGroupRPBeginInfo =
1618                         {
1619                                 VK_STRUCTURE_TYPE_DEVICE_GROUP_RENDER_PASS_BEGIN_INFO,
1620                                 DE_NULL,
1621                                 (deUint32)((1 << m_physicalDeviceCount) - 1),
1622                                 m_physicalDeviceCount,
1623                                 &renderAreas[0]
1624                         };
1625
1626                         const VkRenderPassBeginInfo     passBeginParams =
1627                         {
1628                                 VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,                                                       // sType
1629                                 (m_testMode & TEST_MODE_SFR) ? &deviceGroupRPBeginInfo : DE_NULL,       // pNext
1630                                 *renderPass,                                                                                                            // renderPass
1631                                 *framebuffer,                                                                                                           // framebuffer
1632                                 {
1633                                         { 0, 0 },
1634                                         { renderSize.x(), renderSize.y() }
1635                                 },                                                                                                                              // renderArea
1636                                 1u,                                                                                                                             // clearValueCount
1637                                 &clearValue,                                                                                                    // pClearValues
1638                         };
1639                         vk.cmdBeginRenderPass(*cmdBuffer, &passBeginParams, VK_SUBPASS_CONTENTS_INLINE);
1640                 }
1641
1642                 // Draw
1643                 if (m_testMode & TEST_MODE_AFR)
1644                 {
1645                         vk.cmdSetDeviceMask(*cmdBuffer, 1 << secondDeviceID);
1646                         vk.cmdDrawIndexed(*cmdBuffer, numIndices, 1u, 0, 0, 0);
1647
1648                 }
1649                 else
1650                 {
1651                         vk.cmdSetDeviceMask(*cmdBuffer, ((1 << firstDeviceID) | (1 << secondDeviceID)));
1652                         vk.cmdDrawIndexed(*cmdBuffer, numIndices, 1u, 0, 0, 0);
1653                 }
1654                 vk.cmdEndRenderPass(*cmdBuffer);
1655
1656                 // Change image layout for copy
1657                 {
1658                         const VkImageMemoryBarrier      renderFinishBarrier =
1659                         {
1660                                 VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,         // sType
1661                                 DE_NULL,                                                                        // pNext
1662                                 VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,           // outputMask
1663                                 VK_ACCESS_TRANSFER_READ_BIT,                            // inputMask
1664                                 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,       // oldLayout
1665                                 VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,           // newLayout
1666                                 queueFamilyIndex,                                                       // srcQueueFamilyIndex
1667                                 queueFamilyIndex,                                                       // dstQueueFamilyIndex
1668                                 *renderImage,                                                           // image
1669                                 {
1670                                         VK_IMAGE_ASPECT_COLOR_BIT,                              // aspectMask
1671                                         0u,                                                                             // baseMipLevel
1672                                         1u,                                                                             // mipLevels
1673                                         0u,                                                                             // baseArraySlice
1674                                         1u,                                                                             // arraySize
1675                                 }                                                                                       // subresourceRange
1676                         };
1677                         vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 0, (const VkBufferMemoryBarrier*)DE_NULL, 1, &renderFinishBarrier);
1678                 }
1679
1680                 VK_CHECK(vk.endCommandBuffer(*cmdBuffer));
1681
1682                 // Submit & wait for completion
1683                 {
1684                         const deUint32 deviceMask = (1 << firstDeviceID) | (1 << secondDeviceID);
1685                         deviceGroupSubmitInfo.commandBufferCount = 1;
1686                         deviceGroupSubmitInfo.pCommandBufferDeviceMasks = &deviceMask;
1687
1688                         const VkFenceCreateInfo fenceParams =
1689                         {
1690                                 VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,    // sType
1691                                 DE_NULL,                                                                // pNext
1692                                 0u,                                                                             // flags
1693                         };
1694                         const VkSubmitInfo              submitInfo =
1695                         {
1696                                 VK_STRUCTURE_TYPE_SUBMIT_INFO,                  // sType
1697                                 &deviceGroupSubmitInfo,                                 // pNext
1698                                 0u,                                                                             // waitSemaphoreCount
1699                                 DE_NULL,                                                                // pWaitSemaphores
1700                                 (const VkPipelineStageFlags*)DE_NULL,   // pWaitDstStageMask
1701                                 1u,                                                                             // commandBufferCount
1702                                 &cmdBuffer.get(),                                               // pCommandBuffers
1703                                 0u,                                                                             // signalSemaphoreCount
1704                                 DE_NULL,                                                                // pSignalSemaphores
1705                         };
1706                         const Unique<VkFence>   fence(createFence(vk, *m_deviceGroup, &fenceParams));
1707
1708                         VK_CHECK(vk.queueSubmit(m_deviceGroupQueue, 1u, &submitInfo, *fence));
1709                         VK_CHECK(vk.waitForFences(*m_deviceGroup, 1u, &fence.get(), DE_TRUE, ~0ull));
1710                         VK_CHECK(vk.deviceWaitIdle(*m_deviceGroup));
1711                 }
1712
1713                 // Copy image from secondDeviceID
1714                 if (m_physicalDeviceCount > 1)
1715                 {
1716                         Move<VkImage>                   peerImage;
1717
1718                         // Create and bind peer image
1719                         {
1720                                 const VkImageCreateInfo                                 peerImageParams =
1721                                 {
1722                                         VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,                                                                    // sType
1723                                         DE_NULL,                                                                                                                                // pNext
1724                                         VK_IMAGE_CREATE_ALIAS_BIT,                                                                                              // flags
1725                                         VK_IMAGE_TYPE_2D,                                                                                                               // imageType
1726                                         colorFormat,                                                                                                                    // format
1727                                         { renderSize.x(), renderSize.y(), 1 },                                                                  // extent
1728                                         1u,                                                                                                                                             // mipLevels
1729                                         1u,                                                                                                                                             // arraySize
1730                                         VK_SAMPLE_COUNT_1_BIT,                                                                                                  // samples
1731                                         VK_IMAGE_TILING_OPTIMAL,                                                                                                // tiling
1732                                         imageUsageFlag,                                                                                                                 // usage
1733                                         VK_SHARING_MODE_EXCLUSIVE,                                                                                              // sharingMode
1734                                         1u,                                                                                                                                             // queueFamilyIndexCount
1735                                         &queueFamilyIndex,                                                                                                              // pQueueFamilyIndices
1736                                         VK_IMAGE_LAYOUT_UNDEFINED,                                                                                              // initialLayout
1737                                 };
1738                                 peerImage = createImage(vk, *m_deviceGroup, &peerImageParams);
1739
1740                                 VkBindImageMemoryDeviceGroupInfo        devGroupBindInfo =
1741                                 {
1742                                         VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_DEVICE_GROUP_INFO,          // sType
1743                                         DE_NULL,                                                                                                        // pNext
1744                                         m_physicalDeviceCount,                                                                          // deviceIndexCount
1745                                         &deviceIndices[0],                                                                                      // pDeviceIndices
1746                                         0u,                                                                                                                     // SFRRectCount
1747                                         DE_NULL,                                                                                                        // pSFRRects
1748                                 };
1749
1750                                 VkBindImageMemoryInfo                           bindInfo =
1751                                 {
1752                                         VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO,                                       // sType
1753                                         &devGroupBindInfo,                                                                                      // pNext
1754                                         peerImage.get(),                                                                                        // image
1755                                         imageMemory.get(),                                                                                      // memory
1756                                         0u,                                                                                                                     // memoryOffset
1757                                 };
1758                                 VK_CHECK(vk.bindImageMemory2(*m_deviceGroup, 1, &bindInfo));
1759                         }
1760
1761                         // Copy peer image
1762                         {
1763                                 // Image barriers
1764                                 const VkImageMemoryBarrier      preImageBarrier  =
1765                                 {
1766                                         VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,         // VkStructureType               sType;
1767                                         DE_NULL,                                                                        // const void*                   pNext;
1768                                         0,                                                                                      // VkAccessFlags                 srcAccessMask;
1769                                         VK_ACCESS_TRANSFER_WRITE_BIT,                           // VkAccessFlags                 dstAccessMask;
1770                                         VK_IMAGE_LAYOUT_UNDEFINED,                                      // VkImageLayout                 oldLayout;
1771                                         VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,           // VkImageLayout                 newLayout;
1772                                         VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                              srcQueueFamilyIndex;
1773                                         VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                              dstQueueFamilyIndex;
1774                                         *peerImage,                                                                     // VkImage                               image;
1775                                         {                                                                                       // VkImageSubresourceRange subresourceRange;
1776                                                 VK_IMAGE_ASPECT_COLOR_BIT,                              // VkImageAspectFlags    aspectMask;
1777                                                 0u,                                                                             // deUint32                              baseMipLevel;
1778                                                 1u,                                                                             // deUint32                              mipLevels;
1779                                                 0u,                                                                             // deUint32                              baseArraySlice;
1780                                                 1u                                                                              // deUint32                              arraySize;
1781                                         }
1782                                 };
1783
1784                                 const VkImageMemoryBarrier      postImageBarrier =
1785                                 {
1786                                         VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,         // VkStructureType                      sType;
1787                                         DE_NULL,                                                                        // const void*                          pNext;
1788                                         VK_ACCESS_TRANSFER_WRITE_BIT,                           // VkAccessFlags                        srcAccessMask;
1789                                         VK_ACCESS_TRANSFER_READ_BIT,                            // VkAccessFlags                        dstAccessMask;
1790                                         VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,           // VkImageLayout                        oldLayout;
1791                                         VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,           // VkImageLayout                        newLayout;
1792                                         VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                                     srcQueueFamilyIndex;
1793                                         VK_QUEUE_FAMILY_IGNORED,                                        // deUint32                                     dstQueueFamilyIndex;
1794                                         *peerImage,
1795                                         {                                                                                       // VkImageSubresourceRange      subresourceRange;
1796                                                 VK_IMAGE_ASPECT_COLOR_BIT,                              // VkImageAspectFlags           aspectMask;
1797                                                 0u,                                                                             // deUint32                                     baseMipLevel;
1798                                                 1u,                                                                             // deUint32                                     mipLevels;
1799                                                 0u,                                                                             // deUint32                                     baseArraySlice;
1800                                                 1u                                                                              // deUint32                                     arraySize;
1801                                         }
1802                                 };
1803
1804                                 // AFR: Copy entire image from secondDeviceID
1805                                 // SFR: Copy the right half of image from secondDeviceID to firstDeviceID, so that the copy
1806                                 // to a buffer below (for checking) does not require VK_PEER_MEMORY_FEATURE_COPY_SRC_BIT
1807                                 deInt32 imageOffsetX = (m_testMode & TEST_MODE_AFR) ? 0 : renderSize.x()/2;
1808                                 deUint32 imageExtentX = (m_testMode & TEST_MODE_AFR) ? (deUint32)renderSize.x() : (deUint32)renderSize.x()/2;
1809
1810                                 const VkImageCopy       imageCopy       =
1811                                 {
1812                                         {
1813                                                 VK_IMAGE_ASPECT_COLOR_BIT,
1814                                                 0, // mipLevel
1815                                                 0, // arrayLayer
1816                                                 1  // layerCount
1817                                         },
1818                                         { imageOffsetX, 0, 0 },
1819                                         {
1820                                                 VK_IMAGE_ASPECT_COLOR_BIT,
1821                                                 0, // mipLevel
1822                                                 0, // arrayLayer
1823                                                 1  // layerCount
1824                                         },
1825                                         { imageOffsetX, 0, 0 },
1826                                         {
1827                                                 imageExtentX,
1828                                                 (deUint32)renderSize.y(),
1829                                                 1u
1830                                         }
1831                                 };
1832
1833                                 VK_CHECK(vk.beginCommandBuffer(*cmdBuffer, &cmdBufBeginParams));
1834                                 vk.cmdSetDeviceMask(*cmdBuffer, 1 << secondDeviceID);
1835                                 vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 0, (const VkBufferMemoryBarrier*)DE_NULL, 1u, &preImageBarrier);
1836                                 vk.cmdCopyImage(*cmdBuffer, *renderImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, *peerImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &imageCopy);
1837                                 vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 0, (const VkBufferMemoryBarrier*)DE_NULL, 1u, &postImageBarrier);
1838                                 VK_CHECK(vk.endCommandBuffer(*cmdBuffer));
1839                         }
1840
1841                         // Submit & wait for completion
1842                         {
1843                                 const deUint32 deviceMask = 1 << secondDeviceID;
1844                                 deviceGroupSubmitInfo.pCommandBufferDeviceMasks = &deviceMask;
1845                                 const VkFenceCreateInfo fenceParams =
1846                                 {
1847                                         VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,    // sType
1848                                         DE_NULL,                                                                // pNext
1849                                         0u,                                                                             // flags
1850                                 };
1851                                 const VkSubmitInfo              submitInfo =
1852                                 {
1853                                         VK_STRUCTURE_TYPE_SUBMIT_INFO,                  // sType
1854                                         &deviceGroupSubmitInfo,                                 // pNext
1855                                         0u,                                                                             // waitSemaphoreCount
1856                                         DE_NULL,                                                                // pWaitSemaphores
1857                                         (const VkPipelineStageFlags*)DE_NULL,
1858                                         1u,                                                                             // commandBufferCount
1859                                         &cmdBuffer.get(),                                               // pCommandBuffers
1860                                         0u,                                                                             // signalSemaphoreCount
1861                                         DE_NULL,                                                                // pSignalSemaphores
1862                                 };
1863                                 const Unique<VkFence>   fence(createFence(vk, *m_deviceGroup, &fenceParams));
1864
1865                                 VK_CHECK(vk.queueSubmit(m_deviceGroupQueue, 1u, &submitInfo, *fence));
1866                                 VK_CHECK(vk.waitForFences(*m_deviceGroup, 1u, &fence.get(), DE_TRUE, ~0ull));
1867                                 VK_CHECK(vk.deviceWaitIdle(*m_deviceGroup));
1868                         }
1869                 }
1870
1871                 // copy image to read buffer for checking
1872                 {
1873                         const VkDeviceSize                      imageSizeBytes = (VkDeviceSize)(sizeof(deUint32) * renderSize.x() * renderSize.y());
1874                         const VkBufferCreateInfo        readImageBufferParams =
1875                         {
1876                                 VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,           // sType
1877                                 DE_NULL,                                                                        // pNext
1878                                 (VkBufferCreateFlags)0u,                                        // flags
1879                                 imageSizeBytes,                                                         // size
1880                                 VK_BUFFER_USAGE_TRANSFER_DST_BIT,                       // usage
1881                                 VK_SHARING_MODE_EXCLUSIVE,                                      // sharingMode
1882                                 1u,                                                                                     // queueFamilyIndexCount
1883                                 &queueFamilyIndex,                                                      // pQueueFamilyIndices
1884                         };
1885                         const Unique<VkBuffer>          readImageBuffer(createBuffer(vk, *m_deviceGroup, &readImageBufferParams));
1886                         const UniquePtr<Allocation>     readImageBufferMemory(memAlloc.allocate(getBufferMemoryRequirements(vk, *m_deviceGroup, *readImageBuffer), MemoryRequirement::HostVisible));
1887                         VK_CHECK(vk.bindBufferMemory(*m_deviceGroup, *readImageBuffer, readImageBufferMemory->getMemory(), readImageBufferMemory->getOffset()));
1888
1889                         VK_CHECK(vk.beginCommandBuffer(*cmdBuffer, &cmdBufBeginParams));
1890
1891                         // Copy image to buffer
1892                         {
1893                                 const VkBufferImageCopy copyParams =
1894                                 {
1895                                         (VkDeviceSize)0u,                                               // bufferOffset
1896                                         renderSize.x(),                                                 // bufferRowLength
1897                                         renderSize.y(),                                                 // bufferImageHeight
1898                                         {
1899                                                 VK_IMAGE_ASPECT_COLOR_BIT,                      // aspectMask
1900                                                 0u,                                                                     // mipLevel
1901                                                 0u,                                                                     // baseArrayLayer
1902                                                 1u,                                                                     // layerCount
1903                                         },                                                                              // imageSubresource
1904                                         { 0, 0, 0 },                                                    // imageOffset
1905                                         {
1906                                                 renderSize.x(),
1907                                                 renderSize.y(),
1908                                                 1u
1909                                         }                                                                               // imageExtent
1910                                 };
1911                                 vk.cmdCopyImageToBuffer(*cmdBuffer, *readImage, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, *readImageBuffer, 1u, &copyParams);
1912
1913                                 const VkBufferMemoryBarrier     copyFinishBarrier =
1914                                 {
1915                                         VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,        // sType
1916                                         DE_NULL,                                                                        // pNext
1917                                         VK_ACCESS_TRANSFER_WRITE_BIT,                           // srcAccessMask
1918                                         VK_ACCESS_HOST_READ_BIT,                                        // dstAccessMask
1919                                         queueFamilyIndex,                                                       // srcQueueFamilyIndex
1920                                         queueFamilyIndex,                                                       // dstQueueFamilyIndex
1921                                         *readImageBuffer,                                                       // buffer
1922                                         0u,                                                                                     // offset
1923                                         imageSizeBytes                                                          // size
1924                                 };
1925                                 vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, (VkDependencyFlags)0, 0, (const VkMemoryBarrier*)DE_NULL, 1, &copyFinishBarrier, 0, (const VkImageMemoryBarrier*)DE_NULL);
1926                         }
1927                         VK_CHECK(vk.endCommandBuffer(*cmdBuffer));
1928
1929                         // Submit & wait for completion
1930                         {
1931                                 const deUint32 deviceMask = 1 << firstDeviceID;
1932                                 deviceGroupSubmitInfo.pCommandBufferDeviceMasks = &deviceMask;
1933                                 const VkFenceCreateInfo fenceParams =
1934                                 {
1935                                         VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,    // sType
1936                                         DE_NULL,                                                                // pNext
1937                                         0u,                                                                             // flags
1938                                 };
1939                                 const VkSubmitInfo              submitInfo =
1940                                 {
1941                                         VK_STRUCTURE_TYPE_SUBMIT_INFO,                  // sType
1942                                         &deviceGroupSubmitInfo,                                 // pNext
1943                                         0u,                                                                             // waitSemaphoreCount
1944                                         DE_NULL,                                                                // pWaitSemaphores
1945                                         (const VkPipelineStageFlags*)DE_NULL,   // flags
1946                                         1u,                                                                             // commandBufferCount
1947                                         &cmdBuffer.get(),                                               // pCommandBuffers
1948                                         0u,                                                                             // signalSemaphoreCount
1949                                         DE_NULL,                                                                // pSignalSemaphores
1950                                 };
1951                                 const Unique<VkFence>   fence(createFence(vk, *m_deviceGroup, &fenceParams));
1952
1953                                 VK_CHECK(vk.queueSubmit(m_deviceGroupQueue, 1u, &submitInfo, *fence));
1954                                 VK_CHECK(vk.waitForFences(*m_deviceGroup, 1u, &fence.get(), DE_TRUE, ~0ull));
1955                                 VK_CHECK(vk.deviceWaitIdle(*m_deviceGroup));
1956                         }
1957
1958                         // Read results and check against reference image
1959                         if (m_drawTessellatedSphere)
1960                         {
1961                                 const tcu::TextureFormat                        tcuFormat = vk::mapVkFormat(colorFormat);
1962                                 const VkMappedMemoryRange                       range =
1963                                 {
1964                                         VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,  // sType
1965                                         DE_NULL,                                                                // pNext
1966                                         readImageBufferMemory->getMemory(),             // memory
1967                                         0,                                                                              // offset
1968                                         imageSizeBytes,                                                 // size
1969                                 };
1970                                 const tcu::ConstPixelBufferAccess       resultAccess(tcuFormat, renderSize.x(), renderSize.y(), 1, readImageBufferMemory->getHostPtr());
1971                                 VK_CHECK(vk.invalidateMappedMemoryRanges(*m_deviceGroup, 1u, &range));
1972
1973                                 tcu::TextureLevel referenceImage;
1974                                 string refImage = m_fillModeNonSolid ? "vulkan/data/device_group/sphere.png" : "vulkan/data/device_group/spherefilled.png";
1975                                 tcu::ImageIO::loadPNG(referenceImage, m_context.getTestContext().getArchive(),  refImage.c_str());
1976                                 iterateResultSuccess = tcu::fuzzyCompare(m_context.getTestContext().getLog(), "ImageComparison", "Image Comparison",
1977                                                                                 referenceImage.getAccess(), resultAccess, 0.001f, tcu::COMPARE_LOG_RESULT);
1978                         }
1979                         else
1980                         {
1981                                 const tcu::TextureFormat                        tcuFormat = vk::mapVkFormat(colorFormat);
1982                                 const VkMappedMemoryRange                       range =
1983                                 {
1984                                         VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,  // sType
1985                                         DE_NULL,                                                                // pNext
1986                                         readImageBufferMemory->getMemory(),             // memory
1987                                         0,                                                                              // offset
1988                                         imageSizeBytes,                                                 // size
1989                                 };
1990                                 const tcu::ConstPixelBufferAccess       resultAccess(tcuFormat, renderSize.x(), renderSize.y(), 1, readImageBufferMemory->getHostPtr());
1991                                 VK_CHECK(vk.invalidateMappedMemoryRanges(*m_deviceGroup, 1u, &range));
1992
1993                                 // Render reference and compare
1994                                 {
1995                                         tcu::TextureLevel       refImage(tcuFormat, (deInt32)renderSize.x(), (deInt32)renderSize.y());
1996                                         const tcu::UVec4        threshold(0u);
1997                                         const tcu::IVec3        posDeviation(1, 1, 0);
1998
1999                                         tcu::clear(refImage.getAccess(), clearColor);
2000                                         renderReferenceTriangle(refImage.getAccess(), triVertices);
2001
2002                                         iterateResultSuccess = tcu::intThresholdPositionDeviationCompare(m_context.getTestContext().getLog(),
2003                                                 "ComparisonResult",
2004                                                 "Image comparison result",
2005                                                 refImage.getAccess(),
2006                                                 resultAccess,
2007                                                 threshold,
2008                                                 posDeviation,
2009                                                 false,
2010                                                 tcu::COMPARE_LOG_RESULT);
2011                                 }
2012                         }
2013                 }
2014
2015                 if (!iterateResultSuccess)
2016                         return tcu::TestStatus::fail("Image comparison failed");
2017         }
2018
2019         return tcu::TestStatus(QP_TEST_RESULT_PASS, "Device group verification passed");
2020 }
2021
2022 template<class Instance>
2023 class DeviceGroupTestCase : public TestCase
2024 {
2025 public:
2026         DeviceGroupTestCase (tcu::TestContext& context,
2027                                                 const char*     name,
2028                                                 const char*     description,
2029                                                 deUint32 mode)
2030         : TestCase(context, name, description)
2031         , m_testMode            (mode)
2032         {}
2033
2034 private:
2035
2036         deUint32                        m_testMode;
2037
2038         TestInstance*           createInstance  (Context& context) const
2039         {
2040                 return new Instance(context, m_testMode);
2041         }
2042
2043         void                            initPrograms    (vk::SourceCollections& programCollection) const
2044         {
2045                 programCollection.glslSources.add("vert") << glu::VertexSource("#version 430\n"
2046                         "layout(location = 0) in vec4 in_Position;\n"
2047                         "out gl_PerVertex { vec4 gl_Position; float gl_PointSize; };\n"
2048                         "void main() {\n"
2049                         "       gl_Position     = in_Position;\n"
2050                         "       gl_PointSize = 1.0;\n"
2051                         "}\n");
2052
2053                 if (m_testMode & TEST_MODE_TESSELLATION)
2054                 {
2055                         programCollection.glslSources.add("tesc") << glu::TessellationControlSource("#version 450\n"
2056                                 "#extension GL_EXT_tessellation_shader : require\n"
2057                                 "layout(vertices=3) out;\n"
2058                                 "layout(set=0, binding=1) buffer tessLevel { \n"
2059                                 "  float tessLvl;\n"
2060                                 "};\n"
2061                                 "void main()\n"
2062                                 "{\n"
2063                                 "  gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position;\n"
2064                                 "  if (gl_InvocationID == 0) {\n"
2065                                 "    for (int i = 0; i < 4; i++)\n"
2066                                 "      gl_TessLevelOuter[i] = tessLvl;\n"
2067                                 "    for (int i = 0; i < 2; i++)\n"
2068                                 "      gl_TessLevelInner[i] = tessLvl;\n"
2069                                 "  }\n"
2070                                 "}\n");
2071
2072                         programCollection.glslSources.add("tese") << glu::TessellationEvaluationSource("#version 450\n"
2073                                 "#extension GL_EXT_tessellation_shader : require\n"
2074                                 "layout(triangles) in;\n"
2075                                 "layout(equal_spacing) in;\n"
2076                                 "layout(ccw) in;\n"
2077                                 "void main()\n"
2078                                 "{\n"
2079                                 "  vec4 pos = vec4(0, 0, 0, 0);\n"
2080                                 "  vec3 tessCoord = gl_TessCoord.xyz;\n"
2081                                 "  pos += tessCoord.z * gl_in[0].gl_Position;\n"
2082                                 "  pos += tessCoord.x * gl_in[1].gl_Position;\n"
2083                                 "  pos += tessCoord.y * gl_in[2].gl_Position;\n"
2084                                 "  vec3 sign = sign(pos.xyz);\n"
2085                                 "  pos.xyz = 0.785398 - abs(pos.xyz) * 1.5707963;\n"
2086                                 "  pos.xyz = (1 - tan(pos.xyz))/2.0;\n"
2087                                 "  pos.xyz = (sign * pos.xyz) / length(pos.xyz);\n"
2088                                 "  gl_Position = pos;\n"
2089                                 "}\n");
2090                 }
2091
2092                 programCollection.glslSources.add("frag") << glu::FragmentSource("#version 430\n"
2093                         "layout(location = 0) out vec4 out_FragColor;\n"
2094                         "layout(std140, set=0, binding=0) uniform bufferData { \n"
2095                         "       vec4 color;\n"
2096                         "};\n"
2097                         "void main()\n"
2098                         "{\n"
2099                         "       out_FragColor = color;\n"
2100                         "}\n");
2101         }
2102 };
2103
2104 } //anonymous
2105
2106 class DeviceGroupTestRendering : public tcu::TestCaseGroup
2107 {
2108 public:
2109                                                                 DeviceGroupTestRendering        (tcu::TestContext& testCtx);
2110                                                                 ~DeviceGroupTestRendering       (void) {}
2111         void                                            init(void);
2112
2113 private:
2114                                                                 DeviceGroupTestRendering        (const DeviceGroupTestRendering& other);
2115         DeviceGroupTestRendering&       operator=                                       (const DeviceGroupTestRendering& other);
2116 };
2117
2118 DeviceGroupTestRendering::DeviceGroupTestRendering (tcu::TestContext& testCtx)
2119         : TestCaseGroup (testCtx, "device_group", "Testing device group test cases")
2120 {
2121         // Left blank on purpose
2122 }
2123
2124 void DeviceGroupTestRendering::init (void)
2125 {
2126         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "sfr",                                                     "Test split frame rendering",                                                                                                           TEST_MODE_SFR));
2127         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "sfr_sys",                                         "Test split frame rendering with render target in host memory",                                         TEST_MODE_SFR | TEST_MODE_HOSTMEMORY));
2128         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "sfr_dedicated",                           "Test split frame rendering with dedicated memory allocations",                                         TEST_MODE_SFR | TEST_MODE_DEDICATED));
2129         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "sfr_dedicated_peer",                      "Test split frame rendering with dedicated memory allocations and peer fetching",       TEST_MODE_SFR | TEST_MODE_DEDICATED | TEST_MODE_PEER_FETCH));
2130
2131         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "afr",                                                     "Test alternate frame rendering",                                                                                                       TEST_MODE_AFR));
2132         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "afr_sys",                                         "Test split frame rendering with render target in host memory",                                         TEST_MODE_AFR | TEST_MODE_HOSTMEMORY));
2133         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "afr_dedicated",                           "Test split frame rendering with dedicated memory allocations",                                         TEST_MODE_AFR | TEST_MODE_DEDICATED));
2134         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "afr_dedicated_peer",                      "Test split frame rendering with dedicated memory allocations and peer fetching",       TEST_MODE_AFR | TEST_MODE_DEDICATED | TEST_MODE_PEER_FETCH));
2135
2136         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "sfr_tessellated",                         "Test split frame rendering with tessellated sphere",                                                           TEST_MODE_SFR | TEST_MODE_TESSELLATION | TEST_MODE_DEDICATED | TEST_MODE_PEER_FETCH));
2137         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "sfr_tessellated_linefill",        "Test split frame rendering with tessellated sphere with line segments",                        TEST_MODE_SFR | TEST_MODE_TESSELLATION | TEST_MODE_LINEFILL  | TEST_MODE_DEDICATED | TEST_MODE_PEER_FETCH));
2138         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "afr_tessellated",                         "Test alternate frame rendering with tesselated sphere",                                                        TEST_MODE_AFR | TEST_MODE_TESSELLATION | TEST_MODE_DEDICATED | TEST_MODE_PEER_FETCH));
2139         addChild(new DeviceGroupTestCase<DeviceGroupTestInstance>(m_testCtx, "afr_tessellated_linefill",        "Test alternate frame rendering with tesselated sphere with line segments",                     TEST_MODE_AFR | TEST_MODE_TESSELLATION | TEST_MODE_LINEFILL  | TEST_MODE_DEDICATED | TEST_MODE_PEER_FETCH));
2140 }
2141
2142 tcu::TestCaseGroup* createTests(tcu::TestContext& testCtx)
2143 {
2144         return new DeviceGroupTestRendering(testCtx);
2145 }
2146 }       // DeviceGroup
2147 }       // vkt