From 9bdeac8cd1b5e1bb81ee694dfeda501802b10df0 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Antto=20M=C3=A4kinen?= Date: Wed, 27 Oct 2021 11:33:35 +0300 Subject: [PATCH] Ensure that compute shaders have a subgroup size that is uniform in command scope If the pipeline was created with the VK_PIPELINE_SHADER_STAGE_CREATE_ALLOW_VARYING_SUBGROUP_SIZE_BIT_EXT flag the SubgroupSize may vary in the shader stage but it must be uniform with command scope. VK-GL-CTS issue: 3036 New Tests: dEQP-VK.subgroups.multiple_dispatches.uniform_subgroup_size Components: Vulkan Change-Id: I55a082af6e1b325be7b29c50d49bcb822d81c33e --- AndroidGen.mk | 1 + .../cts/master/vk-master-2021-03-01/subgroups.txt | 1 + android/cts/master/vk-master/subgroups.txt | 1 + .../modules/vulkan/subgroups/CMakeLists.txt | 2 + ...sMultipleDispatchesUniformSubgroupSizeTests.cpp | 309 +++++++++++++++++++++ ...sMultipleDispatchesUniformSubgroupSizeTests.hpp | 40 +++ .../modules/vulkan/subgroups/vktSubgroupsTests.cpp | 2 + .../mustpass/master/vk-default/subgroups.txt | 1 + 8 files changed, 357 insertions(+) create mode 100644 external/vulkancts/modules/vulkan/subgroups/vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.cpp create mode 100644 external/vulkancts/modules/vulkan/subgroups/vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.hpp diff --git a/AndroidGen.mk b/AndroidGen.mk index 39405a7..3be17e7 100644 --- a/AndroidGen.mk +++ b/AndroidGen.mk @@ -462,6 +462,7 @@ LOCAL_SRC_FILES := \ external/vulkancts/modules/vulkan/subgroups/vktSubgroupsBuiltinMaskVarTests.cpp \ external/vulkancts/modules/vulkan/subgroups/vktSubgroupsBuiltinVarTests.cpp \ external/vulkancts/modules/vulkan/subgroups/vktSubgroupsClusteredTests.cpp \ + external/vulkancts/modules/vulkan/subgroups/vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.cpp \ external/vulkancts/modules/vulkan/subgroups/vktSubgroupsPartitionedTests.cpp \ external/vulkancts/modules/vulkan/subgroups/vktSubgroupsQuadTests.cpp \ external/vulkancts/modules/vulkan/subgroups/vktSubgroupsScanHelpers.cpp \ diff --git a/android/cts/master/vk-master-2021-03-01/subgroups.txt b/android/cts/master/vk-master-2021-03-01/subgroups.txt index 5f647fe..9ade86b 100644 --- a/android/cts/master/vk-master-2021-03-01/subgroups.txt +++ b/android/cts/master/vk-master-2021-03-01/subgroups.txt @@ -4881,6 +4881,7 @@ dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgroup dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgroupgtmaskarb dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgrouplemaskarb dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgroupltmaskarb +dEQP-VK.subgroups.multiple_dispatches.uniform_subgroup_size dEQP-VK.subgroups.size_control.generic.subgroup_size_properties dEQP-VK.subgroups.size_control.graphics.allow_varying_subgroup_size dEQP-VK.subgroups.size_control.graphics.required_subgroup_size_max diff --git a/android/cts/master/vk-master/subgroups.txt b/android/cts/master/vk-master/subgroups.txt index 5c8074b..8c38cc3 100644 --- a/android/cts/master/vk-master/subgroups.txt +++ b/android/cts/master/vk-master/subgroups.txt @@ -20865,6 +20865,7 @@ dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgroup dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgroupgtmaskarb dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgrouplemaskarb dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgroupltmaskarb +dEQP-VK.subgroups.multiple_dispatches.uniform_subgroup_size dEQP-VK.subgroups.size_control.generic.subgroup_size_properties dEQP-VK.subgroups.size_control.graphics.allow_varying_subgroup_size dEQP-VK.subgroups.size_control.graphics.required_subgroup_size_max diff --git a/external/vulkancts/modules/vulkan/subgroups/CMakeLists.txt b/external/vulkancts/modules/vulkan/subgroups/CMakeLists.txt index e672eb2..14b1f5e 100644 --- a/external/vulkancts/modules/vulkan/subgroups/CMakeLists.txt +++ b/external/vulkancts/modules/vulkan/subgroups/CMakeLists.txt @@ -39,6 +39,8 @@ set(DEQP_VK_SUBGROUPS_SRCS vktSubgroupsSizeControlTests.hpp vktSubgroupUniformControlFlowTests.cpp vktSubgroupUniformControlFlowTests.hpp + vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.cpp + vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.hpp ) set(DEQP_VK_SUBGROUPS_LIBS diff --git a/external/vulkancts/modules/vulkan/subgroups/vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.cpp b/external/vulkancts/modules/vulkan/subgroups/vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.cpp new file mode 100644 index 0000000..32d2643 --- /dev/null +++ b/external/vulkancts/modules/vulkan/subgroups/vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.cpp @@ -0,0 +1,309 @@ +/*------------------------------------------------------------------------ + * Vulkan Conformance Tests + * ------------------------ + * + * Copyright (c) 2021 Google LLC. + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + *//*! + * \file + * \brief Tests that compute shaders have a subgroup size that is uniform in + * command scope. + *//*--------------------------------------------------------------------*/ + +#include "deUniquePtr.hpp" + +#include "vkRef.hpp" +#include "vkRefUtil.hpp" +#include "vkPrograms.hpp" +#include "vkMemUtil.hpp" +#include "vkBuilderUtil.hpp" +#include "vkCmdUtil.hpp" +#include "vkObjUtil.hpp" +#include "vkTypeUtil.hpp" +#include "vkImageWithMemory.hpp" +#include "vkBarrierUtil.hpp" + +#include "vktTestCaseUtil.hpp" + +using namespace vk; + +namespace vkt +{ +namespace subgroups +{ +namespace +{ +using std::vector; +using de::MovePtr; + +class MultipleDispatchesUniformSubgroupSizeInstance : public TestInstance +{ +public: + MultipleDispatchesUniformSubgroupSizeInstance (Context& context); + tcu::TestStatus iterate (void); +}; + +MultipleDispatchesUniformSubgroupSizeInstance::MultipleDispatchesUniformSubgroupSizeInstance (Context& context) + :TestInstance (context) +{ +} + +tcu::TestStatus MultipleDispatchesUniformSubgroupSizeInstance::iterate (void) +{ + const DeviceInterface& vk = m_context.getDeviceInterface(); + const VkDevice device = m_context.getDevice(); + Allocator& allocator = m_context.getDefaultAllocator(); + const VkQueue queue = m_context.getUniversalQueue(); + const deUint32 queueFamilyIndex = m_context.getUniversalQueueFamilyIndex(); + + const Move cmdPool = createCommandPool(vk, device, VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, queueFamilyIndex); + const Move cmdBuffer = allocateCommandBuffer(vk, device, *cmdPool, VK_COMMAND_BUFFER_LEVEL_PRIMARY); + + Move computeShader = createShaderModule (vk, device, m_context.getBinaryCollection().get("comp"), 0u); + + // The number of invocations in a workgroup. + const deUint32 maxLocalSize = m_context.getDeviceProperties().limits.maxComputeWorkGroupSize[0]; + + // Create a storage buffer to hold the sizes of subgroups. + const VkDeviceSize bufferSize = maxLocalSize * 2 * sizeof(deUint32); + + const VkBufferCreateInfo resultBufferCreateInfo = makeBufferCreateInfo(bufferSize, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + Move resultBuffer = createBuffer(vk, device, &resultBufferCreateInfo); + MovePtr resultBufferMemory = allocator.allocate(getBufferMemoryRequirements(vk, device, *resultBuffer), MemoryRequirement::HostVisible); + + VK_CHECK(vk.bindBufferMemory(device, *resultBuffer, resultBufferMemory->getMemory(), resultBufferMemory->getOffset())); + + // Build descriptors for the storage buffer + const Unique descriptorPool (DescriptorPoolBuilder().addType(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) + .build(vk, device, VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT, 1u)); + const auto descriptorSetLayout1 (DescriptorSetLayoutBuilder().addSingleBinding(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC, VK_SHADER_STAGE_COMPUTE_BIT) + .build(vk, device)); + const VkDescriptorBufferInfo resultInfo = makeDescriptorBufferInfo(*resultBuffer, 0u, + (VkDeviceSize) bufferSize - maxLocalSize * sizeof(deUint32)); + + const VkDescriptorSetAllocateInfo allocInfo = + { + VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, // sType + DE_NULL, // pNext + *descriptorPool, // descriptorPool + 1u, // descriptorSetCount + &(*descriptorSetLayout1) // pSetLayouts + }; + + Move descriptorSet = allocateDescriptorSet(vk, device, &allocInfo); + DescriptorSetUpdateBuilder builder; + + builder.writeSingle(*descriptorSet, DescriptorSetUpdateBuilder::Location::binding(0u), VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC, &resultInfo); + builder.update(vk, device); + + // Compute pipeline + const Move computePipelineLayout = makePipelineLayout (vk, device, *descriptorSetLayout1); + + for (deUint32 localSize1 = 8; localSize1 < maxLocalSize + 1; localSize1 *= 2) + { + for (deUint32 localSize2 = 8; localSize2 < maxLocalSize + 1; localSize2 *= 2) + { + // On each iteration, change the number of invocations which might affect + // the subgroup size if the driver doesn't behave as expected. + const VkSpecializationMapEntry entries = + { + 0u, // deUint32 constantID; + 0u, // deUint32 offset; + sizeof(localSize1) // size_t size; + }; + const VkSpecializationInfo specInfo = + { + 1, // mapEntryCount + &entries, // pMapEntries + sizeof(localSize1), // dataSize + &localSize1 // pData + }; + const VkSpecializationInfo specInfo2 = + { + 1, // mapEntryCount + &entries, // pMapEntries + sizeof(localSize2), // dataSize + &localSize2 // pData + }; + + const VkPipelineShaderStageCreateInfo shaderStageCreateInfo = + { + VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, // sType + DE_NULL, // pNext + VK_PIPELINE_SHADER_STAGE_CREATE_ALLOW_VARYING_SUBGROUP_SIZE_BIT_EXT, // flags + VK_SHADER_STAGE_COMPUTE_BIT, // stage + *computeShader, // module + "main", // pName + &specInfo, // pSpecializationInfo + }; + + const VkPipelineShaderStageCreateInfo shaderStageCreateInfo2 = + { + VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, // sType + DE_NULL, // pNext + VK_PIPELINE_SHADER_STAGE_CREATE_ALLOW_VARYING_SUBGROUP_SIZE_BIT_EXT, // flags + VK_SHADER_STAGE_COMPUTE_BIT, // stage + *computeShader, // module + "main", // pName + &specInfo2, // pSpecializationInfo + }; + + const VkComputePipelineCreateInfo pipelineCreateInfo = + { + VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, // sType + DE_NULL, // pNext + 0u, // flags + shaderStageCreateInfo, // stage + *computePipelineLayout, // layout + (VkPipeline) 0, // basePipelineHandle + 0u, // basePipelineIndex + }; + + const VkComputePipelineCreateInfo pipelineCreateInfo2 = + { + VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, // sType + DE_NULL, // pNext + 0u, // flags + shaderStageCreateInfo2, // stage + *computePipelineLayout, // layout + (VkPipeline) 0, // basePipelineHandle + 0u, // basePipelineIndex + }; + + Move computePipeline = createComputePipeline(vk, device, (VkPipelineCache) 0u, &pipelineCreateInfo); + Move computePipeline2 = createComputePipeline(vk, device, (VkPipelineCache) 0u, &pipelineCreateInfo2); + + beginCommandBuffer(vk, *cmdBuffer); + + // Clears the values written on the previous iteration. + vk.cmdFillBuffer(*cmdBuffer, *resultBuffer, 0u, VK_WHOLE_SIZE, 0); + + const deUint32 zero = 0u; + vk.cmdBindDescriptorSets(*cmdBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, *computePipelineLayout, 0u, 1u, &descriptorSet.get(), 1, &zero); + vk.cmdBindPipeline(*cmdBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, *computePipeline); + vk.cmdDispatch(*cmdBuffer, 1, 1, 1); + + const auto barrier = makeBufferMemoryBarrier(VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT, *resultBuffer, 0ull, bufferSize); + vk.cmdPipelineBarrier(*cmdBuffer, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, (VkDependencyFlags) 0, + 0, (const VkMemoryBarrier *) DE_NULL, 1, &barrier, 0, (const VkImageMemoryBarrier *) DE_NULL); + + const deUint32 offset = static_cast(maxLocalSize * sizeof(deUint32)); + vk.cmdBindDescriptorSets(*cmdBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, *computePipelineLayout, 0u, 1u, &descriptorSet.get(), 1u, &offset); + vk.cmdBindPipeline(*cmdBuffer, VK_PIPELINE_BIND_POINT_COMPUTE, *computePipeline2); + vk.cmdDispatch(*cmdBuffer, 1, 1, 1); + + endCommandBuffer(vk, *cmdBuffer); + submitCommandsAndWait(vk, device, queue, *cmdBuffer); + + invalidateAlloc(vk, device, *resultBufferMemory); + + const deUint32 *res = static_cast(resultBufferMemory->getHostPtr()); + deUint32 size = 0; + + // Search for the first nonzero size. Then go through the data of both pipelines and check that + // the first nonzero size matches with other nonzero values. + for (deUint32 i = 0; i < maxLocalSize; i++) + { + if (res[i] != 0) + { + size = res[i]; + break; + } + } + + // Subgroup size is guaranteed to be at least 1. + DE_ASSERT(size > 0); + + for (deUint32 i = 0; i < maxLocalSize * 2; i++) + { + if (size != res[i] && res[i] != 0) + return tcu::TestStatus::fail("Subgroup size not uniform in command scope. " + std::to_string(res[i]) + " != " + std::to_string(size)); + } + } + } + + return tcu::TestStatus::pass("pass"); +} + +class MultipleDispatchesUniformSubgroupSize : public TestCase +{ +public: + MultipleDispatchesUniformSubgroupSize (tcu::TestContext& testCtx, + const std::string& name, + const std::string& description); + + void initPrograms (SourceCollections& programCollection) const; + TestInstance* createInstance (Context& context) const; + virtual void checkSupport (Context& context) const; + +}; + +MultipleDispatchesUniformSubgroupSize::MultipleDispatchesUniformSubgroupSize (tcu::TestContext& testCtx, + const std::string& name, + const std::string& description) + : TestCase (testCtx, name, description) +{ +} + +void MultipleDispatchesUniformSubgroupSize::checkSupport (Context& context) const +{ + const VkPhysicalDeviceSubgroupSizeControlFeaturesEXT& subgroupSizeControlFeatures = context.getSubgroupSizeControlFeaturesEXT(); + + if (subgroupSizeControlFeatures.subgroupSizeControl == DE_FALSE) + TCU_THROW(NotSupportedError, "Device does not support varying subgroup sizes"); +} + +void MultipleDispatchesUniformSubgroupSize::initPrograms (SourceCollections& programCollection) const +{ + std::ostringstream computeSrc; + computeSrc + << glu::getGLSLVersionDeclaration(glu::GLSL_VERSION_450) << "\n" + << "#extension GL_KHR_shader_subgroup_basic : enable\n" + << "#extension GL_KHR_shader_subgroup_vote : enable\n" + << "#extension GL_KHR_shader_subgroup_ballot : enable\n" + << "layout(std430, binding = 0) buffer Outputs { uint sizes[]; };\n" + + << "layout(local_size_x_id = 0) in;\n" + + << "void main()\n" + << "{\n" + << " if (subgroupElect())\n" + << " {\n" + << " sizes[gl_WorkGroupID.x * gl_NumSubgroups + gl_SubgroupID] = gl_SubgroupSize;\n" + << " }\n" + << "}\n"; + + programCollection.glslSources.add("comp") << glu::ComputeSource(computeSrc.str()) + << ShaderBuildOptions(programCollection.usedVulkanVersion, SPIRV_VERSION_1_3, 0u); +} + +TestInstance* MultipleDispatchesUniformSubgroupSize::createInstance (Context& context) const +{ + return new MultipleDispatchesUniformSubgroupSizeInstance(context); +} + +} // anonymous ns + +tcu::TestCaseGroup* createMultipleDispatchesUniformSubgroupSizeTests (tcu::TestContext& testCtx) +{ + de::MovePtr testGroup(new tcu::TestCaseGroup(testCtx, "multiple_dispatches", "Multiple dispatches uniform subgroup size tests")); + + testGroup->addChild(new MultipleDispatchesUniformSubgroupSize(testCtx, "uniform_subgroup_size", "")); + return testGroup.release(); +} + +} // compute +} // vkt diff --git a/external/vulkancts/modules/vulkan/subgroups/vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.hpp b/external/vulkancts/modules/vulkan/subgroups/vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.hpp new file mode 100644 index 0000000..193283b --- /dev/null +++ b/external/vulkancts/modules/vulkan/subgroups/vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.hpp @@ -0,0 +1,40 @@ +#ifndef _VKTSUBGROUPSMULTIPLEDISPATCHESUNIFORMSUBGROUPSIZETESTS_HPP +#define _VKTSUBGROUPSMULTIPLEDISPATCHESUNIFORMSUBGROUPSIZETESTS_HPP +/*------------------------------------------------------------------------ + * Vulkan Conformance Tests + * ------------------------ + * + * Copyright (c) 2021 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + *//*! + * \file + * \brief Tests that compute shaders have a subgroup size that is uniform in + * command scope. + *//*--------------------------------------------------------------------*/ + +#include "tcuDefs.hpp" +#include "vktTestCase.hpp" + +namespace vkt +{ +namespace subgroups +{ + +tcu::TestCaseGroup* createMultipleDispatchesUniformSubgroupSizeTests (tcu::TestContext& testCtx); + +} // subgroups +} // vkt + +#endif // _VKTSUBGROUPSMULTIPLEDISPATCHESUNIFORMSUBGROUPSIZETESTS_HPP diff --git a/external/vulkancts/modules/vulkan/subgroups/vktSubgroupsTests.cpp b/external/vulkancts/modules/vulkan/subgroups/vktSubgroupsTests.cpp index 6aed26f..025b7c7 100755 --- a/external/vulkancts/modules/vulkan/subgroups/vktSubgroupsTests.cpp +++ b/external/vulkancts/modules/vulkan/subgroups/vktSubgroupsTests.cpp @@ -39,6 +39,7 @@ #include "vktSubgroupsBallotMasksTests.hpp" #include "vktSubgroupsSizeControlTests.hpp" #include "vktSubgroupUniformControlFlowTests.hpp" +#include "vktSubgroupsMultipleDispatchesUniformSubgroupSizeTests.hpp" #include "vktTestGroupUtil.hpp" namespace vkt @@ -67,6 +68,7 @@ void createChildren(tcu::TestCaseGroup* subgroupsTests) subgroupsTests->addChild(createSubgroupsQuadTests(testCtx)); subgroupsTests->addChild(createSubgroupsShapeTests(testCtx)); subgroupsTests->addChild(createSubgroupsBallotMasksTests(testCtx)); + subgroupsTests->addChild(createMultipleDispatchesUniformSubgroupSizeTests(testCtx)); subgroupsTests->addChild(createSubgroupsSizeControlTests(testCtx)); subgroupsTests->addChild(createSubgroupUniformControlFlowTests(testCtx)); } diff --git a/external/vulkancts/mustpass/master/vk-default/subgroups.txt b/external/vulkancts/mustpass/master/vk-default/subgroups.txt index 5c8074b..8c38cc3 100644 --- a/external/vulkancts/mustpass/master/vk-default/subgroups.txt +++ b/external/vulkancts/mustpass/master/vk-default/subgroups.txt @@ -20865,6 +20865,7 @@ dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgroup dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgroupgtmaskarb dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgrouplemaskarb dEQP-VK.subgroups.ballot_mask.ext_shader_subgroup_ballot.ray_tracing.gl_subgroupltmaskarb +dEQP-VK.subgroups.multiple_dispatches.uniform_subgroup_size dEQP-VK.subgroups.size_control.generic.subgroup_size_properties dEQP-VK.subgroups.size_control.graphics.allow_varying_subgroup_size dEQP-VK.subgroups.size_control.graphics.required_subgroup_size_max -- 2.7.4