Loader single EnumPhysDev call through layers
authorMark Young <marky@lunarg.com>
Thu, 2 Dec 2021 23:55:13 +0000 (16:55 -0700)
committerMark Young <marky@lunarg.com>
Wed, 16 Feb 2022 15:29:42 +0000 (08:29 -0700)
The loader trampoline previously would query all devices every time
vkEnumeratePhysicalDevices was called.
To do this, it would make two calls every time:
  - First, it would ignore the passed in user values
  - Second, it would query the total number of available devices.
  - Third, it would query the values for every available device

This resulted in layers reporting 2 vkEnumeratePhysicalDevices call for
every 1 the application made which could get very polluted in output.
It didn't break any functionality, just made things messy.

This change removes that behavior and adds a bunch of test cases to verify
nothing broke in the move.

17 files changed:
loader/loader.c
loader/loader.h
loader/loader_common.h
loader/loader_linux.c
loader/trampoline.c
tests/framework/framework_config.h.in
tests/framework/icd/physical_device.h
tests/framework/icd/test_icd.cpp
tests/framework/icd/test_icd.h
tests/framework/layer/CMakeLists.txt
tests/framework/layer/test_layer.cpp
tests/framework/layer/test_layer.h
tests/framework/layer/wrap_objects.cpp
tests/loader_debug_ext_tests.cpp
tests/loader_layer_tests.cpp
tests/loader_regression_tests.cpp
tests/loader_version_tests.cpp

index 3e563a0f2ad296aa25218ea160e4f35ab47a1446..38df6b37a9ada0d022863a87337815e65090f988 100644 (file)
@@ -6033,85 +6033,132 @@ out:
     return res;
 }
 
-VkResult setup_loader_tramp_phys_devs(struct loader_instance *inst) {
+// Update the trampoline physical devices with the wrapped version.
+// We always want to re-use previous physical device pointers since they may be used by an application
+// after returning previously.
+VkResult setup_loader_tramp_phys_devs(struct loader_instance *inst, uint32_t phys_dev_count, VkPhysicalDevice *phys_devs) {
     VkResult res = VK_SUCCESS;
-    VkPhysicalDevice *local_phys_devs = NULL;
-    uint32_t total_count = 0;
+    uint32_t cur_idx;
+    uint32_t new_idx;
+    uint32_t found_count = 0;
+    uint32_t old_count = inst->phys_dev_count_tramp;
+    uint32_t new_count = inst->total_gpu_count;
     struct loader_physical_device_tramp **new_phys_devs = NULL;
 
-    // Query how many GPUs there
-    res = inst->disp->layer_inst_disp.EnumeratePhysicalDevices(inst->instance, &total_count, NULL);
-    if (res != VK_SUCCESS) {
-        loader_log(
-            inst, VULKAN_LOADER_ERROR_BIT, 0,
-            "setup_loader_tramp_phys_devs:  Failed during dispatch call of \'vkEnumeratePhysicalDevices\' to lower layers or "
-            "loader to get count.");
-        goto out;
+    if (0 == phys_dev_count) {
+        return VK_SUCCESS;
     }
-
-    // Really use what the total GPU count is since Optimus and other layers may mess
-    // the count up.
-    total_count = inst->total_gpu_count;
-
-    // Create an array for the new physical devices, which will be stored
-    // in the instance for the trampoline code.
-    new_phys_devs = (struct loader_physical_device_tramp **)loader_instance_heap_alloc(
-        inst, total_count * sizeof(struct loader_physical_device_tramp *), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
-    if (NULL == new_phys_devs) {
-        loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                   "setup_loader_tramp_phys_devs:  Failed to allocate new physical device array of size %d", total_count);
-        res = VK_ERROR_OUT_OF_HOST_MEMORY;
-        goto out;
+    if (phys_dev_count > new_count) {
+        new_count = phys_dev_count;
     }
-    memset(new_phys_devs, 0, total_count * sizeof(struct loader_physical_device_tramp *));
 
-    // Create a temporary array (on the stack) to keep track of the
-    // returned VkPhysicalDevice values.
-    local_phys_devs = loader_stack_alloc(sizeof(VkPhysicalDevice) * total_count);
-    if (NULL == local_phys_devs) {
-        loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                   "setup_loader_tramp_phys_devs:  Failed to allocate local physical device array of size %d", total_count);
-        res = VK_ERROR_OUT_OF_HOST_MEMORY;
-        goto out;
+    // We want an old to new index array and a new to old index array
+    int32_t *old_to_new_index = (int32_t *)loader_stack_alloc(sizeof(int32_t) * old_count);
+    int32_t *new_to_old_index = (int32_t *)loader_stack_alloc(sizeof(int32_t) * new_count);
+    if (NULL == old_to_new_index || NULL == new_to_old_index) {
+        return VK_ERROR_OUT_OF_HOST_MEMORY;
     }
-    memset(local_phys_devs, 0, sizeof(VkPhysicalDevice) * total_count);
 
-    res = inst->disp->layer_inst_disp.EnumeratePhysicalDevices(inst->instance, &total_count, local_phys_devs);
-    if (VK_SUCCESS != res) {
-        loader_log(
-            inst, VULKAN_LOADER_ERROR_BIT, 0,
-            "setup_loader_tramp_phys_devs:  Failed during dispatch call of \'vkEnumeratePhysicalDevices\' to lower layers or "
-            "loader to get content.");
-        goto out;
+    // Initialize both
+    for (cur_idx = 0; cur_idx < old_count; ++cur_idx) {
+        old_to_new_index[cur_idx] = -1;
+    }
+    for (cur_idx = 0; cur_idx < new_count; ++cur_idx) {
+        new_to_old_index[cur_idx] = -1;
     }
 
-    // Copy or create everything to fill the new array of physical devices
-    for (uint32_t new_idx = 0; new_idx < total_count; new_idx++) {
-        // Check if this physical device is already in the old buffer
-        for (uint32_t old_idx = 0; old_idx < inst->phys_dev_count_tramp; old_idx++) {
-            if (local_phys_devs[new_idx] == inst->phys_devs_tramp[old_idx]->phys_dev) {
-                new_phys_devs[new_idx] = inst->phys_devs_tramp[old_idx];
+    // Figure out the old->new and new->old indices
+    for (cur_idx = 0; cur_idx < old_count; ++cur_idx) {
+        for (new_idx = 0; new_idx < phys_dev_count; ++new_idx) {
+            if (inst->phys_devs_tramp[cur_idx]->phys_dev == phys_devs[new_idx]) {
+                old_to_new_index[cur_idx] = (int32_t)new_idx;
+                new_to_old_index[new_idx] = (int32_t)cur_idx;
+                found_count++;
                 break;
             }
         }
+    }
 
-        // If this physical device isn't in the old buffer, create it
-        if (NULL == new_phys_devs[new_idx]) {
-            new_phys_devs[new_idx] = (struct loader_physical_device_tramp *)loader_instance_heap_alloc(
-                inst, sizeof(struct loader_physical_device_tramp), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
-            if (NULL == new_phys_devs[new_idx]) {
-                loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                           "setup_loader_tramp_phys_devs:  Failed to allocate physical device trampoline object %d", new_idx);
-                total_count = new_idx;
-                res = VK_ERROR_OUT_OF_HOST_MEMORY;
-                goto out;
+    // If we found exactly the number of items we were looking for as we had before.  Then everything
+    // we already have is good enough and we just need to update the array that was passed in with
+    // the loader values.
+    if (found_count == phys_dev_count && 0 != old_count && old_count == new_count) {
+        for (new_idx = 0; new_idx < phys_dev_count; ++new_idx) {
+            for (cur_idx = 0; cur_idx < old_count; ++cur_idx) {
+                if (old_to_new_index[cur_idx] == (int32_t)new_idx) {
+                    phys_devs[new_idx] = (VkPhysicalDevice)inst->phys_devs_tramp[cur_idx];
+                    break;
+                }
+            }
+        }
+        // Nothing else to do for this path
+        return VK_SUCCESS;
+    } else {
+        // Something is different, so do the full path of checking every device and creating a new array to use.
+        // This can happen if a device was added, or removed, or we hadn't previously queried all the data and we
+        // have more to store.
+        new_phys_devs = loader_instance_heap_alloc(inst, sizeof(struct loader_physical_device_tramp *) * new_count,
+                                                   VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
+        if (NULL == new_phys_devs) {
+            loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
+                       "setup_loader_tramp_phys_devs:  Failed to allocate new physical device array of size %d", new_count);
+            res = VK_ERROR_OUT_OF_HOST_MEMORY;
+            goto out;
+        }
+        memset(new_phys_devs, 0, sizeof(struct loader_physical_device_tramp *) * new_count);
+
+        if (new_count > phys_dev_count) {
+            found_count = phys_dev_count;
+        } else {
+            found_count = new_count;
+        }
+
+        // First try to see if an old item exists that matches the new item.  If so, just copy it over.
+        for (new_idx = 0; new_idx < found_count; ++new_idx) {
+            bool old_item_found = false;
+            for (cur_idx = 0; cur_idx < old_count; ++cur_idx) {
+                if (old_to_new_index[cur_idx] == (int32_t)new_idx) {
+                    // Copy over old item to correct spot in the new array
+                    new_phys_devs[new_idx] = inst->phys_devs_tramp[cur_idx];
+                    inst->phys_devs_tramp[cur_idx] = NULL;
+                    old_item_found = true;
+                    break;
+                }
+            }
+            // Something wasn't found, so it's new so add it to the new list
+            if (!old_item_found) {
+                new_phys_devs[new_idx] = loader_instance_heap_alloc(inst, sizeof(struct loader_physical_device_tramp),
+                                                                    VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
+                if (NULL == new_phys_devs[new_idx]) {
+                    loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
+                               "setup_loader_tramp_phys_devs:  Failed to allocate new trampoline physical device");
+                    res = VK_ERROR_OUT_OF_HOST_MEMORY;
+                    goto out;
+                }
+
+                // Initialize the new physicalDevice object
+                loader_set_dispatch((void *)new_phys_devs[new_idx], inst->disp);
+                new_phys_devs[new_idx]->this_instance = inst;
+                new_phys_devs[new_idx]->phys_dev = phys_devs[new_idx];
+                new_phys_devs[new_idx]->magic = PHYS_TRAMP_MAGIC_NUMBER;
             }
 
-            // Initialize the new physicalDevice object
-            loader_set_dispatch((void *)new_phys_devs[new_idx], inst->disp);
-            new_phys_devs[new_idx]->this_instance = inst;
-            new_phys_devs[new_idx]->phys_dev = local_phys_devs[new_idx];
-            new_phys_devs[new_idx]->magic = PHYS_TRAMP_MAGIC_NUMBER;
+            phys_devs[new_idx] = (VkPhysicalDevice)new_phys_devs[new_idx];
+        }
+
+        // We usually get here if the user array is smaller than the total number of devices, so copy the
+        // remaining devices we have over to the new array.
+        uint32_t start = found_count;
+        for (new_idx = start; new_idx < new_count; ++new_idx) {
+            for (cur_idx = 0; cur_idx < old_count; ++cur_idx) {
+                if (old_to_new_index[cur_idx] == -1) {
+                    new_phys_devs[new_idx] = inst->phys_devs_tramp[cur_idx];
+                    inst->phys_devs_tramp[cur_idx] = NULL;
+                    old_to_new_index[cur_idx] = new_idx;
+                    found_count++;
+                    break;
+                }
+            }
         }
     }
 
@@ -6119,47 +6166,41 @@ out:
 
     if (VK_SUCCESS != res) {
         if (NULL != new_phys_devs) {
-            for (uint32_t i = 0; i < total_count; i++) {
+            for (new_idx = 0; new_idx < found_count; ++new_idx) {
                 // If an OOM occurred inside the copying of the new physical devices into the existing array
                 // will leave some of the old physical devices in the array which may have been copied into
                 // the new array, leading to them being freed twice. To avoid this we just make sure to not
                 // delete physical devices which were copied.
                 bool found = false;
-                for (uint32_t old_idx = 0; old_idx < inst->phys_dev_count_tramp; old_idx++) {
-                    if (new_phys_devs[i] == inst->phys_devs_tramp[old_idx]) {
+                for (cur_idx = 0; cur_idx < inst->phys_dev_count_tramp; cur_idx++) {
+                    if (new_phys_devs[new_idx] == inst->phys_devs_tramp[cur_idx]) {
                         found = true;
                         break;
                     }
                 }
                 if (!found) {
-                    loader_instance_heap_free(inst, new_phys_devs[i]);
+                    loader_instance_heap_free(inst, new_phys_devs[new_idx]);
                 }
             }
             loader_instance_heap_free(inst, new_phys_devs);
         }
-        total_count = 0;
+        inst->total_gpu_count = 0;
     } else {
-        // Free everything that didn't carry over to the new array of
-        // physical devices
+        if (new_count > inst->total_gpu_count) {
+            inst->total_gpu_count = new_count;
+        }
+        // Look for any items that were not used this time.
         if (NULL != inst->phys_devs_tramp) {
-            for (uint32_t i = 0; i < inst->phys_dev_count_tramp; i++) {
-                bool found = false;
-                for (uint32_t j = 0; j < total_count; j++) {
-                    if (inst->phys_devs_tramp[i] == new_phys_devs[j]) {
-                        found = true;
-                        break;
-                    }
-                }
-                if (!found) {
-                    loader_instance_heap_free(inst, inst->phys_devs_tramp[i]);
+            for (cur_idx = 0; cur_idx < inst->phys_dev_count_tramp; ++cur_idx) {
+                if (NULL != inst->phys_devs_tramp[cur_idx]) {
+                    loader_instance_heap_free(inst, inst->phys_devs_tramp[cur_idx]);
+                    break;
                 }
             }
             loader_instance_heap_free(inst, inst->phys_devs_tramp);
         }
-
-        // Swap in the new physical device list
-        inst->phys_dev_count_tramp = total_count;
         inst->phys_devs_tramp = new_phys_devs;
+        inst->phys_dev_count_tramp = found_count;
     }
 
     return res;
@@ -6443,6 +6484,50 @@ out:
     return res;
 }
 
+VkResult setup_loader_tramp_phys_dev_groups(struct loader_instance *inst, uint32_t group_count,
+                                            VkPhysicalDeviceGroupProperties *groups) {
+    VkResult res = VK_SUCCESS;
+    uint32_t cur_idx;
+    uint32_t dev_idx;
+
+    if (0 == group_count) {
+        return VK_SUCCESS;
+    }
+
+    // Generate a list of all the devices and convert them to the loader ID
+    uint32_t phys_dev_count = 0;
+    for (cur_idx = 0; cur_idx < group_count; ++cur_idx) {
+        phys_dev_count += groups[cur_idx].physicalDeviceCount;
+    }
+    VkPhysicalDevice *devices = (VkPhysicalDevice *)loader_stack_alloc(sizeof(VkPhysicalDevice) * phys_dev_count);
+    if (NULL == devices) {
+        return VK_ERROR_OUT_OF_HOST_MEMORY;
+    }
+
+    uint32_t cur_device = 0;
+    for (cur_idx = 0; cur_idx < group_count; ++cur_idx) {
+        for (dev_idx = 0; dev_idx < groups[cur_idx].physicalDeviceCount; ++dev_idx) {
+            devices[cur_device++] = groups[cur_idx].physicalDevices[dev_idx];
+        }
+    }
+
+    // Update the devices based on the loader physical device values.
+    res = setup_loader_tramp_phys_devs(inst, phys_dev_count, devices);
+    if (VK_SUCCESS != res) {
+        return res;
+    }
+
+    // Update the devices in the group structures now
+    cur_device = 0;
+    for (cur_idx = 0; cur_idx < group_count; ++cur_idx) {
+        for (dev_idx = 0; dev_idx < groups[cur_idx].physicalDeviceCount; ++dev_idx) {
+            groups[cur_idx].physicalDevices[dev_idx] = devices[cur_device++];
+        }
+    }
+
+    return res;
+}
+
 VKAPI_ATTR VkResult VKAPI_CALL terminator_EnumeratePhysicalDevices(VkInstance instance, uint32_t *pPhysicalDeviceCount,
                                                                    VkPhysicalDevice *pPhysicalDevices) {
     struct loader_instance *inst = (struct loader_instance *)instance;
@@ -6802,7 +6887,10 @@ out:
 
 // ---- Vulkan Core 1.1 terminators
 
-VkResult setup_loader_term_phys_dev_groups(struct loader_instance *inst) {
+VKAPI_ATTR VkResult VKAPI_CALL terminator_EnumeratePhysicalDeviceGroups(
+    VkInstance instance, uint32_t *pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties *pPhysicalDeviceGroupProperties) {
+    struct loader_instance *inst = (struct loader_instance *)instance;
+
     VkResult res = VK_SUCCESS;
     struct loader_icd_term *icd_term;
     uint32_t total_count = 0;
@@ -6815,15 +6903,6 @@ VkResult setup_loader_term_phys_dev_groups(struct loader_instance *inst) {
     uint32_t sorted_count = 0;
     uint32_t icd_idx = 0;
 
-    if (0 == inst->phys_dev_count_term) {
-        loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                   "setup_loader_term_phys_dev_groups:  Loader failed to setup physical device terminator info before calling "
-                   "\'EnumeratePhysicalDeviceGroups\'.");
-        assert(false);
-        res = VK_ERROR_INITIALIZATION_FAILED;
-        goto out;
-    }
-
     // For each ICD, query the number of physical device groups, and then get an
     // internal value for those physical devices.
     icd_term = inst->icd_terms;
@@ -6860,356 +6939,380 @@ VkResult setup_loader_term_phys_dev_groups(struct loader_instance *inst) {
         total_count += cur_icd_group_count;
     }
 
-    if (total_count == 0) {
-        loader_log(inst, VULKAN_LOADER_INFO_BIT, 0,
-                   "setupLoaderTermPhysDevGroups:  Did not detect any GPU Groups"
-                   " in the current config");
-        goto out;
+    // If GPUs not sorted yet, look through them and generate list of all available GPUs
+    if (0 == total_count || 0 == inst->total_gpu_count) {
+        if (VK_SUCCESS != setup_loader_term_phys_devs(inst)) {
+            res = VK_ERROR_INITIALIZATION_FAILED;
+            loader_log(inst, VULKAN_LOADER_INFO_BIT, 0,
+                       "setupLoaderTermPhysDevGroups:  Did not detect any GPU Groups"
+                       " in the current config");
+            goto out;
+        }
     }
 
-    // Create an array for the new physical device groups, which will be stored
-    // in the instance for the Terminator code.
-    new_phys_dev_groups = (VkPhysicalDeviceGroupProperties **)loader_instance_heap_alloc(
-        inst, total_count * sizeof(VkPhysicalDeviceGroupProperties *), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
-    if (NULL == new_phys_dev_groups) {
-        loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                   "setup_loader_term_phys_dev_groups:  Failed to allocate new physical device group array of size %d",
-                   total_count);
-        res = VK_ERROR_OUT_OF_HOST_MEMORY;
-        goto out;
-    }
-    memset(new_phys_dev_groups, 0, total_count * sizeof(VkPhysicalDeviceGroupProperties *));
+    if (NULL != pPhysicalDeviceGroupProperties) {
+        // Create an array for the new physical device groups, which will be stored
+        // in the instance for the Terminator code.
+        new_phys_dev_groups = (VkPhysicalDeviceGroupProperties **)loader_instance_heap_alloc(
+            inst, total_count * sizeof(VkPhysicalDeviceGroupProperties *), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
+        if (NULL == new_phys_dev_groups) {
+            loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
+                       "setup_loader_term_phys_dev_groups:  Failed to allocate new physical device group array of size %d",
+                       total_count);
+            res = VK_ERROR_OUT_OF_HOST_MEMORY;
+            goto out;
+        }
+        memset(new_phys_dev_groups, 0, total_count * sizeof(VkPhysicalDeviceGroupProperties *));
 
-    // Create a temporary array (on the stack) to keep track of the
-    // returned VkPhysicalDevice values.
-    local_phys_dev_groups = loader_stack_alloc(sizeof(struct loader_physical_device_group_term) * total_count);
-    local_phys_dev_group_sorted = loader_stack_alloc(sizeof(bool) * total_count);
-    if (NULL == local_phys_dev_groups || NULL == local_phys_dev_group_sorted) {
-        loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                   "setup_loader_term_phys_dev_groups:  Failed to allocate local physical device group array of size %d",
-                   total_count);
-        res = VK_ERROR_OUT_OF_HOST_MEMORY;
-        goto out;
-    }
-    // Initialize the memory to something valid
-    memset(local_phys_dev_groups, 0, sizeof(struct loader_physical_device_group_term) * total_count);
-    memset(local_phys_dev_group_sorted, 0, sizeof(bool) * total_count);
-    for (uint32_t group = 0; group < total_count; group++) {
-        local_phys_dev_groups[group].group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR;
-        local_phys_dev_groups[group].group_props.pNext = NULL;
-        local_phys_dev_groups[group].group_props.subsetAllocation = false;
-    }
+        // Create a temporary array (on the stack) to keep track of the
+        // returned VkPhysicalDevice values.
+        local_phys_dev_groups = loader_stack_alloc(sizeof(struct loader_physical_device_group_term) * total_count);
+        local_phys_dev_group_sorted = loader_stack_alloc(sizeof(bool) * total_count);
+        if (NULL == local_phys_dev_groups || NULL == local_phys_dev_group_sorted) {
+            loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
+                       "setup_loader_term_phys_dev_groups:  Failed to allocate local physical device group array of size %d",
+                       total_count);
+            res = VK_ERROR_OUT_OF_HOST_MEMORY;
+            goto out;
+        }
+        // Initialize the memory to something valid
+        memset(local_phys_dev_groups, 0, sizeof(struct loader_physical_device_group_term) * total_count);
+        memset(local_phys_dev_group_sorted, 0, sizeof(bool) * total_count);
 
 #if defined(_WIN32)
-    // Get the physical devices supported by platform sorting mechanism into a separate list
-    res = windows_read_sorted_physical_devices(inst, &sorted_phys_dev_array, &sorted_count);
-    if (VK_SUCCESS != res) {
-        goto out;
-    }
+        // Get the physical devices supported by platform sorting mechanism into a separate list
+        res = windows_read_sorted_physical_devices(inst, &sorted_phys_dev_array, &sorted_count);
+        if (VK_SUCCESS != res) {
+            goto out;
+        }
 #endif
 
-    cur_icd_group_count = 0;
-    icd_term = inst->icd_terms;
-    for (icd_idx = 0; NULL != icd_term; icd_term = icd_term->next, icd_idx++) {
-        uint32_t count_this_time = total_count - cur_icd_group_count;
+        cur_icd_group_count = 0;
+        icd_term = inst->icd_terms;
+        for (icd_idx = 0; NULL != icd_term; icd_term = icd_term->next, icd_idx++) {
+            uint32_t count_this_time = total_count - cur_icd_group_count;
 
-        // Check if this group can be sorted
+            // Check if this group can be sorted
 #if defined(VK_USE_PLATFORM_WIN32_KHR)
-        bool icd_sorted = sorted_count && (icd_term->scanned_icd->EnumerateAdapterPhysicalDevices != NULL);
+            bool icd_sorted = sorted_count && (icd_term->scanned_icd->EnumerateAdapterPhysicalDevices != NULL);
 #else
-        bool icd_sorted = false;
+            bool icd_sorted = false;
 #endif
 
-        // Get the function pointer to use to call into the ICD. This could be the core or KHR version
-        if (inst->enabled_known_extensions.khr_device_group_creation) {
-            fpEnumeratePhysicalDeviceGroups = icd_term->dispatch.EnumeratePhysicalDeviceGroupsKHR;
-        } else {
-            fpEnumeratePhysicalDeviceGroups = icd_term->dispatch.EnumeratePhysicalDeviceGroups;
-        }
-
-        if (NULL == fpEnumeratePhysicalDeviceGroups) {
-            VkPhysicalDevice *phys_dev_array = loader_stack_alloc(sizeof(VkPhysicalDevice) * count_this_time);
-            if (NULL == phys_dev_array) {
-                loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                           "setup_loader_term_phys_dev_groups:  Failed to allocate local physical device array of size %d",
-                           count_this_time);
-                res = VK_ERROR_OUT_OF_HOST_MEMORY;
-                goto out;
+            // Get the function pointer to use to call into the ICD. This could be the core or KHR version
+            if (inst->enabled_known_extensions.khr_device_group_creation) {
+                fpEnumeratePhysicalDeviceGroups = icd_term->dispatch.EnumeratePhysicalDeviceGroupsKHR;
+            } else {
+                fpEnumeratePhysicalDeviceGroups = icd_term->dispatch.EnumeratePhysicalDeviceGroups;
             }
 
-            res = icd_term->dispatch.EnumeratePhysicalDevices(icd_term->instance, &count_this_time, phys_dev_array);
-            if (res != VK_SUCCESS) {
-                loader_log(
-                    inst, VULKAN_LOADER_ERROR_BIT, 0,
-                    "setup_loader_term_phys_dev_groups:  Failed during dispatch call of \'EnumeratePhysicalDevices\' to ICD %d "
-                    "to get plain phys dev count.",
-                    icd_idx);
-                goto out;
-            }
+            if (NULL == fpEnumeratePhysicalDeviceGroups) {
+                icd_term->dispatch.EnumeratePhysicalDevices(icd_term->instance, &count_this_time, NULL);
 
-            // Add each GPU as it's own group
-            for (uint32_t indiv_gpu = 0; indiv_gpu < count_this_time; indiv_gpu++) {
-                local_phys_dev_groups[indiv_gpu + cur_icd_group_count].this_icd_term = icd_term;
-                local_phys_dev_groups[indiv_gpu + cur_icd_group_count].icd_index = icd_idx;
-                local_phys_dev_groups[indiv_gpu + cur_icd_group_count].group_props.physicalDeviceCount = 1;
-                local_phys_dev_groups[indiv_gpu + cur_icd_group_count].group_props.physicalDevices[0] = phys_dev_array[indiv_gpu];
-                local_phys_dev_group_sorted[indiv_gpu + cur_icd_group_count] = icd_sorted;
-            }
+                VkPhysicalDevice *phys_dev_array = loader_stack_alloc(sizeof(VkPhysicalDevice) * count_this_time);
+                if (NULL == phys_dev_array) {
+                    loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
+                               "setup_loader_term_phys_dev_groups:  Failed to allocate local physical device array of size %d",
+                               count_this_time);
+                    res = VK_ERROR_OUT_OF_HOST_MEMORY;
+                    goto out;
+                }
 
-        } else {
-            res = fpEnumeratePhysicalDeviceGroups(icd_term->instance, &count_this_time,
-                                                  &local_phys_dev_groups[cur_icd_group_count].group_props);
-            for (uint32_t group = 0; group < count_this_time; ++group) {
-                local_phys_dev_group_sorted[group + cur_icd_group_count] = icd_sorted;
-                local_phys_dev_groups[group + cur_icd_group_count].this_icd_term = icd_term;
-                local_phys_dev_groups[group + cur_icd_group_count].icd_index = icd_idx;
-            }
-            if (VK_SUCCESS != res) {
-                loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                           "setup_loader_term_phys_dev_groups:  Failed during dispatch call of "
-                           "\'EnumeratePhysicalDeviceGroups\' to ICD %d to get content.",
-                           icd_idx);
-                goto out;
+                res = icd_term->dispatch.EnumeratePhysicalDevices(icd_term->instance, &count_this_time, phys_dev_array);
+                if (res != VK_SUCCESS) {
+                    loader_log(
+                        inst, VULKAN_LOADER_ERROR_BIT, 0,
+                        "setup_loader_term_phys_dev_groups:  Failed during dispatch call of \'EnumeratePhysicalDevices\' to ICD %d "
+                        "to get plain phys dev count.",
+                        icd_idx);
+                    goto out;
+                }
+
+                // Add each GPU as it's own group
+                for (uint32_t indiv_gpu = 0; indiv_gpu < count_this_time; indiv_gpu++) {
+                    uint32_t cur_index = indiv_gpu + cur_icd_group_count;
+                    local_phys_dev_groups[cur_index].this_icd_term = icd_term;
+                    local_phys_dev_groups[cur_index].icd_index = icd_idx;
+                    local_phys_dev_groups[cur_index].group_props.physicalDeviceCount = 1;
+                    local_phys_dev_groups[cur_index].group_props.physicalDevices[0] = phys_dev_array[indiv_gpu];
+                    local_phys_dev_group_sorted[cur_index] = icd_sorted;
+                }
+
+            } else {
+                fpEnumeratePhysicalDeviceGroups(icd_term->instance, &count_this_time, NULL);
+                if (cur_icd_group_count + count_this_time < *pPhysicalDeviceGroupCount) {
+                    // Can just use passed in structs
+                    res = fpEnumeratePhysicalDeviceGroups(icd_term->instance, &count_this_time,
+                                                          &pPhysicalDeviceGroupProperties[cur_icd_group_count]);
+                    for (uint32_t group = 0; group < count_this_time; ++group) {
+                        uint32_t cur_index = group + cur_icd_group_count;
+                        local_phys_dev_groups[cur_index].group_props = pPhysicalDeviceGroupProperties[cur_index];
+                        local_phys_dev_group_sorted[cur_index] = icd_sorted;
+                        local_phys_dev_groups[cur_index].this_icd_term = icd_term;
+                        local_phys_dev_groups[cur_index].icd_index = icd_idx;
+                    }
+                } else {
+                    // Have to use a temporary copy
+                    VkPhysicalDeviceGroupProperties *tmp_group_props =
+                        loader_stack_alloc(count_this_time * sizeof(VkPhysicalDeviceGroupProperties));
+                    for (uint32_t group = 0; group < count_this_time; group++) {
+                        tmp_group_props[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR;
+                        uint32_t cur_index = group + cur_icd_group_count;
+                        if (*pPhysicalDeviceGroupCount > cur_index) {
+                            tmp_group_props[group].pNext = pPhysicalDeviceGroupProperties[cur_index].pNext;
+                        } else {
+                            tmp_group_props[group].pNext = NULL;
+                        }
+                        tmp_group_props[group].subsetAllocation = false;
+                    }
+
+                    res = fpEnumeratePhysicalDeviceGroups(icd_term->instance, &count_this_time, tmp_group_props);
+                    for (uint32_t group = 0; group < count_this_time; ++group) {
+                        uint32_t cur_index = group + cur_icd_group_count;
+                        local_phys_dev_groups[cur_index].group_props = tmp_group_props[group];
+                        local_phys_dev_group_sorted[cur_index] = icd_sorted;
+                        local_phys_dev_groups[cur_index].this_icd_term = icd_term;
+                        local_phys_dev_groups[cur_index].icd_index = icd_idx;
+                    }
+                }
+                if (VK_SUCCESS != res) {
+                    loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
+                               "setup_loader_term_phys_dev_groups:  Failed during dispatch call of "
+                               "\'EnumeratePhysicalDeviceGroups\' to ICD %d to get content.",
+                               icd_idx);
+                    goto out;
+                }
             }
-        }
 
-        cur_icd_group_count += count_this_time;
-    }
+            cur_icd_group_count += count_this_time;
+        }
 
 #ifdef LOADER_ENABLE_LINUX_SORT
-    if (is_linux_sort_enabled(inst)) {
-        // Get the physical devices supported by platform sorting mechanism into a separate list
-        res = linux_read_sorted_physical_device_groups(inst, total_count, local_phys_dev_groups);
-    }
+        if (is_linux_sort_enabled(inst)) {
+            // Get the physical devices supported by platform sorting mechanism into a separate list
+            res = linux_read_sorted_physical_device_groups(inst, total_count, local_phys_dev_groups);
+        }
 #endif  // LOADER_ENABLE_LINUX_SORT
 
-    // Replace all the physical device IDs with the proper loader values
-    for (uint32_t group = 0; group < total_count; group++) {
-        for (uint32_t group_gpu = 0; group_gpu < local_phys_dev_groups[group].group_props.physicalDeviceCount; group_gpu++) {
-            bool found = false;
-            for (uint32_t term_gpu = 0; term_gpu < inst->phys_dev_count_term; term_gpu++) {
-                if (local_phys_dev_groups[group].group_props.physicalDevices[group_gpu] ==
-                    inst->phys_devs_term[term_gpu]->phys_dev) {
-                    local_phys_dev_groups[group].group_props.physicalDevices[group_gpu] =
-                        (VkPhysicalDevice)inst->phys_devs_term[term_gpu];
-                    found = true;
-                    break;
+        // Replace all the physical device IDs with the proper loader values
+        if (NULL != inst->phys_devs_term) {
+            for (uint32_t group = 0; group < total_count; group++) {
+                for (uint32_t group_gpu = 0; group_gpu < local_phys_dev_groups[group].group_props.physicalDeviceCount;
+                     group_gpu++) {
+                    bool found = false;
+                    for (uint32_t term_gpu = 0; term_gpu < inst->phys_dev_count_term; term_gpu++) {
+                        if (local_phys_dev_groups[group].group_props.physicalDevices[group_gpu] ==
+                            inst->phys_devs_term[term_gpu]->phys_dev) {
+                            local_phys_dev_groups[group].group_props.physicalDevices[group_gpu] =
+                                (VkPhysicalDevice)inst->phys_devs_term[term_gpu];
+                            found = true;
+                            break;
+                        }
+                    }
+                    if (!found) {
+                        loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
+                                   "setup_loader_term_phys_dev_groups:  Failed to find GPU %d in group %d returned by "
+                                   "\'EnumeratePhysicalDeviceGroups\' in list returned by \'EnumeratePhysicalDevices\'",
+                                   group_gpu, group);
+                        res = VK_ERROR_INITIALIZATION_FAILED;
+                        goto out;
+                    }
                 }
             }
-            if (!found) {
-                loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                           "setup_loader_term_phys_dev_groups:  Failed to find GPU %d in group %d returned by "
-                           "\'EnumeratePhysicalDeviceGroups\' in list returned by \'EnumeratePhysicalDevices\'",
-                           group_gpu, group);
-                res = VK_ERROR_INITIALIZATION_FAILED;
-                goto out;
-            }
         }
-    }
 
-    uint32_t idx = 0;
+        uint32_t idx = 0;
 
 #if defined(_WIN32)
-    // Copy over everything found through sorted enumeration
-    for (uint32_t i = 0; i < sorted_count; ++i) {
-        // Find the VkPhysicalDeviceGroupProperties object in local_phys_dev_groups
-        VkPhysicalDeviceGroupProperties *group_properties = NULL;
-        for (uint32_t group = 0; group < total_count; group++) {
-            if (sorted_phys_dev_array[i].device_count != local_phys_dev_groups[group].group_props.physicalDeviceCount) {
-                continue;
-            }
+        // Copy over everything found through sorted enumeration
+        for (uint32_t i = 0; i < sorted_count; ++i) {
+            // Find the VkPhysicalDeviceGroupProperties object in local_phys_dev_groups
+            VkPhysicalDeviceGroupProperties *group_properties = NULL;
+            for (uint32_t group = 0; group < total_count; group++) {
+                if (sorted_phys_dev_array[i].device_count != local_phys_dev_groups[group].group_props.physicalDeviceCount) {
+                    continue;
+                }
 
-            bool match = true;
-            for (uint32_t group_gpu = 0; group_gpu < local_phys_dev_groups[group].group_props.physicalDeviceCount; group_gpu++) {
-                if (sorted_phys_dev_array[i].physical_devices[group_gpu] !=
-                    ((struct loader_physical_device_term *)local_phys_dev_groups[group].group_props.physicalDevices[group_gpu])
-                        ->phys_dev) {
-                    match = false;
-                    break;
+                bool match = true;
+                for (uint32_t group_gpu = 0; group_gpu < local_phys_dev_groups[group].group_props.physicalDeviceCount;
+                     group_gpu++) {
+                    if (sorted_phys_dev_array[i].physical_devices[group_gpu] !=
+                        ((struct loader_physical_device_term *)local_phys_dev_groups[group].group_props.physicalDevices[group_gpu])
+                            ->phys_dev) {
+                        match = false;
+                        break;
+                    }
                 }
-            }
 
-            if (match) {
-                group_properties = &local_phys_dev_groups[group].group_props;
+                if (match) {
+                    group_properties = &local_phys_dev_groups[group].group_props;
+                }
             }
-        }
 
-        // Check if this physical device group with the same contents is already in the old buffer
-        for (uint32_t old_idx = 0; old_idx < inst->phys_dev_group_count_term; old_idx++) {
-            if (NULL != group_properties &&
-                group_properties->physicalDeviceCount == inst->phys_dev_groups_term[old_idx]->physicalDeviceCount) {
-                bool found_all_gpus = true;
-                for (uint32_t old_gpu = 0; old_gpu < inst->phys_dev_groups_term[old_idx]->physicalDeviceCount; old_gpu++) {
-                    bool found_gpu = false;
-                    for (uint32_t new_gpu = 0; new_gpu < group_properties->physicalDeviceCount; new_gpu++) {
-                        if (group_properties->physicalDevices[new_gpu] ==
-                            inst->phys_dev_groups_term[old_idx]->physicalDevices[old_gpu]) {
-                            found_gpu = true;
+            // Check if this physical device group with the same contents is already in the old buffer
+            for (uint32_t old_idx = 0; old_idx < inst->phys_dev_group_count_term; old_idx++) {
+                if (NULL != group_properties &&
+                    group_properties->physicalDeviceCount == inst->phys_dev_groups_term[old_idx]->physicalDeviceCount) {
+                    bool found_all_gpus = true;
+                    for (uint32_t old_gpu = 0; old_gpu < inst->phys_dev_groups_term[old_idx]->physicalDeviceCount; old_gpu++) {
+                        bool found_gpu = false;
+                        for (uint32_t new_gpu = 0; new_gpu < group_properties->physicalDeviceCount; new_gpu++) {
+                            if (group_properties->physicalDevices[new_gpu] ==
+                                inst->phys_dev_groups_term[old_idx]->physicalDevices[old_gpu]) {
+                                found_gpu = true;
+                                break;
+                            }
+                        }
+
+                        if (!found_gpu) {
+                            found_all_gpus = false;
                             break;
                         }
                     }
-
-                    if (!found_gpu) {
-                        found_all_gpus = false;
+                    if (!found_all_gpus) {
+                        continue;
+                    } else {
+                        new_phys_dev_groups[idx] = inst->phys_dev_groups_term[old_idx];
                         break;
                     }
                 }
-                if (!found_all_gpus) {
-                    continue;
-                } else {
-                    new_phys_dev_groups[idx] = inst->phys_dev_groups_term[old_idx];
-                    break;
-                }
             }
-        }
 
-        // If this physical device group isn't in the old buffer, create it
-        if (group_properties != NULL && NULL == new_phys_dev_groups[idx]) {
-            new_phys_dev_groups[idx] = (VkPhysicalDeviceGroupPropertiesKHR *)loader_instance_heap_alloc(
-                inst, sizeof(VkPhysicalDeviceGroupPropertiesKHR), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
-            if (NULL == new_phys_dev_groups[idx]) {
-                loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                           "setup_loader_term_phys_dev_groups:  Failed to allocate physical device group Terminator object %d",
-                           idx);
-                total_count = idx;
-                res = VK_ERROR_OUT_OF_HOST_MEMORY;
-                goto out;
+            // If this physical device group isn't in the old buffer, create it
+            if (group_properties != NULL && NULL == new_phys_dev_groups[idx]) {
+                new_phys_dev_groups[idx] = (VkPhysicalDeviceGroupPropertiesKHR *)loader_instance_heap_alloc(
+                    inst, sizeof(VkPhysicalDeviceGroupPropertiesKHR), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
+                if (NULL == new_phys_dev_groups[idx]) {
+                    loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
+                               "setup_loader_term_phys_dev_groups:  Failed to allocate physical device group Terminator object %d",
+                               idx);
+                    total_count = idx;
+                    res = VK_ERROR_OUT_OF_HOST_MEMORY;
+                    goto out;
+                }
+                memcpy(new_phys_dev_groups[idx], group_properties, sizeof(VkPhysicalDeviceGroupPropertiesKHR));
             }
-            memcpy(new_phys_dev_groups[idx], group_properties, sizeof(VkPhysicalDeviceGroupPropertiesKHR));
-        }
 
-        ++idx;
-    }
+            ++idx;
+        }
 #endif
 
-    // Copy or create everything to fill the new array of physical device groups
-    for (uint32_t new_idx = 0; new_idx < total_count; new_idx++) {
-        // Skip groups which have been included through sorting
-        if (local_phys_dev_group_sorted[new_idx] || local_phys_dev_groups[new_idx].group_props.physicalDeviceCount == 0) {
-            continue;
-        }
+        // Copy or create everything to fill the new array of physical device groups
+        for (uint32_t new_idx = 0; new_idx < total_count; new_idx++) {
+            // Skip groups which have been included through sorting
+            if (local_phys_dev_group_sorted[new_idx] || local_phys_dev_groups[new_idx].group_props.physicalDeviceCount == 0) {
+                continue;
+            }
+
+            // Check if this physical device group with the same contents is already in the old buffer
+            for (uint32_t old_idx = 0; old_idx < inst->phys_dev_group_count_term; old_idx++) {
+                if (local_phys_dev_groups[new_idx].group_props.physicalDeviceCount ==
+                    inst->phys_dev_groups_term[old_idx]->physicalDeviceCount) {
+                    bool found_all_gpus = true;
+                    for (uint32_t old_gpu = 0; old_gpu < inst->phys_dev_groups_term[old_idx]->physicalDeviceCount; old_gpu++) {
+                        bool found_gpu = false;
+                        for (uint32_t new_gpu = 0; new_gpu < local_phys_dev_groups[new_idx].group_props.physicalDeviceCount;
+                             new_gpu++) {
+                            if (local_phys_dev_groups[new_idx].group_props.physicalDevices[new_gpu] ==
+                                inst->phys_dev_groups_term[old_idx]->physicalDevices[old_gpu]) {
+                                found_gpu = true;
+                                break;
+                            }
+                        }
 
-        // Check if this physical device group with the same contents is already in the old buffer
-        for (uint32_t old_idx = 0; old_idx < inst->phys_dev_group_count_term; old_idx++) {
-            if (local_phys_dev_groups[new_idx].group_props.physicalDeviceCount ==
-                inst->phys_dev_groups_term[old_idx]->physicalDeviceCount) {
-                bool found_all_gpus = true;
-                for (uint32_t old_gpu = 0; old_gpu < inst->phys_dev_groups_term[old_idx]->physicalDeviceCount; old_gpu++) {
-                    bool found_gpu = false;
-                    for (uint32_t new_gpu = 0; new_gpu < local_phys_dev_groups[new_idx].group_props.physicalDeviceCount;
-                         new_gpu++) {
-                        if (local_phys_dev_groups[new_idx].group_props.physicalDevices[new_gpu] ==
-                            inst->phys_dev_groups_term[old_idx]->physicalDevices[old_gpu]) {
-                            found_gpu = true;
+                        if (!found_gpu) {
+                            found_all_gpus = false;
                             break;
                         }
                     }
-
-                    if (!found_gpu) {
-                        found_all_gpus = false;
+                    if (!found_all_gpus) {
+                        continue;
+                    } else {
+                        new_phys_dev_groups[idx] = inst->phys_dev_groups_term[old_idx];
                         break;
                     }
                 }
-                if (!found_all_gpus) {
-                    continue;
-                } else {
-                    new_phys_dev_groups[idx] = inst->phys_dev_groups_term[old_idx];
-                    break;
-                }
             }
-        }
 
-        // If this physical device group isn't in the old buffer, create it
-        if (NULL == new_phys_dev_groups[idx]) {
-            new_phys_dev_groups[idx] = (VkPhysicalDeviceGroupPropertiesKHR *)loader_instance_heap_alloc(
-                inst, sizeof(VkPhysicalDeviceGroupPropertiesKHR), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
+            // If this physical device group isn't in the old buffer, create it
             if (NULL == new_phys_dev_groups[idx]) {
-                loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                           "setup_loader_term_phys_dev_groups:  Failed to allocate physical device group Terminator object %d",
-                           idx);
-                total_count = idx;
-                res = VK_ERROR_OUT_OF_HOST_MEMORY;
-                goto out;
+                new_phys_dev_groups[idx] = (VkPhysicalDeviceGroupPropertiesKHR *)loader_instance_heap_alloc(
+                    inst, sizeof(VkPhysicalDeviceGroupPropertiesKHR), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
+                if (NULL == new_phys_dev_groups[idx]) {
+                    loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
+                               "setup_loader_term_phys_dev_groups:  Failed to allocate physical device group Terminator object %d",
+                               idx);
+                    total_count = idx;
+                    res = VK_ERROR_OUT_OF_HOST_MEMORY;
+                    goto out;
+                }
+                memcpy(new_phys_dev_groups[idx], &local_phys_dev_groups[new_idx].group_props,
+                       sizeof(VkPhysicalDeviceGroupPropertiesKHR));
             }
-            memcpy(new_phys_dev_groups[idx], &local_phys_dev_groups[new_idx].group_props,
-                   sizeof(VkPhysicalDeviceGroupPropertiesKHR));
-        }
 
-        ++idx;
+            ++idx;
+        }
     }
 
 out:
 
-    if (VK_SUCCESS != res) {
-        if (NULL != new_phys_dev_groups) {
-            for (uint32_t i = 0; i < total_count; i++) {
-                loader_instance_heap_free(inst, new_phys_dev_groups[i]);
+    if (NULL != pPhysicalDeviceGroupProperties) {
+        if (VK_SUCCESS != res) {
+            if (NULL != new_phys_dev_groups) {
+                for (uint32_t i = 0; i < total_count; i++) {
+                    loader_instance_heap_free(inst, new_phys_dev_groups[i]);
+                }
+                loader_instance_heap_free(inst, new_phys_dev_groups);
             }
-            loader_instance_heap_free(inst, new_phys_dev_groups);
-        }
-    } else {
-        // Free everything that didn't carry over to the new array of
-        // physical device groups
-        if (NULL != inst->phys_dev_groups_term) {
-            for (uint32_t i = 0; i < inst->phys_dev_group_count_term; i++) {
-                bool found = false;
-                for (uint32_t j = 0; j < total_count; j++) {
-                    if (inst->phys_dev_groups_term[i] == new_phys_dev_groups[j]) {
-                        found = true;
-                        break;
+        } else {
+            // Free everything that didn't carry over to the new array of
+            // physical device groups
+            if (NULL != inst->phys_dev_groups_term) {
+                for (uint32_t i = 0; i < inst->phys_dev_group_count_term; i++) {
+                    bool found = false;
+                    for (uint32_t j = 0; j < total_count; j++) {
+                        if (inst->phys_dev_groups_term[i] == new_phys_dev_groups[j]) {
+                            found = true;
+                            break;
+                        }
+                    }
+                    if (!found) {
+                        loader_instance_heap_free(inst, inst->phys_dev_groups_term[i]);
                     }
                 }
-                if (!found) {
-                    loader_instance_heap_free(inst, inst->phys_dev_groups_term[i]);
-                }
+                loader_instance_heap_free(inst, inst->phys_dev_groups_term);
             }
-            loader_instance_heap_free(inst, inst->phys_dev_groups_term);
-        }
 
-        // Swap in the new physical device group list
-        inst->phys_dev_group_count_term = total_count;
-        inst->phys_dev_groups_term = new_phys_dev_groups;
-    }
+            // Swap in the new physical device group list
+            inst->phys_dev_group_count_term = total_count;
+            inst->phys_dev_groups_term = new_phys_dev_groups;
+        }
 
-    if (sorted_phys_dev_array != NULL) {
-        for (uint32_t i = 0; i < sorted_count; ++i) {
-            if (sorted_phys_dev_array[i].device_count > 0 && sorted_phys_dev_array[i].physical_devices != NULL) {
-                loader_instance_heap_free(inst, sorted_phys_dev_array[i].physical_devices);
+        if (sorted_phys_dev_array != NULL) {
+            for (uint32_t i = 0; i < sorted_count; ++i) {
+                if (sorted_phys_dev_array[i].device_count > 0 && sorted_phys_dev_array[i].physical_devices != NULL) {
+                    loader_instance_heap_free(inst, sorted_phys_dev_array[i].physical_devices);
+                }
             }
+            loader_instance_heap_free(inst, sorted_phys_dev_array);
         }
-        loader_instance_heap_free(inst, sorted_phys_dev_array);
-    }
-
-    return res;
-}
-
-VKAPI_ATTR VkResult VKAPI_CALL terminator_EnumeratePhysicalDeviceGroups(
-    VkInstance instance, uint32_t *pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties *pPhysicalDeviceGroupProperties) {
-    struct loader_instance *inst = (struct loader_instance *)instance;
-    VkResult res = VK_SUCCESS;
-
-    // Always call the setup loader terminator physical device groups because they may
-    // have changed at any point.
-    res = setup_loader_term_phys_dev_groups(inst);
-    if (VK_SUCCESS != res) {
-        goto out;
-    }
 
-    uint32_t copy_count = inst->phys_dev_group_count_term;
-    if (NULL != pPhysicalDeviceGroupProperties) {
-        if (copy_count > *pPhysicalDeviceGroupCount) {
-            copy_count = *pPhysicalDeviceGroupCount;
-            res = VK_INCOMPLETE;
-        }
+        uint32_t copy_count = inst->phys_dev_group_count_term;
+        if (NULL != pPhysicalDeviceGroupProperties) {
+            if (copy_count > *pPhysicalDeviceGroupCount) {
+                copy_count = *pPhysicalDeviceGroupCount;
+                res = VK_INCOMPLETE;
+            }
 
-        for (uint32_t i = 0; i < copy_count; i++) {
-            memcpy(&pPhysicalDeviceGroupProperties[i], inst->phys_dev_groups_term[i], sizeof(VkPhysicalDeviceGroupPropertiesKHR));
+            for (uint32_t i = 0; i < copy_count; i++) {
+                memcpy(&pPhysicalDeviceGroupProperties[i], inst->phys_dev_groups_term[i], sizeof(VkPhysicalDeviceGroupProperties));
+            }
         }
-    }
 
-    *pPhysicalDeviceGroupCount = copy_count;
-
-out:
+        *pPhysicalDeviceGroupCount = copy_count;
 
+    } else {
+        *pPhysicalDeviceGroupCount = total_count;
+    }
     return res;
 }
index 5700d15730372097739881d72ec56b90a3c1a5fb..4ef699664fe6604bcc1fb9e9f455ce2072bcd256 100644 (file)
@@ -1,8 +1,8 @@
 /*
  *
- * Copyright (c) 2014-2021 The Khronos Group Inc.
- * Copyright (c) 2014-2021 Valve Corporation
- * Copyright (c) 2014-2021 LunarG, Inc.
+ * Copyright (c) 2014-2022 The Khronos Group Inc.
+ * Copyright (c) 2014-2022 Valve Corporation
+ * Copyright (c) 2014-2022 LunarG, Inc.
  * Copyright (C) 2015 Google Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -164,8 +164,8 @@ VkResult loader_validate_device_extensions(struct loader_instance *this_instance
                                            const struct loader_layer_list *activated_device_layers,
                                            const struct loader_extension_list *icd_exts, const VkDeviceCreateInfo *pCreateInfo);
 
-VkResult setup_loader_tramp_phys_devs(struct loader_instance *inst);
-VkResult setup_loader_term_phys_devs(struct loader_instance *inst);
+VkResult setup_loader_tramp_phys_devs(struct loader_instance *inst, uint32_t phys_dev_count, VkPhysicalDevice *phys_devs);
+VkResult setup_loader_tramp_phys_dev_groups(struct loader_instance *inst, uint32_t group_count, VkPhysicalDeviceGroupProperties *groups);
 
 VkStringErrorFlags vk_string_validate(const int max_length, const char *char_array);
 char *loader_get_next_path(char *path);
index 5f9234341db83e4ca644bb7d51ef4c83a95fc5b3..de9aa1a9e703cd16e9a9f6781edb7cefe46eeaf8 100644 (file)
@@ -271,8 +271,6 @@ struct loader_instance {
     // device stored internal to the public structures.
     uint32_t phys_dev_group_count_term;
     struct VkPhysicalDeviceGroupProperties **phys_dev_groups_term;
-    uint32_t phys_dev_group_count_tramp;
-    struct VkPhysicalDeviceGroupProperties **phys_dev_groups_tramp;
 
     struct loader_instance *next;
 
index ff82c9c39eb94f01c2a77b05ec2d69b1036edbea..6f914b07f2d6ff19bbf8c970de481cdde4e41159 100644 (file)
@@ -219,7 +219,8 @@ static void linux_env_var_default_device(struct loader_instance *inst, uint32_t
             for (int32_t i = 0; i < (int32_t)device_count; ++i) {
                 if (sorted_device_info[i].vendor_id == vendor_id && sorted_device_info[i].device_id == device_id) {
                     loader_log(inst, VULKAN_LOADER_INFO_BIT | VULKAN_LOADER_DRIVER_BIT, 0,
-                            "linux_env_var_default_device:  Found default at index %u \'%s\'", i, sorted_device_info[i].device_name);
+                               "linux_env_var_default_device:  Found default at index %u \'%s\'", i,
+                               sorted_device_info[i].device_name);
                     sorted_device_info[i].default_device = true;
                     break;
                 }
@@ -424,6 +425,11 @@ VkResult linux_read_sorted_physical_device_groups(struct loader_instance *inst,
         // Sort GPUs in each group
         qsort(sorted_group_term[group].internal_device_info, sorted_group_term[group].group_props.physicalDeviceCount,
               sizeof(struct LinuxSortedDeviceInfo), compare_devices);
+
+        // Match the externally used physical device list with the sorted physical device list for this group.
+        for (uint32_t dev = 0; dev < sorted_group_term[group].group_props.physicalDeviceCount; ++dev) {
+            sorted_group_term[group].group_props.physicalDevices[dev] = sorted_group_term[group].internal_device_info[dev].physical_device;
+        }
     }
 
     // Sort device groups by PCI info
@@ -435,8 +441,9 @@ VkResult linux_read_sorted_physical_device_groups(struct loader_instance *inst,
         for (uint32_t group = 0; group < group_count; ++group) {
             loader_log(inst, VULKAN_LOADER_INFO_BIT | VULKAN_LOADER_DRIVER_BIT, 0, "           Group %u", group);
             for (uint32_t gpu = 0; gpu < sorted_group_term[group].group_props.physicalDeviceCount; ++gpu) {
-                loader_log(inst, VULKAN_LOADER_INFO_BIT | VULKAN_LOADER_DRIVER_BIT, 0, "               [%u] %s %s", gpu,
+                loader_log(inst, VULKAN_LOADER_INFO_BIT | VULKAN_LOADER_DRIVER_BIT, 0, "               [%u] %s %p %s", gpu,
                            sorted_group_term[group].internal_device_info[gpu].device_name,
+                           sorted_group_term[group].internal_device_info[gpu].physical_device,
                            (sorted_group_term[group].internal_device_info[gpu].default_device ? "[default]" : ""));
             }
         }
index be06f57eca7ece1d3b1ba9fcd6ad592285567eb1..69723b535eb09ba4a56c3202195cfd88b2bda5f8 100644 (file)
@@ -724,13 +724,6 @@ LOADER_EXPORT VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(VkInstance instance,
         loader_instance_heap_free(ptr_instance, ptr_instance->phys_devs_tramp);
     }
 
-    if (ptr_instance->phys_dev_groups_tramp) {
-        for (uint32_t i = 0; i < ptr_instance->phys_dev_group_count_tramp; i++) {
-            loader_instance_heap_free(ptr_instance, ptr_instance->phys_dev_groups_tramp[i]);
-        }
-        loader_instance_heap_free(ptr_instance, ptr_instance->phys_dev_groups_tramp);
-    }
-
     if (messenger_setup) {
         loader_log(ptr_instance, VULKAN_LOADER_INFO_BIT, 0,
                    "vkDestroyInstance: destroying temporary instance debug util messenger");
@@ -760,8 +753,6 @@ LOADER_EXPORT VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(VkInstance instance,
 LOADER_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDevices(VkInstance instance, uint32_t *pPhysicalDeviceCount,
                                                                         VkPhysicalDevice *pPhysicalDevices) {
     VkResult res = VK_SUCCESS;
-    uint32_t count;
-    uint32_t i;
     struct loader_instance *inst;
 
     loader_platform_thread_lock_mutex(&loader_lock);
@@ -784,38 +775,16 @@ LOADER_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDevices(VkInstan
     // Setup the trampoline loader physical devices.  This will actually
     // call down and setup the terminator loader physical devices during the
     // process.
-    VkResult setup_res = setup_loader_tramp_phys_devs(inst);
-    if (setup_res != VK_SUCCESS && setup_res != VK_INCOMPLETE) {
-        res = setup_res;
-        goto out;
-    }
-
-    count = inst->phys_dev_count_tramp;
-
-    if (inst->phys_dev_count_tramp != inst->total_gpu_count) {
-        loader_log(inst, VULKAN_LOADER_WARN_BIT | VULKAN_LOADER_LAYER_BIT, 0,
-                   "vkEnumeratePhysicalDevices: One or more layers modified physical devices!"
-                   "Count returned by ICDs = %d, count returned above layers = %d",
-                   inst->total_gpu_count, inst->phys_dev_count_tramp);
-    }
+    res = inst->disp->layer_inst_disp.EnumeratePhysicalDevices(inst->instance, pPhysicalDeviceCount, pPhysicalDevices);
 
     // Wrap the PhysDev object for loader usage, return wrapped objects
-    if (NULL != pPhysicalDevices) {
-        if (inst->phys_dev_count_tramp > *pPhysicalDeviceCount) {
-            loader_log(inst, VULKAN_LOADER_INFO_BIT, 0,
-                       "vkEnumeratePhysicalDevices: Trimming device count down"
-                       " by application request from %d to %d physical devices",
-                       inst->phys_dev_count_tramp, *pPhysicalDeviceCount);
-            count = *pPhysicalDeviceCount;
-            res = VK_INCOMPLETE;
-        }
-        for (i = 0; i < count; i++) {
-            pPhysicalDevices[i] = (VkPhysicalDevice)inst->phys_devs_tramp[i];
+    if (NULL != pPhysicalDevices && (VK_SUCCESS == res || VK_INCOMPLETE == res)) {
+        VkResult update_res = setup_loader_tramp_phys_devs(inst, *pPhysicalDeviceCount, pPhysicalDevices);
+        if (VK_SUCCESS != update_res) {
+            res = update_res;
         }
     }
 
-    *pPhysicalDeviceCount = count;
-
 out:
 
     loader_platform_thread_unlock_mutex(&loader_lock);
@@ -2516,193 +2485,9 @@ LOADER_EXPORT VKAPI_ATTR void VKAPI_CALL vkCmdExecuteCommands(VkCommandBuffer co
 
 // ---- Vulkan core 1.1 trampolines
 
-VkResult setupLoaderTrampPhysDevGroups(VkInstance instance, struct loader_instance *inst) {
-    VkResult res = VK_SUCCESS;
-    uint32_t total_count = 0;
-    VkPhysicalDeviceGroupPropertiesKHR **new_phys_dev_groups = NULL;
-    VkPhysicalDeviceGroupPropertiesKHR *local_phys_dev_groups = NULL;
-    PFN_vkEnumeratePhysicalDeviceGroups fpEnumeratePhysicalDeviceGroups = NULL;
-
-    // Get the function pointer to use to call into the ICD. This could be the core or KHR version
-    if (inst->enabled_known_extensions.khr_device_group_creation) {
-        fpEnumeratePhysicalDeviceGroups = inst->disp->layer_inst_disp.EnumeratePhysicalDeviceGroupsKHR;
-    } else {
-        fpEnumeratePhysicalDeviceGroups = inst->disp->layer_inst_disp.EnumeratePhysicalDeviceGroups;
-    }
-
-    // Setup the trampoline loader physical devices.  This will actually
-    // call down and setup the terminator loader physical devices during the
-    // process.
-    VkResult setup_res = setup_loader_tramp_phys_devs(inst);
-    if (setup_res != VK_SUCCESS && setup_res != VK_INCOMPLETE) {
-        res = setup_res;
-        goto out;
-    }
-
-    // Query how many physical device groups there
-    res = fpEnumeratePhysicalDeviceGroups(inst->instance, &total_count, NULL);
-    if (res != VK_SUCCESS) {
-        loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                   "setupLoaderTrampPhysDevGroups:  Failed during dispatch call of "
-                   "\'EnumeratePhysicalDeviceGroupsKHR\' to lower layers or "
-                   "loader to get count.");
-        goto out;
-    }
-
-    // Create an array for the new physical device groups, which will be stored
-    // in the instance for the trampoline code.
-    new_phys_dev_groups = (VkPhysicalDeviceGroupPropertiesKHR **)loader_instance_heap_alloc(
-        inst, total_count * sizeof(VkPhysicalDeviceGroupPropertiesKHR *), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
-    if (NULL == new_phys_dev_groups) {
-        loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                   "setupLoaderTrampPhysDevGroups:  Failed to allocate new physical device"
-                   " group array of size %d",
-                   total_count);
-        res = VK_ERROR_OUT_OF_HOST_MEMORY;
-        goto out;
-    }
-    memset(new_phys_dev_groups, 0, total_count * sizeof(VkPhysicalDeviceGroupPropertiesKHR *));
-
-    // Create a temporary array (on the stack) to keep track of the
-    // returned VkPhysicalDevice values.
-    local_phys_dev_groups = loader_stack_alloc(sizeof(VkPhysicalDeviceGroupPropertiesKHR) * total_count);
-    if (NULL == local_phys_dev_groups) {
-        loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                   "setupLoaderTrampPhysDevGroups:  Failed to allocate local "
-                   "physical device group array of size %d",
-                   total_count);
-        res = VK_ERROR_OUT_OF_HOST_MEMORY;
-        goto out;
-    }
-    // Initialize the memory to something valid
-    memset(local_phys_dev_groups, 0, sizeof(VkPhysicalDeviceGroupPropertiesKHR) * total_count);
-    for (uint32_t group = 0; group < total_count; group++) {
-        local_phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR;
-        local_phys_dev_groups[group].pNext = NULL;
-        local_phys_dev_groups[group].subsetAllocation = false;
-    }
-
-    // Call down and get the content
-    fpEnumeratePhysicalDeviceGroups(inst->instance, &total_count, local_phys_dev_groups);
-    if (VK_SUCCESS != res) {
-        loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                   "setupLoaderTrampPhysDevGroups:  Failed during dispatch call of "
-                   "\'EnumeratePhysicalDeviceGroupsKHR\' to lower layers or "
-                   "loader to get content.");
-        goto out;
-    }
-
-    // Replace all the physical device IDs with the proper loader values
-    for (uint32_t group = 0; group < total_count; group++) {
-        for (uint32_t group_gpu = 0; group_gpu < local_phys_dev_groups[group].physicalDeviceCount; group_gpu++) {
-            bool found = false;
-            for (uint32_t tramp_gpu = 0; tramp_gpu < inst->phys_dev_count_tramp; tramp_gpu++) {
-                if (local_phys_dev_groups[group].physicalDevices[group_gpu] == inst->phys_devs_tramp[tramp_gpu]->phys_dev) {
-                    local_phys_dev_groups[group].physicalDevices[group_gpu] = (VkPhysicalDevice)inst->phys_devs_tramp[tramp_gpu];
-                    found = true;
-                    break;
-                }
-            }
-            if (!found) {
-                loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                           "setupLoaderTrampPhysDevGroups:  Failed to find GPU %d in group %d"
-                           " returned by \'EnumeratePhysicalDeviceGroupsKHR\' in list returned"
-                           " by \'EnumeratePhysicalDevices\'",
-                           group_gpu, group);
-                res = VK_ERROR_INITIALIZATION_FAILED;
-                goto out;
-            }
-        }
-    }
-
-    // Copy or create everything to fill the new array of physical device groups
-    for (uint32_t new_idx = 0; new_idx < total_count; new_idx++) {
-        // Check if this physical device group with the same contents is already in the old buffer
-        for (uint32_t old_idx = 0; old_idx < inst->phys_dev_group_count_tramp; old_idx++) {
-            if (local_phys_dev_groups[new_idx].physicalDeviceCount == inst->phys_dev_groups_tramp[old_idx]->physicalDeviceCount) {
-                bool found_all_gpus = true;
-                for (uint32_t old_gpu = 0; old_gpu < inst->phys_dev_groups_tramp[old_idx]->physicalDeviceCount; old_gpu++) {
-                    bool found_gpu = false;
-                    for (uint32_t new_gpu = 0; new_gpu < local_phys_dev_groups[new_idx].physicalDeviceCount; new_gpu++) {
-                        if (local_phys_dev_groups[new_idx].physicalDevices[new_gpu] ==
-                            inst->phys_dev_groups_tramp[old_idx]->physicalDevices[old_gpu]) {
-                            found_gpu = true;
-                            break;
-                        }
-                    }
-
-                    if (!found_gpu) {
-                        found_all_gpus = false;
-                        break;
-                    }
-                }
-                if (!found_all_gpus) {
-                    continue;
-                } else {
-                    new_phys_dev_groups[new_idx] = inst->phys_dev_groups_tramp[old_idx];
-                    break;
-                }
-            }
-        }
-
-        // If this physical device group isn't in the old buffer, create it
-        if (NULL == new_phys_dev_groups[new_idx]) {
-            new_phys_dev_groups[new_idx] = (VkPhysicalDeviceGroupPropertiesKHR *)loader_instance_heap_alloc(
-                inst, sizeof(VkPhysicalDeviceGroupPropertiesKHR), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
-            if (NULL == new_phys_dev_groups[new_idx]) {
-                loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0,
-                           "setupLoaderTrampPhysDevGroups:  Failed to allocate "
-                           "physical device group trampoline object %d",
-                           new_idx);
-                total_count = new_idx;
-                res = VK_ERROR_OUT_OF_HOST_MEMORY;
-                goto out;
-            }
-            memcpy(new_phys_dev_groups[new_idx], &local_phys_dev_groups[new_idx], sizeof(VkPhysicalDeviceGroupPropertiesKHR));
-        }
-    }
-
-out:
-    if (VK_SUCCESS != res) {
-        if (NULL != new_phys_dev_groups) {
-            for (uint32_t i = 0; i < total_count; i++) {
-                loader_instance_heap_free(inst, new_phys_dev_groups[i]);
-            }
-            loader_instance_heap_free(inst, new_phys_dev_groups);
-        }
-        total_count = 0;
-    } else {
-        // Free everything that didn't carry over to the new array of
-        // physical device groups
-        if (NULL != inst->phys_dev_groups_tramp) {
-            for (uint32_t i = 0; i < inst->phys_dev_group_count_tramp; i++) {
-                bool found = false;
-                for (uint32_t j = 0; j < total_count; j++) {
-                    if (inst->phys_dev_groups_tramp[i] == new_phys_dev_groups[j]) {
-                        found = true;
-                        break;
-                    }
-                }
-                if (!found) {
-                    loader_instance_heap_free(inst, inst->phys_dev_groups_tramp[i]);
-                }
-            }
-            loader_instance_heap_free(inst, inst->phys_dev_groups_tramp);
-        }
-
-        // Swap in the new physical device group list
-        inst->phys_dev_group_count_tramp = total_count;
-        inst->phys_dev_groups_tramp = new_phys_dev_groups;
-    }
-
-    return res;
-}
-
 LOADER_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDeviceGroups(
     VkInstance instance, uint32_t *pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties *pPhysicalDeviceGroupProperties) {
     VkResult res = VK_SUCCESS;
-    uint32_t count;
-    uint32_t i;
     struct loader_instance *inst = NULL;
 
     loader_platform_thread_lock_mutex(&loader_lock);
@@ -2723,31 +2508,19 @@ LOADER_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDeviceGroups(
         goto out;
     }
 
-    VkResult setup_res = setupLoaderTrampPhysDevGroups(instance, inst);
-    if (VK_SUCCESS != setup_res) {
-        res = setup_res;
-        goto out;
-    }
-
-    count = inst->phys_dev_group_count_tramp;
-
+    // Setup the trampoline loader physical devices.  This will actually
+    // call down and setup the terminator loader physical devices during the
+    // process.
+    res = inst->disp->layer_inst_disp.EnumeratePhysicalDeviceGroups(inst->instance, pPhysicalDeviceGroupCount,
+                                                                    pPhysicalDeviceGroupProperties);
     // Wrap the PhysDev object for loader usage, return wrapped objects
-    if (NULL != pPhysicalDeviceGroupProperties) {
-        if (inst->phys_dev_group_count_tramp > *pPhysicalDeviceGroupCount) {
-            loader_log(inst, VULKAN_LOADER_INFO_BIT, 0,
-                       "vkEnumeratePhysicalDeviceGroupsKHR: Trimming device group count down"
-                       " by application request from %d to %d physical device groups",
-                       inst->phys_dev_group_count_tramp, *pPhysicalDeviceGroupCount);
-            count = *pPhysicalDeviceGroupCount;
-            res = VK_INCOMPLETE;
-        }
-        for (i = 0; i < count; i++) {
-            memcpy(&pPhysicalDeviceGroupProperties[i], inst->phys_dev_groups_tramp[i], sizeof(VkPhysicalDeviceGroupPropertiesKHR));
+    if (NULL != pPhysicalDeviceGroupProperties && (VK_SUCCESS == res || VK_INCOMPLETE == res)) {
+        VkResult update_res = setup_loader_tramp_phys_dev_groups(inst, *pPhysicalDeviceGroupCount, pPhysicalDeviceGroupProperties);
+        if (VK_SUCCESS != update_res) {
+            res = update_res;
         }
     }
 
-    *pPhysicalDeviceGroupCount = count;
-
 out:
 
     loader_platform_thread_unlock_mutex(&loader_lock);
index 56b218e4847b6f4e965ae72418e6680784ea8162..88b28677a47039869a262974bf92fa9cbc72eff9 100644 (file)
Binary files a/tests/framework/framework_config.h.in and b/tests/framework/framework_config.h.in differ
index 8ee79030b8f2898e5715af6fb07cd1c729f682c7..b2916ce7d43632f79129a6347f01b286f131563c 100644 (file)
@@ -76,6 +76,8 @@ struct PhysicalDevice {
 };
 
 struct PhysicalDeviceGroup {
+    PhysicalDeviceGroup() {}
+    PhysicalDeviceGroup(PhysicalDevice const& physical_device) { physical_device_handles.push_back(&physical_device); }
     PhysicalDeviceGroup& use_physical_device(PhysicalDevice const& physical_device) {
         physical_device_handles.push_back(&physical_device);
         return *this;
index 389bddc63326c1727440f3ae6eaf14eb180a6059..cee7a03ebcc6948fb1703eb1d392bb326ae00540 100644 (file)
@@ -209,25 +209,73 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumeratePhysicalDevices(VkInstance instan
 // VK_SUCCESS,VK_INCOMPLETE, VK_ERROR_INITIALIZATION_FAILED
 VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumeratePhysicalDeviceGroups(
     VkInstance instance, uint32_t* pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties* pPhysicalDeviceGroupProperties) {
+    VkResult result = VK_SUCCESS;
+
     if (pPhysicalDeviceGroupProperties == nullptr) {
-        *pPhysicalDeviceGroupCount = static_cast<uint32_t>(icd.physical_device_groups.size());
+        if (0 == icd.physical_device_groups.size()) {
+            *pPhysicalDeviceGroupCount = static_cast<uint32_t>(icd.physical_devices.size());
+        } else {
+            *pPhysicalDeviceGroupCount = static_cast<uint32_t>(icd.physical_device_groups.size());
+        }
     } else {
-        for (size_t device_group = 0; device_group < icd.physical_device_groups.size(); device_group++) {
-            if (device_group >= *pPhysicalDeviceGroupCount) {
-                return VK_INCOMPLETE;
+        // NOTE: This is a fake struct to make sure the pNext chain is properly passed down to the ICD
+        //       vkEnumeratePhysicalDeviceGroups.
+        //       The two versions must match:
+        //           "FakePNext" test in loader_regresion_tests.cpp
+        //           "test_vkEnumeratePhysicalDeviceGroups" in test_icd.cpp
+        struct FakePnextSharedWithICD {
+            VkStructureType sType;
+            void* pNext;
+            uint32_t value;
+        };
+
+        uint32_t group_count = 0;
+        if (0 == icd.physical_device_groups.size()) {
+            group_count = icd.physical_devices.size();
+            for (size_t device_group = 0; device_group < icd.physical_devices.size(); device_group++) {
+                if (device_group >= *pPhysicalDeviceGroupCount) {
+                    group_count = *pPhysicalDeviceGroupCount;
+                    result = VK_INCOMPLETE;
+                    break;
+                }
+                pPhysicalDeviceGroupProperties[device_group].subsetAllocation = false;
+                pPhysicalDeviceGroupProperties[device_group].physicalDeviceCount = 1;
+                pPhysicalDeviceGroupProperties[device_group].physicalDevices[0] =
+                    icd.physical_devices[device_group].vk_physical_device.handle;
             }
-            pPhysicalDeviceGroupProperties[device_group].subsetAllocation =
-                icd.physical_device_groups[device_group].subset_allocation;
-            uint32_t handles_written = 0;
-            for (size_t i = 0; i < icd.physical_device_groups[device_group].physical_device_handles.size(); i++) {
-                handles_written++;
-                pPhysicalDeviceGroupProperties[device_group].physicalDevices[i] =
-                    icd.physical_device_groups[device_group].physical_device_handles[i]->vk_physical_device.handle;
+        } else {
+            group_count = icd.physical_device_groups.size();
+            for (size_t device_group = 0; device_group < icd.physical_device_groups.size(); device_group++) {
+                if (device_group >= *pPhysicalDeviceGroupCount) {
+                    group_count = *pPhysicalDeviceGroupCount;
+                    result = VK_INCOMPLETE;
+                    break;
+                }
+                pPhysicalDeviceGroupProperties[device_group].subsetAllocation =
+                    icd.physical_device_groups[device_group].subset_allocation;
+                uint32_t handles_written = 0;
+                for (size_t i = 0; i < icd.physical_device_groups[device_group].physical_device_handles.size(); i++) {
+                    handles_written++;
+                    pPhysicalDeviceGroupProperties[device_group].physicalDevices[i] =
+                        icd.physical_device_groups[device_group].physical_device_handles[i]->vk_physical_device.handle;
+                }
+                pPhysicalDeviceGroupProperties[device_group].physicalDeviceCount = handles_written;
             }
-            pPhysicalDeviceGroupProperties[device_group].physicalDeviceCount = handles_written;
         }
+        // NOTE: The following code is purely to test pNext passing in vkEnumeratePhysicalDevice groups
+        //       and includes normally invalid information.
+        for (size_t device_group = 0; device_group < group_count; device_group++) {
+            if (nullptr != pPhysicalDeviceGroupProperties[device_group].pNext) {
+                VkBaseInStructure* base = reinterpret_cast<VkBaseInStructure*>(pPhysicalDeviceGroupProperties[device_group].pNext);
+                if (base->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTI_DRAW_PROPERTIES_EXT) {
+                    FakePnextSharedWithICD* fake = reinterpret_cast<FakePnextSharedWithICD*>(base);
+                    fake->value = 0xDECAFBAD;
+                }
+            }
+        }
+        *pPhysicalDeviceGroupCount = static_cast<uint32_t>(group_count);
     }
-    return VK_SUCCESS;
+    return result;
 }
 
 VKAPI_ATTR VkResult VKAPI_CALL test_vkCreateDebugUtilsMessengerEXT(VkInstance instance,
@@ -962,6 +1010,9 @@ PFN_vkVoidFunction get_instance_func_ver_1_1(VkInstance instance, const char* pN
         if (string_eq(pName, "test_vkEnumerateInstanceVersion")) {
             return to_vkVoidFunction(test_vkEnumerateInstanceVersion);
         }
+        if (string_eq(pName, "vkEnumeratePhysicalDeviceGroups")) {
+            return to_vkVoidFunction(test_vkEnumeratePhysicalDeviceGroups);
+        }
     }
     return nullptr;
 }
@@ -1225,8 +1276,10 @@ PFN_vkVoidFunction get_instance_func(VkInstance instance, const char* pName) {
     if (string_eq(pName, "vkCreateInstance")) return to_vkVoidFunction(test_vkCreateInstance);
     if (string_eq(pName, "vkDestroyInstance")) return to_vkVoidFunction(test_vkDestroyInstance);
     if (string_eq(pName, "vkEnumeratePhysicalDevices")) return to_vkVoidFunction(test_vkEnumeratePhysicalDevices);
-    if (string_eq(pName, "vkEnumeratePhysicalDeviceGroups") || string_eq(pName, "vkEnumeratePhysicalDeviceGroupsKHR"))
-        return to_vkVoidFunction(test_vkEnumeratePhysicalDeviceGroups);
+
+    if (IsInstanceExtensionEnabled(VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME)) {
+        if (string_eq(pName, "vkEnumeratePhysicalDeviceGroupsKHR")) return to_vkVoidFunction(test_vkEnumeratePhysicalDeviceGroups);
+    }
 
     PFN_vkVoidFunction ret_phys_dev = get_physical_device_func(instance, pName);
     if (ret_phys_dev != nullptr) return ret_phys_dev;
index dbcbb5f3b6c2494b7391687f8b819bd7a76f55b0..a55d15200e76158fa8646d0c40ae834fb2f2cb18 100644 (file)
@@ -1,7 +1,7 @@
 /*
- * Copyright (c) 2021 The Khronos Group Inc.
- * Copyright (c) 2021 Valve Corporation
- * Copyright (c) 2021 LunarG, Inc.
+ * Copyright (c) 2021-2022 The Khronos Group Inc.
+ * Copyright (c) 2021-2022 Valve Corporation
+ * Copyright (c) 2021-2022 LunarG, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and/or associated documentation files (the "Materials"), to
index d5af270a19608f46486c1f9d03d1e9fd1f6580ed..b65e2e35ce191e142e0c9bd98864dd988c99f8b3 100644 (file)
@@ -1,6 +1,6 @@
 # ~~~
-# Copyright (c) 2021 Valve Corporation
-# Copyright (c) 2021 LunarG, Inc.
+# Copyright (c) 2021-2022 Valve Corporation
+# Copyright (c) 2021-2022 LunarG, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@ target_compile_definitions(test_layer_deps  INTERFACE
 
 target_link_libraries(test_layer_deps INTERFACE testing_framework_util)
 
+# Interface testing layer
 set(TEST_LAYER_SOURCES test_layer.cpp test_layer.h)
 
 set(TEST_LAYER_BASE_EXPORTS TEST_LAYER_EXPORT_ENUMERATE_FUNCTIONS=1)
@@ -53,6 +54,48 @@ target_link_libraries(test_layer_export_version_2 PRIVATE test_layer_deps)
 target_compile_definitions(test_layer_export_version_2 PRIVATE
     ${TEST_LAYER_VERSION_0_EXPORTS} ${TEST_LAYER_VERSION_1_EXPORTS} ${TEST_LAYER_VERSION_2_EXPORTS} )
 
+# Physical device list altering layer (add phys dev, remove phys dev, re-order phys devs)
+set(PHYSDEV_LAYER_ADD_EXPORTS
+        ${TEST_LAYER_VERSION_0_EXPORTS} ${TEST_LAYER_VERSION_1_EXPORTS} ${TEST_LAYER_VERSION_2_EXPORTS}
+        TEST_LAYER_NAME="VkLayer_LunarG_add_phys_dev"
+        TEST_PHYSDEV_LAYER_ADD=1
+    )
+set(PHYSDEV_REMOVE_EXPORTS
+        ${TEST_LAYER_VERSION_0_EXPORTS} ${TEST_LAYER_VERSION_1_EXPORTS} ${TEST_LAYER_VERSION_2_EXPORTS}
+        TEST_LAYER_NAME="VkLayer_LunarG_remove_phys_dev"
+        TEST_PHYSDEV_LAYER_REMOVE=1
+    )
+set(PHYSDEV_LAYER_REORDER_EXPORTS
+        ${TEST_LAYER_VERSION_0_EXPORTS} ${TEST_LAYER_VERSION_1_EXPORTS} ${TEST_LAYER_VERSION_2_EXPORTS}
+        TEST_LAYER_NAME="VkLayer_LunarG_reorder_phys_dev"
+        TEST_PHYSDEV_LAYER_REORDER=1
+    )
+set(PHYSDEV_LAYER_ALL_EXPORTS
+        ${TEST_LAYER_VERSION_0_EXPORTS} ${TEST_LAYER_VERSION_1_EXPORTS} ${TEST_LAYER_VERSION_2_EXPORTS}
+        TEST_LAYER_NAME="VkLayer_LunarG_all_phys_dev"
+        TEST_PHYSDEV_LAYER_ADD=1
+        TEST_PHYSDEV_LAYER_REMOVE=1
+        TEST_PHYSDEV_LAYER_REORDER=1
+    )
+
+add_library(test_layer_physdev_add SHARED ${TEST_LAYER_SOURCES})
+target_link_libraries(test_layer_physdev_add PRIVATE test_layer_deps)
+target_compile_definitions(test_layer_physdev_add PRIVATE ${PHYSDEV_LAYER_ADD_EXPORTS})
+
+add_library(test_layer_physdev_remove SHARED ${TEST_LAYER_SOURCES})
+target_link_libraries(test_layer_physdev_remove PRIVATE test_layer_deps)
+target_compile_definitions(test_layer_physdev_remove PRIVATE ${PHYSDEV_REMOVE_EXPORTS})
+
+add_library(test_layer_physdev_reorder SHARED ${TEST_LAYER_SOURCES})
+target_link_libraries(test_layer_physdev_reorder PRIVATE test_layer_deps)
+target_compile_definitions(test_layer_physdev_reorder PRIVATE ${PHYSDEV_LAYER_REORDER_EXPORTS})
+
+add_library(test_layer_physdev_all SHARED ${TEST_LAYER_SOURCES})
+target_link_libraries(test_layer_physdev_all PRIVATE test_layer_deps)
+target_compile_definitions(test_layer_physdev_all PRIVATE ${PHYSDEV_LAYER_ALL_EXPORTS})
+
+# Wrap Objects layer which wraps dispatchable handles to test that the loader will properly
+# work in the case where handles in the terminator don't match handles in the trampoline portion.
 set(WRAP_LAYER_VERSION_1_EXPORTS
         TEST_LAYER_EXPORT_DIRECT_DISP=1
         TEST_LAYER_EXPORT_MAINT_1=1
@@ -83,11 +126,16 @@ add_library(test_layer_wrap_objects_3 SHARED wrap_objects)
 target_link_libraries(test_layer_wrap_objects_3 PRIVATE test_layer_deps)
 target_compile_definitions(test_layer_wrap_objects_3 PRIVATE ${WRAP_LAYER_VERSION_3_EXPORTS})
 
+# Windows requires export definitions for these layers
 if(WIN32)
     target_sources(test_layer_export_base PRIVATE export_definitions/test_layer_base.def)
     target_sources(test_layer_export_version_0 PRIVATE export_definitions/test_layer_0.def)
     target_sources(test_layer_export_version_1 PRIVATE export_definitions/test_layer_1.def)
     target_sources(test_layer_export_version_2 PRIVATE export_definitions/test_layer_2.def)
+    target_sources(test_layer_physdev_add PRIVATE export_definitions/test_layer_2.def)
+    target_sources(test_layer_physdev_remove PRIVATE export_definitions/test_layer_2.def)
+    target_sources(test_layer_physdev_reorder PRIVATE export_definitions/test_layer_2.def)
+    target_sources(test_layer_physdev_all PRIVATE export_definitions/test_layer_2.def)
     target_sources(test_layer_wrap_objects PRIVATE export_definitions/test_layer_wrap_objects.def)
     target_sources(test_layer_wrap_objects_1 PRIVATE export_definitions/test_layer_wrap_objects.def)
     target_sources(test_layer_wrap_objects_2 PRIVATE export_definitions/test_layer_wrap_objects.def)
index 1efa6a4bad0ebb6ef73f6a2b42175a248b30c093..409da3502ed9bb5edbd7180f54823b383efe9c28 100644 (file)
@@ -1,7 +1,7 @@
 /*
- * Copyright (c) 2021 The Khronos Group Inc.
- * Copyright (c) 2021 Valve Corporation
- * Copyright (c) 2021 LunarG, Inc.
+ * Copyright (c) 2021-2022 The Khronos Group Inc.
+ * Copyright (c) 2021-2022 Valve Corporation
+ * Copyright (c) 2021-2022 LunarG, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and/or associated documentation files (the "Materials"), to
 #define LAYER_EXPORT_NEGOTIATE_LOADER_LAYER_INTERFACE_VERSION 0
 #endif
 
+#ifndef TEST_LAYER_NAME
+#define TEST_LAYER_NAME "VkLayer_LunarG_test_layer"
+#endif
+
 TestLayer layer;
 extern "C" {
 FRAMEWORK_EXPORT TestLayer* get_test_layer_func() { return &layer; }
@@ -107,7 +111,7 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumerateInstanceLayerProperties(uint32_t*
 
 VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumerateInstanceExtensionProperties(const char* pLayerName, uint32_t* pPropertyCount,
                                                                            VkExtensionProperties* pProperties) {
-    if (pLayerName && string_eq(pLayerName, layer.unique_name.c_str())) {
+    if (pLayerName && string_eq(pLayerName, TEST_LAYER_NAME)) {
         *pPropertyCount = 0;
         return VK_SUCCESS;
     }
@@ -122,7 +126,7 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumerateDeviceLayerProperties(VkPhysicalD
 VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumerateDeviceExtensionProperties(VkPhysicalDevice physicalDevice, const char* pLayerName,
                                                                          uint32_t* pPropertyCount,
                                                                          VkExtensionProperties* pProperties) {
-    if (pLayerName && string_eq(pLayerName, layer.unique_name.c_str())) {
+    if (pLayerName && string_eq(pLayerName, TEST_LAYER_NAME)) {
         *pPropertyCount = 0;
         return VK_SUCCESS;
     }
@@ -171,7 +175,7 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vkCreateInstance(const VkInstanceCreateInfo*
 }
 
 VKAPI_ATTR VkResult VKAPI_CALL test_override_vkCreateInstance(const VkInstanceCreateInfo* pCreateInfo,
-                                                     const VkAllocationCallbacks* pAllocator, VkInstance* pInstance) {
+                                                              const VkAllocationCallbacks* pAllocator, VkInstance* pInstance) {
     return VK_ERROR_INVALID_SHADER_NV;
 }
 
@@ -214,6 +218,195 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vkCreateDevice(VkPhysicalDevice physicalDevi
     return result;
 }
 
+VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumeratePhysicalDevices(VkInstance instance, uint32_t* pPhysicalDeviceCount,
+                                                               VkPhysicalDevice* pPhysicalDevices) {
+#if !TEST_PHYSDEV_LAYER_REMOVE && !TEST_PHYSDEV_LAYER_ADD && !TEST_PHYSDEV_LAYER_REORDER
+    return layer.instance_dispatch_table.EnumeratePhysicalDevices(instance, pPhysicalDeviceCount, pPhysicalDevices);
+#else  // TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER
+    VkResult res = VK_SUCCESS;
+
+    if (layer.complete_physical_devices.size() == 0) {
+        // Get list of all physical devices from lower down
+        // NOTE: This only works if we don't test changing the number of devices
+        //       underneath us when using this test.
+        uint32_t icd_count = 0;
+        layer.instance_dispatch_table.EnumeratePhysicalDevices(instance, &icd_count, nullptr);
+        std::vector<VkPhysicalDevice> tmp_vector;
+        tmp_vector.resize(icd_count);
+        layer.instance_dispatch_table.EnumeratePhysicalDevices(instance, &icd_count, tmp_vector.data());
+        layer.complete_physical_devices.clear();
+
+#if TEST_PHYSDEV_LAYER_REMOVE
+        // Erase the 3rd and 4th items
+        layer.removed_physical_devices.push_back(tmp_vector[3]);
+        layer.removed_physical_devices.push_back(tmp_vector[4]);
+        tmp_vector.erase(tmp_vector.begin() + 3);
+        tmp_vector.erase(tmp_vector.begin() + 3);
+#endif  // TEST_PHYSDEV_LAYER_REMOVE
+
+#if TEST_PHYSDEV_LAYER_ADD
+        // Insert a new device in the beginning, middle, and end
+        uint32_t middle = tmp_vector.size() / 2;
+        VkPhysicalDevice new_phys_dev = reinterpret_cast<VkPhysicalDevice>((size_t)(0xABCD0000));
+        layer.added_physical_devices.push_back(new_phys_dev);
+        tmp_vector.insert(tmp_vector.begin(), new_phys_dev);
+        new_phys_dev = reinterpret_cast<VkPhysicalDevice>((size_t)(0xBADC0000));
+        layer.added_physical_devices.push_back(new_phys_dev);
+        tmp_vector.insert(tmp_vector.begin() + middle, new_phys_dev);
+        new_phys_dev = reinterpret_cast<VkPhysicalDevice>((size_t)(0xDCBA0000));
+        layer.added_physical_devices.push_back(new_phys_dev);
+        tmp_vector.push_back(new_phys_dev);
+#endif  // TEST_PHYSDEV_LAYER_ADD
+
+#if TEST_PHYSDEV_LAYER_REORDER
+        // Flip the order of items
+        for (int32_t dev = tmp_vector.size() - 1; dev >= 0; --dev) {
+            layer.complete_physical_devices.push_back(tmp_vector[dev]);
+        }
+#else   // !TEST_PHYSDEV_LAYER_REORDER
+        // Otherwise, keep the order the same
+        for (uint32_t dev = 0; dev < tmp_vector.size(); ++dev) {
+            layer.complete_physical_devices.push_back(tmp_vector[dev]);
+        }
+#endif  // !TEST_PHYSDEV_LAYER_REORDER
+    }
+
+    if (nullptr == pPhysicalDevices) {
+        *pPhysicalDeviceCount = layer.complete_physical_devices.size();
+    } else {
+        uint32_t adj_count = layer.complete_physical_devices.size();
+        if (*pPhysicalDeviceCount < adj_count) {
+            adj_count = *pPhysicalDeviceCount;
+            res = VK_INCOMPLETE;
+        }
+        for (uint32_t dev = 0; dev < adj_count; ++dev) {
+            pPhysicalDevices[dev] = layer.complete_physical_devices[dev];
+        }
+        *pPhysicalDeviceCount = adj_count;
+    }
+
+    return res;
+#endif  // TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER
+}
+
+VKAPI_ATTR void VKAPI_CALL test_vkGetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice,
+                                                              VkPhysicalDeviceProperties* pProperties) {
+#if TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER
+    if (std::find(layer.removed_physical_devices.begin(), layer.removed_physical_devices.end(), physicalDevice) !=
+        layer.removed_physical_devices.end()) {
+        // Should not get here since the application should not know about those devices
+        assert(false);
+    } else if (std::find(layer.added_physical_devices.begin(), layer.added_physical_devices.end(), physicalDevice) !=
+               layer.added_physical_devices.end()) {
+        // Added device so put in some placeholder info we can test against
+        pProperties->apiVersion = VK_API_VERSION_1_2;
+        pProperties->driverVersion = VK_MAKE_API_VERSION(0, 12, 14, 196);
+        pProperties->vendorID = 0xDECAFBAD;
+        pProperties->deviceID = 0xDEADBADD;
+#if defined(_WIN32)
+        strncpy_s(pProperties->deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, "physdev_added_xx", 17);
+#else
+        strncpy(pProperties->deviceName, "physdev_added_xx", VK_MAX_PHYSICAL_DEVICE_NAME_SIZE);
+#endif
+    } else {
+#else  // !TEST_PHYSDEV_LAYER_REMOVE && !TEST_PHYSDEV_LAYER_ADD && !TEST_PHYSDEV_LAYER_REORDER
+    {
+#endif
+        // Not an affected device so just return
+        layer.instance_dispatch_table.GetPhysicalDeviceProperties(physicalDevice, pProperties);
+    }
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumeratePhysicalDeviceGroups(
+    VkInstance instance, uint32_t* pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties* pPhysicalDeviceGroupProperties) {
+#if !TEST_PHYSDEV_LAYER_REMOVE && !TEST_PHYSDEV_LAYER_ADD && !TEST_PHYSDEV_LAYER_REORDER
+    return layer.instance_dispatch_table.EnumeratePhysicalDeviceGroups(instance, pPhysicalDeviceGroupCount,
+                                                                       pPhysicalDeviceGroupProperties);
+#else  // TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER
+    VkResult res = VK_SUCCESS;
+
+    if (layer.complete_physical_device_groups.size() == 0) {
+        uint32_t fake_count = 1000;
+        // Call EnumerateDevices to add remove devices as needed
+        test_vkEnumeratePhysicalDevices(instance, &fake_count, nullptr);
+
+        // Get list of all physical devices from lower down
+        // NOTE: This only works if we don't test changing the number of devices
+        //       underneath us when using this test.
+        uint32_t icd_group_count = 0;
+        layer.instance_dispatch_table.EnumeratePhysicalDeviceGroups(instance, &icd_group_count, nullptr);
+        std::vector<VkPhysicalDeviceGroupProperties> tmp_vector(icd_group_count);
+        for (uint32_t group = 0; group < icd_group_count; ++group) {
+            tmp_vector[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES;
+        }
+        layer.instance_dispatch_table.EnumeratePhysicalDeviceGroups(instance, &icd_group_count, tmp_vector.data());
+        layer.complete_physical_device_groups.clear();
+
+#if TEST_PHYSDEV_LAYER_REMOVE
+        // Now, if a device has been removed, and it was the only group, we need to remove the group as well.
+        for (uint32_t rem_dev = 0; rem_dev < layer.removed_physical_devices.size(); ++rem_dev) {
+            for (uint32_t group = 0; group < icd_group_count; ++group) {
+                for (uint32_t grp_dev = 0; grp_dev < tmp_vector[group].physicalDeviceCount; ++grp_dev) {
+                    if (tmp_vector[group].physicalDevices[grp_dev] == layer.removed_physical_devices[rem_dev]) {
+                        for (uint32_t cp_item = grp_dev + 1; cp_item < tmp_vector[group].physicalDeviceCount; ++cp_item) {
+                            tmp_vector[group].physicalDevices[grp_dev] = tmp_vector[group].physicalDevices[cp_item];
+                        }
+                        tmp_vector[group].physicalDeviceCount--;
+                    }
+                }
+            }
+        }
+        for (uint32_t group = 0; group < tmp_vector.size(); ++group) {
+            if (tmp_vector[group].physicalDeviceCount == 0) {
+                layer.removed_physical_device_groups.push_back(tmp_vector[group]);
+                tmp_vector.erase(tmp_vector.begin() + group);
+                --group;
+            }
+        }
+#endif  // TEST_PHYSDEV_LAYER_REMOVE
+
+#if TEST_PHYSDEV_LAYER_ADD
+        // Add a new group for each physical device not associated with a current group
+        for (uint32_t dev = 0; dev < layer.added_physical_devices.size(); ++dev) {
+            VkPhysicalDeviceGroupProperties props{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES};
+            props.physicalDeviceCount = 1;
+            props.physicalDevices[0] = layer.added_physical_devices[dev];
+            tmp_vector.push_back(props);
+            layer.added_physical_device_groups.push_back(props);
+        }
+#endif  // TEST_PHYSDEV_LAYER_ADD
+
+#if TEST_PHYSDEV_LAYER_REORDER
+        // Flip the order of items
+        for (int32_t dev = tmp_vector.size() - 1; dev >= 0; --dev) {
+            layer.complete_physical_device_groups.push_back(tmp_vector[dev]);
+        }
+#else   // !TEST_PHYSDEV_LAYER_REORDER
+        // Otherwise, keep the order the same
+        for (uint32_t dev = 0; dev < tmp_vector.size(); ++dev) {
+            layer.complete_physical_device_groups.push_back(tmp_vector[dev]);
+        }
+#endif  // !TEST_PHYSDEV_LAYER_REORDER
+    }
+
+    if (nullptr == pPhysicalDeviceGroupProperties) {
+        *pPhysicalDeviceGroupCount = layer.complete_physical_device_groups.size();
+    } else {
+        uint32_t adj_count = layer.complete_physical_device_groups.size();
+        if (*pPhysicalDeviceGroupCount < adj_count) {
+            adj_count = *pPhysicalDeviceGroupCount;
+            res = VK_INCOMPLETE;
+        }
+        for (uint32_t dev = 0; dev < adj_count; ++dev) {
+            pPhysicalDeviceGroupProperties[dev] = layer.complete_physical_device_groups[dev];
+        }
+        *pPhysicalDeviceGroupCount = adj_count;
+    }
+
+    return res;
+#endif  // TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER
+}
+
 // device functions
 
 VKAPI_ATTR void VKAPI_CALL test_vkDestroyDevice(VkDevice device, const VkAllocationCallbacks* pAllocator) {
@@ -244,6 +437,9 @@ VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL get_instance_func(VkInstance instance,
     if (string_eq(pName, "vkEnumerateDeviceLayerProperties")) return to_vkVoidFunction(test_vkEnumerateDeviceLayerProperties);
     if (string_eq(pName, "vkEnumerateDeviceExtensionProperties"))
         return to_vkVoidFunction(test_vkEnumerateDeviceExtensionProperties);
+    if (string_eq(pName, "vkEnumeratePhysicalDevices")) return to_vkVoidFunction(test_vkEnumeratePhysicalDevices);
+    if (string_eq(pName, "vkEnumeratePhysicalDeviceGroups")) return to_vkVoidFunction(test_vkEnumeratePhysicalDeviceGroups);
+    if (string_eq(pName, "vkGetPhysicalDeviceProperties")) return to_vkVoidFunction(test_vkGetPhysicalDeviceProperties);
     if (string_eq(pName, "vkCreateInstance")) return to_vkVoidFunction(test_vkCreateInstance);
     if (string_eq(pName, "vkDestroyInstance")) return to_vkVoidFunction(test_vkDestroyInstance);
     if (string_eq(pName, "vkCreateDevice")) return to_vkVoidFunction(test_vkCreateDevice);
@@ -326,13 +522,27 @@ FRAMEWORK_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumerateDeviceExtensionProper
                                                                                      VkExtensionProperties* pProperties) {
     return test_vkEnumerateDeviceExtensionProperties(physicalDevice, pLayerName, pPropertyCount, pProperties);
 }
+FRAMEWORK_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDevices(VkInstance instance, uint32_t* pPhysicalDeviceCount,
+                                                                           VkPhysicalDevice* pPhysicalDevices) {
+    return test_vkEnumeratePhysicalDevices(instance, pPhysicalDeviceCount, pPhysicalDevices);
+}
+FRAMEWORK_EXPORT VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice,
+                                                                          VkPhysicalDeviceProperties* pProperties) {
+    return test_vkGetPhysicalDeviceProperties(physicalDevice, pProperties);
+}
+FRAMEWORK_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDeviceGroups(
+    VkInstance instance, uint32_t* pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties* pPhysicalDeviceGroupProperties) {
+    return test_vkEnumeratePhysicalDeviceGroups(instance, pPhysicalDeviceGroupCount, pPhysicalDeviceGroupProperties);
+}
+
 #endif
 
 #if TEST_LAYER_EXPORT_LAYER_NAMED_GIPA
 FRAMEWORK_EXPORT VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL test_layer_GetInstanceProcAddr(VkInstance instance, const char* pName) {
     return get_instance_func(instance, pName);
 }
-FRAMEWORK_EXPORT VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL test_override_vkGetInstanceProcAddr(VkInstance instance, const char* pName) {
+FRAMEWORK_EXPORT VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL test_override_vkGetInstanceProcAddr(VkInstance instance,
+                                                                                              const char* pName) {
     if (string_eq(pName, "vkCreateInstance")) return to_vkVoidFunction(test_override_vkCreateInstance);
     return get_instance_func(instance, pName);
 }
index 6c421f4d8fb0c4dddcc92945523177fcd728c9b8..2944a55d551b2e63513fd10ce80e08db0cf5cfe9 100644 (file)
@@ -1,7 +1,7 @@
 /*
- * Copyright (c) 2021 The Khronos Group Inc.
- * Copyright (c) 2021 Valve Corporation
- * Copyright (c) 2021 LunarG, Inc.
+ * Copyright (c) 2021-2022 The Khronos Group Inc.
+ * Copyright (c) 2021-2022 Valve Corporation
+ * Copyright (c) 2021-2022 LunarG, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and/or associated documentation files (the "Materials"), to
@@ -80,6 +80,21 @@ Interface Version 2
 
 // Added manifest version 1.1.0
 
+// vkEnumeratePhysicalDevices and vkEnumeratePhysicalDeviceGroups add a new item
+#ifndef TEST_PHYSDEV_LAYER_ADD
+#define TEST_PHYSDEV_LAYER_ADD 0
+#endif
+
+// vkEnumeratePhysicalDevices and vkEnumeratePhysicalDeviceGroups remove an item
+#ifndef TEST_PHYSDEV_LAYER_REMOVE
+#define TEST_PHYSDEV_LAYER_REMOVE 0
+#endif
+
+// vkEnumeratePhysicalDevices and vkEnumeratePhysicalDeviceGroups reorders items
+#ifndef TEST_PHYSDEV_LAYER_REORDER
+#define TEST_PHYSDEV_LAYER_REORDER 0
+#endif
+
 struct TestLayer;
 
 // Callbacks allow tests to implement custom functionality without modifying the layer binary
@@ -94,7 +109,6 @@ struct TestLayer {
 
     BUILDER_VALUE(TestLayer, bool, is_meta_layer, false)
 
-    BUILDER_VALUE(TestLayer, std::string, unique_name, {})
     BUILDER_VALUE(TestLayer, uint32_t, api_version, VK_API_VERSION_1_0)
     BUILDER_VALUE(TestLayer, uint32_t, reported_layer_props, 1)
     BUILDER_VALUE(TestLayer, uint32_t, reported_extension_props, 1)
@@ -119,6 +133,14 @@ struct TestLayer {
     BUILDER_VALUE(TestLayer, std::function<VkResult(TestLayer& layer)>, create_instance_callback, {})
     // Called in vkCreateDevice after calling down the chain & returning
     BUILDER_VALUE(TestLayer, std::function<VkResult(TestLayer& layer)>, create_device_callback, {})
+#if TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER
+    BUILDER_VECTOR(TestLayer, VkPhysicalDevice, complete_physical_devices, complete_physical_device)
+    BUILDER_VECTOR(TestLayer, VkPhysicalDevice, removed_physical_devices, removed_physical_device)
+    BUILDER_VECTOR(TestLayer, VkPhysicalDevice, added_physical_devices, added_physical_device)
+    BUILDER_VECTOR(TestLayer, VkPhysicalDeviceGroupProperties, complete_physical_device_groups, complete_physical_device_group)
+    BUILDER_VECTOR(TestLayer, VkPhysicalDeviceGroupProperties, removed_physical_device_groups, removed_physical_device_group)
+    BUILDER_VECTOR(TestLayer, VkPhysicalDeviceGroupProperties, added_physical_device_groups, added_physical_device_group)
+#endif // TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER
 
     PFN_vkGetInstanceProcAddr next_vkGetInstanceProcAddr = VK_NULL_HANDLE;
     PFN_GetPhysicalDeviceProcAddr next_GetPhysicalDeviceProcAddr = VK_NULL_HANDLE;
index a8b99fe209d9a6bedb52396178e7724a960e12c3..f78fab1a4fbdff1cd17b6b5f24c34ad5ad29ad0a 100644 (file)
@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2015-2021 Valve Corporation
- * Copyright (c) 2015-2021 LunarG, Inc.
+ * Copyright (c) 2015-2022 Valve Corporation
+ * Copyright (c) 2015-2022 LunarG, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -425,14 +425,25 @@ VKAPI_ATTR VkResult VKAPI_CALL wrap_vkEnumerateDeviceExtensionProperties(VkPhysi
                 ext_count = 0;
 #if TEST_LAYER_EXPORT_MAINT_1
                 if (ext_count < count) {
+#if defined(_WIN32)
+                    strncpy_s(pProperties[ext_count].extensionName, VK_MAX_EXTENSION_NAME_SIZE, VK_KHR_MAINTENANCE1_EXTENSION_NAME,
+                              strlen(VK_KHR_MAINTENANCE1_EXTENSION_NAME) + 1);
+#else
                     strcpy(pProperties[ext_count].extensionName, VK_KHR_MAINTENANCE1_EXTENSION_NAME);
+#endif
                     pProperties[ext_count].specVersion = 2;
                     ext_count++;
                 }
 #endif
 #if TEST_LAYER_EXPORT_PRESENT_IMAGE
                 if (ext_count < count) {
+#if defined(_WIN32)
+                    strncpy_s(pProperties[ext_count].extensionName, VK_MAX_EXTENSION_NAME_SIZE,
+                              VK_KHR_SHARED_PRESENTABLE_IMAGE_EXTENSION_NAME,
+                              strlen(VK_KHR_SHARED_PRESENTABLE_IMAGE_EXTENSION_NAME) + 1);
+#else
                     strcpy(pProperties[ext_count].extensionName, VK_KHR_SHARED_PRESENTABLE_IMAGE_EXTENSION_NAME);
+#endif
                     pProperties[ext_count].specVersion = 1;
                     ext_count++;
                 }
index 411b13b72b764a5d90128ff898653fc38e297619..e9c297c3bd5a0744ae502f8085b15885df824431 100644 (file)
@@ -782,7 +782,7 @@ TEST_F(SeparateMessenger, InfoInEnumDevsIgnoredSeverity) {
 }
 
 // Test debug utils created outside of vkCreateInstance with info in vkEnumeratePhysicalDevices.
-TEST_F(SeparateMessenger, DebugUtilsInfoInEnumDevs) {
+TEST_F(SeparateMessenger, InfoInEnumDevs) {
     expected_message = "Trimming device count down by application request";
     expected_object_type = VK_OBJECT_TYPE_INSTANCE;
     expected_message_flags = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT;
index 969508baaf71374a9a245d914632cf3a0f4071ab..8db228089c0231320fcb1b9971e9091a5a7d5d88 100644 (file)
@@ -1,7 +1,7 @@
 /*
- * Copyright (c) 2021 The Khronos Group Inc.
- * Copyright (c) 2021 Valve Corporation
- * Copyright (c) 2021 LunarG, Inc.
+ * Copyright (c) 2021-2022 The Khronos Group Inc.
+ * Copyright (c) 2021-2022 Valve Corporation
+ * Copyright (c) 2021-2022 LunarG, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and/or associated documentation files (the "Materials"), to
@@ -45,6 +45,7 @@ class LayerExtensions : public LayerTests {};
 class MetaLayers : public LayerTests {};
 class OverrideMetaLayer : public LayerTests {};
 class LayerCreateInstance : public LayerTests {};
+class LayerPhysDeviceMod : public LayerTests {};
 
 void CheckLogForLayerString(FrameworkEnvironment& env, const char* implicit_layer_name, bool check_for_enable) {
     {
@@ -2776,3 +2777,544 @@ TEST(TestLayers, DeviceLayerNotPresent) {
     dev.create_info.add_layer(explicit_layer_name);
     dev.CheckCreate(phys_dev);
 }
+
+TEST_F(LayerPhysDeviceMod, AddPhysicalDevices) {
+    FrameworkEnvironment env;
+    env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{}
+                                                         .set_name("VkLayer_LunarG_add_phys_dev")
+                                                         .set_lib_path(TEST_LAYER_PHYSDEV_ADD)
+                                                         .set_api_version(VK_API_VERSION_1_1)
+                                                         .set_disable_environment("TEST_DISABLE_ADD_PHYS_DEV")),
+                           "test_layer_remove.json");
+
+    for (uint32_t icd = 0; icd < 2; ++icd) {
+        env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2));
+        auto& cur_icd = env.get_test_icd(icd);
+        cur_icd.icd_api_version = VK_API_VERSION_1_2;
+        VkPhysicalDeviceProperties properties{};
+        properties.apiVersion = VK_API_VERSION_1_2;
+        properties.vendorID = 0x11000000 + (icd << 6);
+        char vendor_char = 'a' + icd;
+        for (uint32_t dev = 0; dev < 3; ++dev) {
+            properties.deviceID = properties.vendorID + dev;
+            properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
+            char dev_char = '0' + dev;
+            std::string dev_name = "physdev_";
+            dev_name += vendor_char;
+            dev_name += "_";
+            dev_name += dev_char;
+#if defined(_WIN32)
+            strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1);
+#else
+            strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE);
+#endif
+            cur_icd.add_physical_device({});
+            cur_icd.physical_devices.back().set_properties(properties);
+        }
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]);
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]);
+        cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]);
+    }
+    const uint32_t icd_devices = 6;
+
+    InstWrapper inst{env.vulkan_functions};
+    inst.create_info.set_api_version(VK_API_VERSION_1_1);
+    inst.CheckCreate();
+
+    uint32_t dev_count = 0;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &dev_count, nullptr));
+    ASSERT_GT(dev_count, icd_devices);
+
+    auto not_exp_physical_devices = std::vector<VkPhysicalDevice>(icd_devices);
+    uint32_t returned_phys_dev_count = icd_devices;
+    ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, not_exp_physical_devices.data()));
+    ASSERT_EQ(icd_devices, returned_phys_dev_count);
+
+    auto physical_devices = std::vector<VkPhysicalDevice>(dev_count);
+    returned_phys_dev_count = dev_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data()));
+    ASSERT_EQ(dev_count, returned_phys_dev_count);
+
+    uint32_t diff_count = dev_count - icd_devices;
+    uint32_t found_incomplete = 0;
+    uint32_t found_added_count = 0;
+    for (uint32_t dev = 0; dev < dev_count; ++dev) {
+        VkPhysicalDeviceProperties props{};
+        inst->vkGetPhysicalDeviceProperties(physical_devices[dev], &props);
+        if (string_eq(props.deviceName, "physdev_added_xx")) {
+            found_added_count++;
+        }
+        for (uint32_t incomp = 0; incomp < icd_devices; ++incomp) {
+            if (not_exp_physical_devices[incomp] == physical_devices[dev]) {
+                found_incomplete++;
+                break;
+            }
+        }
+    }
+
+    // We should have added the number of diff items, and the incomplete count should match the number of
+    // original physical devices.
+    ASSERT_EQ(found_added_count, diff_count);
+    ASSERT_EQ(found_incomplete, icd_devices);
+}
+
+TEST_F(LayerPhysDeviceMod, RemovePhysicalDevices) {
+    FrameworkEnvironment env;
+    env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{}
+                                                         .set_name("VkLayer_LunarG_remove_phys_dev")
+                                                         .set_lib_path(TEST_LAYER_PHYSDEV_REMOVE)
+                                                         .set_api_version(VK_API_VERSION_1_1)
+                                                         .set_disable_environment("TEST_DISABLE_REMOVE_PHYS_DEV")),
+                           "test_layer_remove.json");
+
+    for (uint32_t icd = 0; icd < 2; ++icd) {
+        env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2));
+        auto& cur_icd = env.get_test_icd(icd);
+        cur_icd.icd_api_version = VK_API_VERSION_1_2;
+        VkPhysicalDeviceProperties properties{};
+        properties.apiVersion = VK_API_VERSION_1_2;
+        properties.vendorID = 0x11000000 + (icd << 6);
+        char vendor_char = 'a' + icd;
+        for (uint32_t dev = 0; dev < 3; ++dev) {
+            properties.deviceID = properties.vendorID + dev;
+            properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
+            char dev_char = '0' + dev;
+            std::string dev_name = "physdev_";
+            dev_name += vendor_char;
+            dev_name += "_";
+            dev_name += dev_char;
+#if defined(_WIN32)
+            strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1);
+#else
+            strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE);
+#endif
+            cur_icd.add_physical_device({});
+            cur_icd.physical_devices.back().set_properties(properties);
+        }
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]);
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]);
+        cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]);
+    }
+    const uint32_t icd_devices = 6;
+
+    InstWrapper inst{env.vulkan_functions};
+    inst.create_info.set_api_version(VK_API_VERSION_1_1);
+    inst.CheckCreate();
+
+    uint32_t dev_count = 0;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &dev_count, nullptr));
+    ASSERT_LT(dev_count, icd_devices);
+
+    auto physical_devices = std::vector<VkPhysicalDevice>(dev_count);
+    uint32_t returned_phys_dev_count = dev_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data()));
+    ASSERT_EQ(dev_count, returned_phys_dev_count);
+}
+
+TEST_F(LayerPhysDeviceMod, ReorderPhysicalDevices) {
+    FrameworkEnvironment env;
+    env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{}
+                                                         .set_name("VkLayer_LunarG_reorder_phys_dev")
+                                                         .set_lib_path(TEST_LAYER_PHYSDEV_REORDER)
+                                                         .set_api_version(VK_API_VERSION_1_1)
+                                                         .set_disable_environment("TEST_DISABLE_REORDER_PHYS_DEV")),
+                           "test_layer_reorder.json");
+
+    for (uint32_t icd = 0; icd < 2; ++icd) {
+        env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2));
+        auto& cur_icd = env.get_test_icd(icd);
+        cur_icd.icd_api_version = VK_API_VERSION_1_2;
+        VkPhysicalDeviceProperties properties{};
+        properties.apiVersion = VK_API_VERSION_1_2;
+        properties.vendorID = 0x11000000 + (icd << 6);
+        char vendor_char = 'a' + icd;
+        for (uint32_t dev = 0; dev < 3; ++dev) {
+            properties.deviceID = properties.vendorID + dev;
+            properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
+            char dev_char = '0' + dev;
+            std::string dev_name = "physdev_";
+            dev_name += vendor_char;
+            dev_name += "_";
+            dev_name += dev_char;
+#if defined(_WIN32)
+            strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1);
+#else
+            strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE);
+#endif
+            cur_icd.add_physical_device({});
+            cur_icd.physical_devices.back().set_properties(properties);
+        }
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]);
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]);
+        cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]);
+    }
+    const uint32_t icd_devices = 6;
+
+    InstWrapper inst{env.vulkan_functions};
+    inst.create_info.set_api_version(VK_API_VERSION_1_1);
+    inst.CheckCreate();
+
+    uint32_t dev_count = 0;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &dev_count, nullptr));
+    ASSERT_EQ(dev_count, icd_devices);
+
+    auto physical_devices = std::vector<VkPhysicalDevice>(dev_count);
+    uint32_t returned_phys_dev_count = dev_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data()));
+    ASSERT_EQ(dev_count, returned_phys_dev_count);
+}
+
+TEST_F(LayerPhysDeviceMod, AddRemoveAndReorderPhysicalDevices) {
+    FrameworkEnvironment env;
+    env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{}
+                                                         .set_name("VkLayer_LunarG_all_phys_dev")
+                                                         .set_lib_path(TEST_LAYER_PHYSDEV_ALL)
+                                                         .set_api_version(VK_API_VERSION_1_1)
+                                                         .set_disable_environment("TEST_DISABLE_ALL_PHYS_DEV")),
+                           "test_layer_all.json");
+
+    for (uint32_t icd = 0; icd < 2; ++icd) {
+        env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2));
+        auto& cur_icd = env.get_test_icd(icd);
+        cur_icd.icd_api_version = VK_API_VERSION_1_2;
+        VkPhysicalDeviceProperties properties{};
+        properties.apiVersion = VK_API_VERSION_1_2;
+        properties.vendorID = 0x11000000 + (icd << 6);
+        char vendor_char = 'a' + icd;
+        for (uint32_t dev = 0; dev < 3; ++dev) {
+            properties.deviceID = properties.vendorID + dev;
+            properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
+            char dev_char = '0' + dev;
+            std::string dev_name = "physdev_";
+            dev_name += vendor_char;
+            dev_name += "_";
+            dev_name += dev_char;
+#if defined(_WIN32)
+            strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1);
+#else
+            strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE);
+#endif
+            cur_icd.add_physical_device({});
+            cur_icd.physical_devices.back().set_properties(properties);
+        }
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]);
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]);
+        cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]);
+    }
+    const uint32_t icd_devices = 6;
+
+    InstWrapper inst{env.vulkan_functions};
+    inst.create_info.set_api_version(VK_API_VERSION_1_1);
+    inst.CheckCreate();
+
+    uint32_t dev_count = 0;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &dev_count, nullptr));
+    ASSERT_GT(dev_count, icd_devices);
+
+    auto physical_devices = std::vector<VkPhysicalDevice>(dev_count);
+    uint32_t returned_phys_dev_count = dev_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data()));
+    ASSERT_EQ(dev_count, returned_phys_dev_count);
+
+    uint32_t found_added_count = 0;
+    for (uint32_t dev = 0; dev < dev_count; ++dev) {
+        VkPhysicalDeviceProperties props{};
+        inst->vkGetPhysicalDeviceProperties(physical_devices[dev], &props);
+        if (string_eq(props.deviceName, "physdev_added_xx")) {
+            found_added_count++;
+        }
+    }
+
+    // Should see 2 removed, but 3 added so a diff count of 1
+    uint32_t diff_count = dev_count - icd_devices;
+    ASSERT_EQ(1, diff_count);
+    ASSERT_EQ(found_added_count, 3);
+}
+
+static bool GroupsAreTheSame(VkPhysicalDeviceGroupProperties a, VkPhysicalDeviceGroupProperties b) {
+    if (a.physicalDeviceCount != b.physicalDeviceCount) {
+        return false;
+    }
+    for (uint32_t dev = 0; dev < a.physicalDeviceCount; ++dev) {
+        if (a.physicalDevices[dev] != b.physicalDevices[dev]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+TEST_F(LayerPhysDeviceMod, AddPhysicalDeviceGroups) {
+    FrameworkEnvironment env;
+    env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{}
+                                                         .set_name("VkLayer_LunarG_add_phys_dev")
+                                                         .set_lib_path(TEST_LAYER_PHYSDEV_ADD)
+                                                         .set_api_version(VK_API_VERSION_1_1)
+                                                         .set_disable_environment("TEST_DISABLE_ADD_PHYS_DEV")),
+                           "test_layer_remove.json");
+
+    for (uint32_t icd = 0; icd < 2; ++icd) {
+        env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2));
+        auto& cur_icd = env.get_test_icd(icd);
+        cur_icd.icd_api_version = VK_API_VERSION_1_2;
+        VkPhysicalDeviceProperties properties{};
+        properties.apiVersion = VK_API_VERSION_1_2;
+        properties.vendorID = 0x11000000 + (icd << 6);
+        char vendor_char = 'a' + icd;
+        for (uint32_t dev = 0; dev < 3; ++dev) {
+            properties.deviceID = properties.vendorID + dev;
+            properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
+            char dev_char = '0' + dev;
+            std::string dev_name = "physdev_";
+            dev_name += vendor_char;
+            dev_name += "_";
+            dev_name += dev_char;
+#if defined(_WIN32)
+            strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1);
+#else
+            strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE);
+#endif
+            cur_icd.add_physical_device({});
+            cur_icd.physical_devices.back().set_properties(properties);
+        }
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]);
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]);
+        cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]);
+    }
+    const uint32_t icd_groups = 4;
+
+    InstWrapper inst{env.vulkan_functions};
+    inst.create_info.set_api_version(VK_API_VERSION_1_1);
+    inst.CheckCreate();
+
+    uint32_t grp_count = 0;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &grp_count, nullptr));
+    ASSERT_GT(grp_count, icd_groups);
+
+    auto not_exp_phys_dev_groups = std::vector<VkPhysicalDeviceGroupProperties>(icd_groups);
+    for (uint32_t group = 0; group < icd_groups; ++group) {
+        not_exp_phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES;
+    }
+    uint32_t returned_group_count = icd_groups;
+    ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, not_exp_phys_dev_groups.data()));
+    ASSERT_EQ(icd_groups, returned_group_count);
+
+    auto phys_dev_groups = std::vector<VkPhysicalDeviceGroupProperties>(grp_count);
+    for (uint32_t group = 0; group < grp_count; ++group) {
+        phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES;
+    }
+    returned_group_count = grp_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, phys_dev_groups.data()));
+    ASSERT_EQ(grp_count, returned_group_count);
+
+    uint32_t diff_count = grp_count - icd_groups;
+    uint32_t found_incomplete = 0;
+    uint32_t found_added_count = 0;
+    for (uint32_t grp = 0; grp < grp_count; ++grp) {
+        // Shortcut, only groups with 1 device could be added in the newly added count
+        if (1 == phys_dev_groups[grp].physicalDeviceCount) {
+            for (uint32_t dev = 0; dev < phys_dev_groups[grp].physicalDeviceCount; ++dev) {
+                VkPhysicalDeviceProperties props{};
+                inst->vkGetPhysicalDeviceProperties(phys_dev_groups[grp].physicalDevices[dev], &props);
+                if (string_eq(props.deviceName, "physdev_added_xx")) {
+                    found_added_count++;
+                }
+            }
+        }
+        for (uint32_t incomp = 0; incomp < icd_groups; ++incomp) {
+            if (GroupsAreTheSame(not_exp_phys_dev_groups[incomp], phys_dev_groups[grp])) {
+                found_incomplete++;
+                break;
+            }
+        }
+    }
+
+    // We should have added the number of diff items, and the incomplete count should match the number of
+    // original physical devices.
+    ASSERT_EQ(found_added_count, diff_count);
+    ASSERT_EQ(found_incomplete, icd_groups);
+}
+
+TEST_F(LayerPhysDeviceMod, RemovePhysicalDeviceGroups) {
+    FrameworkEnvironment env;
+    env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{}
+                                                         .set_name("VkLayer_LunarG_remove_phys_dev")
+                                                         .set_lib_path(TEST_LAYER_PHYSDEV_REMOVE)
+                                                         .set_api_version(VK_API_VERSION_1_1)
+                                                         .set_disable_environment("TEST_DISABLE_REMOVE_PHYS_DEV")),
+                           "test_layer_remove.json");
+
+    for (uint32_t icd = 0; icd < 2; ++icd) {
+        env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2));
+        auto& cur_icd = env.get_test_icd(icd);
+        cur_icd.icd_api_version = VK_API_VERSION_1_2;
+        VkPhysicalDeviceProperties properties{};
+        properties.apiVersion = VK_API_VERSION_1_2;
+        properties.vendorID = 0x11000000 + (icd << 6);
+        char vendor_char = 'a' + icd;
+        for (uint32_t dev = 0; dev < 3; ++dev) {
+            properties.deviceID = properties.vendorID + dev;
+            properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
+            char dev_char = '0' + dev;
+            std::string dev_name = "physdev_";
+            dev_name += vendor_char;
+            dev_name += "_";
+            dev_name += dev_char;
+            strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE);
+            cur_icd.add_physical_device({});
+            cur_icd.physical_devices.back().set_properties(properties);
+        }
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]);
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]);
+        cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]);
+    }
+    const uint32_t icd_groups = 4;
+
+    InstWrapper inst{env.vulkan_functions};
+    inst.create_info.set_api_version(VK_API_VERSION_1_1);
+    inst.CheckCreate();
+
+    uint32_t grp_count = 0;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &grp_count, nullptr));
+    ASSERT_LT(grp_count, icd_groups);
+
+    auto phys_dev_groups = std::vector<VkPhysicalDeviceGroupProperties>(grp_count);
+    for (uint32_t group = 0; group < grp_count; ++group) {
+        phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES;
+    }
+    uint32_t returned_group_count = grp_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, phys_dev_groups.data()));
+    ASSERT_EQ(grp_count, returned_group_count);
+}
+
+TEST_F(LayerPhysDeviceMod, ReorderPhysicalDeviceGroups) {
+    FrameworkEnvironment env;
+    env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{}
+                                                         .set_name("VkLayer_LunarG_reorder_phys_dev")
+                                                         .set_lib_path(TEST_LAYER_PHYSDEV_REORDER)
+                                                         .set_api_version(VK_API_VERSION_1_1)
+                                                         .set_disable_environment("TEST_DISABLE_REORDER_PHYS_DEV")),
+                           "test_layer_reorder.json");
+
+    for (uint32_t icd = 0; icd < 2; ++icd) {
+        env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2));
+        auto& cur_icd = env.get_test_icd(icd);
+        cur_icd.icd_api_version = VK_API_VERSION_1_2;
+        VkPhysicalDeviceProperties properties{};
+        properties.apiVersion = VK_API_VERSION_1_2;
+        properties.vendorID = 0x11000000 + (icd << 6);
+        char vendor_char = 'a' + icd;
+        for (uint32_t dev = 0; dev < 3; ++dev) {
+            properties.deviceID = properties.vendorID + dev;
+            properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
+            char dev_char = '0' + dev;
+            std::string dev_name = "physdev_";
+            dev_name += vendor_char;
+            dev_name += "_";
+            dev_name += dev_char;
+#if defined(_WIN32)
+            strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1);
+#else
+            strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE);
+#endif
+            cur_icd.add_physical_device({});
+            cur_icd.physical_devices.back().set_properties(properties);
+        }
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]);
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]);
+        cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]);
+    }
+    const uint32_t icd_groups = 4;
+
+    InstWrapper inst{env.vulkan_functions};
+    inst.create_info.set_api_version(VK_API_VERSION_1_1);
+    inst.CheckCreate();
+
+    uint32_t grp_count = 0;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &grp_count, nullptr));
+    ASSERT_EQ(grp_count, icd_groups);
+
+    auto phys_dev_groups = std::vector<VkPhysicalDeviceGroupProperties>(grp_count);
+    for (uint32_t group = 0; group < grp_count; ++group) {
+        phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES;
+    }
+    uint32_t returned_group_count = grp_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, phys_dev_groups.data()));
+    ASSERT_EQ(grp_count, returned_group_count);
+}
+
+TEST_F(LayerPhysDeviceMod, AddRemoveAndReorderPhysicalDeviceGroups) {
+    FrameworkEnvironment env;
+    env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{}
+                                                         .set_name("VkLayer_LunarG_all_phys_dev")
+                                                         .set_lib_path(TEST_LAYER_PHYSDEV_ALL)
+                                                         .set_api_version(VK_API_VERSION_1_1)
+                                                         .set_disable_environment("TEST_DISABLE_ALL_PHYS_DEV")),
+                           "test_layer_all.json");
+
+    for (uint32_t icd = 0; icd < 2; ++icd) {
+        env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2));
+        auto& cur_icd = env.get_test_icd(icd);
+        cur_icd.icd_api_version = VK_API_VERSION_1_2;
+        VkPhysicalDeviceProperties properties{};
+        properties.apiVersion = VK_API_VERSION_1_2;
+        properties.vendorID = 0x11000000 + (icd << 6);
+        char vendor_char = 'a' + icd;
+        for (uint32_t dev = 0; dev < 3; ++dev) {
+            properties.deviceID = properties.vendorID + dev;
+            properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU;
+            char dev_char = '0' + dev;
+            std::string dev_name = "physdev_";
+            dev_name += vendor_char;
+            dev_name += "_";
+            dev_name += dev_char;
+#if defined(_WIN32)
+            strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1);
+#else
+            strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE);
+#endif
+            cur_icd.add_physical_device({});
+            cur_icd.physical_devices.back().set_properties(properties);
+        }
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]);
+        cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]);
+        cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]);
+    }
+    const uint32_t icd_groups = 4;
+
+    InstWrapper inst{env.vulkan_functions};
+    inst.create_info.set_api_version(VK_API_VERSION_1_1);
+    inst.CheckCreate();
+
+    uint32_t grp_count = 0;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &grp_count, nullptr));
+    ASSERT_GT(grp_count, icd_groups);
+
+    auto phys_dev_groups = std::vector<VkPhysicalDeviceGroupProperties>(grp_count);
+    for (uint32_t group = 0; group < grp_count; ++group) {
+        phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES;
+    }
+    uint32_t returned_group_count = grp_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, phys_dev_groups.data()));
+    ASSERT_EQ(grp_count, returned_group_count);
+
+    uint32_t diff_count = grp_count - icd_groups;
+    uint32_t found_added_count = 0;
+    for (uint32_t grp = 0; grp < grp_count; ++grp) {
+        // Shortcut, only groups with 1 device could be added in the newly added count
+        if (1 == phys_dev_groups[grp].physicalDeviceCount) {
+            for (uint32_t dev = 0; dev < phys_dev_groups[grp].physicalDeviceCount; ++dev) {
+                VkPhysicalDeviceProperties props{};
+                inst->vkGetPhysicalDeviceProperties(phys_dev_groups[grp].physicalDevices[dev], &props);
+                if (string_eq(props.deviceName, "physdev_added_xx")) {
+                    found_added_count++;
+                }
+            }
+        }
+    }
+
+    // Should see 2 devices removed which should result in 1 group removed and since 3
+    // devices were added we should have 3 new groups.  So we should have a diff of 2
+    // groups and 3 new groups
+    ASSERT_EQ(2, diff_count);
+    ASSERT_EQ(found_added_count, 3);
+}
index 4ce8263c8d7ad9400d663ee6cfd37e868639a9a1..142a9f0e0cd3f76a6ed15ba143eb07b62042559f 100644 (file)
@@ -385,8 +385,7 @@ TEST_F(EnumeratePhysicalDevices, OneCall) {
 
 TEST_F(EnumeratePhysicalDevices, TwoCall) {
     auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
-    Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME};  // known instance extensions
-    env->reset_icd().add_instance_extension(first_ext);
+    driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME});
 
     const uint32_t real_device_count = 2;
     for (uint32_t i = 0; i < real_device_count; i++) {
@@ -411,8 +410,7 @@ TEST_F(EnumeratePhysicalDevices, TwoCall) {
 TEST_F(EnumeratePhysicalDevices, MatchOneAndTwoCallNumbers) {
     auto& driver = env->get_test_icd();
     driver.set_min_icd_interface_version(5);
-    Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME};  // known instance extensions
-    env->reset_icd().add_instance_extension(first_ext);
+    driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME});
 
     const uint32_t real_device_count = 3;
     for (uint32_t i = 0; i < real_device_count; i++) {
@@ -446,8 +444,7 @@ TEST_F(EnumeratePhysicalDevices, MatchOneAndTwoCallNumbers) {
 
 TEST_F(EnumeratePhysicalDevices, TwoCallIncomplete) {
     auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
-    Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME};  // known instance extensions
-    env->reset_icd().add_instance_extension(first_ext);
+    driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME});
 
     const uint32_t real_device_count = 2;
     for (uint32_t i = 0; i < real_device_count; i++) {
@@ -469,6 +466,20 @@ TEST_F(EnumeratePhysicalDevices, TwoCallIncomplete) {
 
     ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &physical_count, physical.data()));
     ASSERT_EQ(physical_count, 1);
+
+    physical_count = 2;
+    std::array<VkPhysicalDevice, real_device_count> physical_2;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &physical_count, physical_2.data()));
+
+    // Verify that the first physical device shows up in the list of the second ones
+    bool found = false;
+    for (uint32_t dev = 0; dev < physical_count; ++dev) {
+        if (physical_2[dev] == physical[0]) {
+            found = true;
+            break;
+        }
+    }
+    ASSERT_EQ(true, found);
 }
 
 TEST_F(EnumeratePhysicalDevices, ZeroPhysicalDevices) {
@@ -502,10 +513,325 @@ TEST_F(EnumeratePhysicalDevices, ZeroPhysicalDevicesAfterCreateInstance) {
               inst->vkEnumeratePhysicalDeviceGroups(inst, &physical_device_group_count, &physical_device_group_properties));
 }
 
-// LX535 / MI-76: Device layers are deprecated.
-// Ensure that no errors occur if a bogus device layer list is passed to vkCreateDevice.
-// https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/vkspec.html#extendingvulkan-layers-devicelayerdeprecation
-TEST_F(CreateDevice, LayersNotPresent) {
+TEST_F(EnumeratePhysicalDevices, CallTwiceNormal) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
+
+    for (size_t i = 0; i < 4; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.CheckCreate();
+
+    // Call twice in a row and make sure nothing bad happened
+    uint32_t physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    uint32_t returned_physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    std::vector<VkPhysicalDevice> physical_device_handles_1 = std::vector<VkPhysicalDevice>(physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_1.data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+    std::vector<VkPhysicalDevice> physical_device_handles_2 = std::vector<VkPhysicalDevice>(physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_2.data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+    // Make sure devices are same between the two
+    for (uint32_t count = 0; count < driver.physical_devices.size(); ++count) {
+        ASSERT_EQ(physical_device_handles_1[count], physical_device_handles_2[count]);
+    }
+}
+
+TEST_F(EnumeratePhysicalDevices, CallTwiceIncompleteOnceNormal) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
+
+    for (size_t i = 0; i < 8; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.CheckCreate();
+
+    // Query 3, then 5, then all
+    uint32_t physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    uint32_t returned_physical_count = 3;
+    std::vector<VkPhysicalDevice> physical_device_handles_1 = std::vector<VkPhysicalDevice>(returned_physical_count);
+    ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_1.data()));
+    ASSERT_EQ(3, returned_physical_count);
+    returned_physical_count = 5;
+    std::vector<VkPhysicalDevice> physical_device_handles_2 = std::vector<VkPhysicalDevice>(returned_physical_count);
+    ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_2.data()));
+    ASSERT_EQ(5, returned_physical_count);
+    returned_physical_count = physical_count;
+    std::vector<VkPhysicalDevice> physical_device_handles_3 = std::vector<VkPhysicalDevice>(returned_physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_3.data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+    // Make sure devices are same between the three
+    for (uint32_t count = 0; count < driver.physical_devices.size(); ++count) {
+        if (count < physical_device_handles_1.size()) {
+            ASSERT_EQ(physical_device_handles_1[count], physical_device_handles_3[count]);
+        }
+        if (count < physical_device_handles_2.size()) {
+            ASSERT_EQ(physical_device_handles_2[count], physical_device_handles_3[count]);
+        }
+    }
+}
+
+TEST_F(EnumeratePhysicalDevices, CallThriceSuccessReduce) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
+
+    for (size_t i = 0; i < 8; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.CheckCreate();
+
+    // Query all at first, then 5, then 3
+    uint32_t physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    uint32_t returned_physical_count = physical_count;
+    std::vector<VkPhysicalDevice> physical_device_handles_1 = std::vector<VkPhysicalDevice>(physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_1.data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+    returned_physical_count = 5;
+    std::vector<VkPhysicalDevice> physical_device_handles_2 = std::vector<VkPhysicalDevice>(returned_physical_count);
+    ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_2.data()));
+    ASSERT_EQ(5, returned_physical_count);
+    returned_physical_count = 3;
+    std::vector<VkPhysicalDevice> physical_device_handles_3 = std::vector<VkPhysicalDevice>(returned_physical_count);
+    ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_3.data()));
+    ASSERT_EQ(3, returned_physical_count);
+    // Make sure devices are same between the three
+    for (uint32_t count = 0; count < driver.physical_devices.size(); ++count) {
+        if (count < physical_device_handles_2.size()) {
+            ASSERT_EQ(physical_device_handles_2[count], physical_device_handles_1[count]);
+        }
+        if (count < physical_device_handles_3.size()) {
+            ASSERT_EQ(physical_device_handles_3[count], physical_device_handles_1[count]);
+        }
+    }
+}
+
+TEST_F(EnumeratePhysicalDevices, CallThriceAddInBetween) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
+
+    driver.physical_devices.emplace_back("physical_device_0");
+    driver.physical_devices.emplace_back("physical_device_1");
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.CheckCreate();
+
+    uint32_t physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    uint32_t returned_physical_count = physical_count;
+    std::vector<VkPhysicalDevice> physical_device_handles_1 = std::vector<VkPhysicalDevice>(physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_1.data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+
+    driver.physical_devices.emplace_back("physical_device_2");
+    driver.physical_devices.emplace_back("physical_device_3");
+
+    std::vector<VkPhysicalDevice> physical_device_handles_2 = std::vector<VkPhysicalDevice>(returned_physical_count);
+    ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_2.data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+
+    physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    returned_physical_count = physical_count;
+    std::vector<VkPhysicalDevice> physical_device_handles_3 = std::vector<VkPhysicalDevice>(returned_physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_3.data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+    // Make sure devices are same between the three
+    for (uint32_t count = 0; count < physical_device_handles_3.size(); ++count) {
+        if (count < physical_device_handles_1.size()) {
+            ASSERT_EQ(physical_device_handles_1[count], physical_device_handles_3[count]);
+        }
+        if (count < physical_device_handles_2.size()) {
+            ASSERT_EQ(physical_device_handles_2[count], physical_device_handles_3[count]);
+        }
+    }
+}
+
+TEST_F(EnumeratePhysicalDevices, CallThriceRemoveInBetween) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
+
+    for (size_t i = 0; i < 4; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.CheckCreate();
+
+    uint32_t physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    uint32_t returned_physical_count = physical_count;
+    std::vector<VkPhysicalDevice> physical_device_handles_1 = std::vector<VkPhysicalDevice>(physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_1.data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+
+    // Delete the 2nd physical device
+    driver.physical_devices.erase(std::next(driver.physical_devices.begin()));
+
+    physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    std::vector<VkPhysicalDevice> physical_device_handles_2 = std::vector<VkPhysicalDevice>(returned_physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_2.data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+    physical_device_handles_2.resize(returned_physical_count);
+
+    returned_physical_count = physical_count;
+    std::vector<VkPhysicalDevice> physical_device_handles_3 = std::vector<VkPhysicalDevice>(returned_physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_3.data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+
+    // Make sure one has 1 more device that two or three
+    ASSERT_EQ(physical_device_handles_1.size(), physical_device_handles_2.size() + 1);
+    ASSERT_EQ(physical_device_handles_1.size(), physical_device_handles_3.size() + 1);
+
+    // Make sure the devices in two and three are all found in one
+    uint32_t two_found = 0;
+    uint32_t three_found = 0;
+    for (uint32_t count = 0; count < physical_device_handles_1.size(); ++count) {
+        for (uint32_t int_count = 0; int_count < physical_device_handles_2.size(); ++int_count) {
+            if (physical_device_handles_2[int_count] == physical_device_handles_1[count]) {
+                two_found++;
+                break;
+            }
+        }
+        for (uint32_t int_count = 0; int_count < physical_device_handles_3.size(); ++int_count) {
+            if (physical_device_handles_3[int_count] == physical_device_handles_1[count]) {
+                three_found++;
+                break;
+            }
+        }
+    }
+    ASSERT_EQ(two_found, returned_physical_count);
+    ASSERT_EQ(three_found, returned_physical_count);
+}
+
+TEST_F(EnumeratePhysicalDevices, MultipleAddRemoves) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
+
+    for (size_t i = 0; i < 4; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+    std::array<std::vector<VkPhysicalDevice>, 8> physical_dev_handles;
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.CheckCreate();
+
+    uint32_t physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    uint32_t returned_physical_count = physical_count;
+    physical_dev_handles[0].resize(physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[0].data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+
+    // Delete the 2nd physical device (0, 2, 3)
+    driver.physical_devices.erase(std::next(driver.physical_devices.begin()));
+
+    // Query using old number from last call (4), but it should only return 3
+    physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    physical_dev_handles[1].resize(returned_physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[1].data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+    physical_dev_handles[1].resize(returned_physical_count);
+
+    // Add two new physical devices to the front (A, B, 0, 2, 3)
+    driver.physical_devices.emplace(driver.physical_devices.begin(), "physical_device_B");
+    driver.physical_devices.emplace(driver.physical_devices.begin(), "physical_device_A");
+
+    // Query using old number from last call (3), but it should be 5
+    physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    physical_dev_handles[2].resize(returned_physical_count);
+    ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[2].data()));
+    ASSERT_EQ(physical_count - 2, returned_physical_count);
+    physical_dev_handles[2].resize(returned_physical_count);
+
+    // Query again to get all 5
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, nullptr));
+    physical_dev_handles[3].resize(returned_physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[3].data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+
+    // Delete last two physical devices (A, B, 0, 2)
+    driver.physical_devices.pop_back();
+
+    // Query using old number from last call (5), but it should be 4
+    physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    physical_dev_handles[4].resize(returned_physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[4].data()));
+    ASSERT_EQ(physical_count, returned_physical_count);
+    physical_dev_handles[4].resize(returned_physical_count);
+    // Adjust size and query again, should be the same
+    physical_dev_handles[5].resize(returned_physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[5].data()));
+
+    // Insert a new physical device (A, B, C, 0, 2)
+    driver.physical_devices.insert(driver.physical_devices.begin() + 2, "physical_device_C");
+
+    // Query using old number from last call (4), but it should be 5
+    physical_count = static_cast<uint32_t>(driver.physical_devices.size());
+    physical_dev_handles[6].resize(returned_physical_count);
+    ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[6].data()));
+    ASSERT_EQ(physical_count - 1, returned_physical_count);
+    // Query again to get all 5
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, nullptr));
+    physical_dev_handles[7].resize(returned_physical_count);
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[7].data()));
+
+    // Check final results
+    // One   [4] - 0, 1, 2, 3
+    // Two   [3] - 0, 2, 3
+    // Three [3] - A, B, 0
+    // Four  [5] - A, B, 0, 2, 3
+    // Five  [4] - A, B, 0, 2
+    // Six   [4] - A, B, 0, 2
+    // Seven [4] - A, B, C, 0
+    // Eight [5] - A, B, C, 0, 2
+    ASSERT_EQ(4, physical_dev_handles[0].size());
+    ASSERT_EQ(3, physical_dev_handles[1].size());
+    ASSERT_EQ(3, physical_dev_handles[2].size());
+    ASSERT_EQ(5, physical_dev_handles[3].size());
+    ASSERT_EQ(4, physical_dev_handles[4].size());
+    ASSERT_EQ(4, physical_dev_handles[5].size());
+    ASSERT_EQ(4, physical_dev_handles[6].size());
+    ASSERT_EQ(5, physical_dev_handles[7].size());
+
+    // Make sure the devices in two and three are all found in one
+    uint32_t found_items[8]{};
+    for (uint32_t handle = 1; handle < 8; ++handle) {
+        for (uint32_t count = 0; count < physical_dev_handles[0].size(); ++count) {
+            for (uint32_t int_count = 0; int_count < physical_dev_handles[handle].size(); ++int_count) {
+                if (physical_dev_handles[handle][int_count] == physical_dev_handles[0][count]) {
+                    found_items[handle]++;
+                    break;
+                }
+            }
+        }
+    }
+    // Items matching from first call (must be >= since handle re-use does occur)
+    ASSERT_EQ(found_items[1], 3);
+    ASSERT_GE(found_items[2], 1);
+    ASSERT_GE(found_items[3], 3);
+    ASSERT_GE(found_items[4], 2);
+    ASSERT_GE(found_items[5], 2);
+    ASSERT_GE(found_items[6], 1);
+    ASSERT_GE(found_items[7], 2);
+
+    memset(found_items, 0, 8 * sizeof(uint32_t));
+    for (uint32_t handle = 0; handle < 7; ++handle) {
+        for (uint32_t count = 0; count < physical_dev_handles[7].size(); ++count) {
+            for (uint32_t int_count = 0; int_count < physical_dev_handles[handle].size(); ++int_count) {
+                if (physical_dev_handles[handle][int_count] == physical_dev_handles[7][count]) {
+                    found_items[handle]++;
+                    break;
+                }
+            }
+        }
+    }
+    // Items matching from last call (must be >= since handle re-use does occur)
+    ASSERT_GE(found_items[0], 2);
+    ASSERT_GE(found_items[1], 2);
+    ASSERT_GE(found_items[2], 3);
+    ASSERT_GE(found_items[3], 4);
+    ASSERT_GE(found_items[4], 4);
+    ASSERT_GE(found_items[5], 4);
+    ASSERT_GE(found_items[6], 4);
+}
+
+TEST_F(CreateDevice, ExtensionNotPresent) {
     auto& driver = env->get_test_icd();
 
     MockQueueFamilyProperties family_props{{VK_QUEUE_GRAPHICS_BIT, 1, 0, {1, 1, 1}}, true};
@@ -528,13 +854,15 @@ TEST_F(CreateDevice, LayersNotPresent) {
     ASSERT_EQ(families, family_props.properties);
 
     DeviceWrapper dev{inst};
-    DeviceCreateInfo dev_create_info;
-    dev.create_info.add_layer("NotPresent").add_device_queue(DeviceQueueCreateInfo{}.add_priority(0.0f));
+    dev.create_info.add_extension("NotPresent").add_device_queue(DeviceQueueCreateInfo{}.add_priority(0.0f));
 
-    dev.CheckCreate(phys_dev);
+    dev.CheckCreate(phys_dev, VK_ERROR_EXTENSION_NOT_PRESENT);
 }
 
-TEST_F(CreateDevice, ExtensionNotPresent) {
+// LX535 / MI-76: Device layers are deprecated.
+// Ensure that no errors occur if a bogus device layer list is passed to vkCreateDevice.
+// https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/vkspec.html#extendingvulkan-layers-devicelayerdeprecation
+TEST_F(CreateDevice, LayersNotPresent) {
     auto& driver = env->get_test_icd();
 
     MockQueueFamilyProperties family_props{{VK_QUEUE_GRAPHICS_BIT, 1, 0, {1, 1, 1}}, true};
@@ -557,9 +885,10 @@ TEST_F(CreateDevice, ExtensionNotPresent) {
     ASSERT_EQ(families, family_props.properties);
 
     DeviceWrapper dev{inst};
-    dev.create_info.add_extension("NotPresent").add_device_queue(DeviceQueueCreateInfo{}.add_priority(0.0f));
+    DeviceCreateInfo dev_create_info;
+    dev.create_info.add_layer("NotPresent").add_device_queue(DeviceQueueCreateInfo{}.add_priority(0.0f));
 
-    dev.CheckCreate(phys_dev, VK_ERROR_EXTENSION_NOT_PRESENT);
+    dev.CheckCreate(phys_dev);
 }
 
 TEST(TryLoadWrongBinaries, WrongICD) {
@@ -853,21 +1182,18 @@ TEST(TryLoadWrongBinaries, BadExplicitAndImplicit) {
 }
 
 TEST_F(EnumeratePhysicalDeviceGroups, OneCall) {
-    auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
-    Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME};  // known instance extensions
-    env->reset_icd().add_instance_extension(first_ext);
-
-    // ICD contains 2 devices
-    driver.physical_devices.emplace_back("PhysicalDevice0", 12);
-    driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
-    driver.physical_devices.emplace_back("PhysicalDevice1", 24);
-    driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
-    // ICD contains 1 group, which contains both devices
-    driver.physical_device_groups.push_back({});
-    driver.physical_device_groups.back()
-        .use_physical_device(driver.physical_devices[0])
-        .use_physical_device(driver.physical_devices[1]);
-    uint32_t physical_device_count = 2;
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1);
+    driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME});
+
+    // ICD contains 3 devices in two groups
+    for (size_t i = 0; i < 3; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i), rand() % 50 + 3);
+        driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    }
+    driver.physical_device_groups.emplace_back(driver.physical_devices[0]);
+    driver.physical_device_groups.back().use_physical_device(driver.physical_devices[1]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[2]);
+    const uint32_t max_physical_device_count = 3;
 
     // Core function
     {
@@ -875,58 +1201,89 @@ TEST_F(EnumeratePhysicalDeviceGroups, OneCall) {
         inst.create_info.set_api_version(VK_API_VERSION_1_1);
         inst.CheckCreate();
 
-        auto physical_devices = std::vector<VkPhysicalDevice>(physical_device_count);
-        uint32_t returned_phys_dev_count = physical_device_count;
+        auto physical_devices = std::vector<VkPhysicalDevice>(max_physical_device_count);
+        uint32_t returned_phys_dev_count = max_physical_device_count;
         ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data()));
         handle_assert_has_values(physical_devices);
 
         uint32_t group_count = static_cast<uint32_t>(driver.physical_device_groups.size());
         uint32_t returned_group_count = group_count;
-        VkPhysicalDeviceGroupProperties group_props{};
-        group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES;
-        ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, &group_props));
+        std::vector<VkPhysicalDeviceGroupProperties> group_props{};
+        group_props.resize(group_count, VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+        ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props.data()));
         ASSERT_EQ(group_count, returned_group_count);
+
+        // Make sure each physical device shows up in a group, but only once
+        std::array<bool, max_physical_device_count> found{false};
+        for (uint32_t group = 0; group < group_count; ++group) {
+            for (uint32_t g_dev = 0; g_dev < group_props[group].physicalDeviceCount; ++g_dev) {
+                for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) {
+                    if (physical_devices[dev] == group_props[group].physicalDevices[g_dev]) {
+                        ASSERT_EQ(false, found[dev]);
+                        found[dev] = true;
+                        break;
+                    }
+                }
+            }
+        }
+        for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) {
+            ASSERT_EQ(true, found[dev]);
+        }
     }
-    driver.add_instance_extension({"VK_KHR_device_group_creation"});
+    driver.add_instance_extension({VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME});
     // Extension
     {
         InstWrapper inst{env->vulkan_functions};
-        inst.create_info.add_extension("VK_KHR_device_group_creation");
+        inst.create_info.add_extension(VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME);
         inst.CheckCreate();
 
         auto vkEnumeratePhysicalDeviceGroupsKHR = reinterpret_cast<PFN_vkEnumeratePhysicalDeviceGroupsKHR>(
             env->vulkan_functions.vkGetInstanceProcAddr(inst.inst, "vkEnumeratePhysicalDeviceGroupsKHR"));
 
-        auto physical_devices = std::vector<VkPhysicalDevice>(physical_device_count);
-        uint32_t returned_phys_dev_count = physical_device_count;
+        auto physical_devices = std::vector<VkPhysicalDevice>(max_physical_device_count);
+        uint32_t returned_phys_dev_count = max_physical_device_count;
         ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data()));
         handle_assert_has_values(physical_devices);
 
         uint32_t group_count = static_cast<uint32_t>(driver.physical_device_groups.size());
         uint32_t returned_group_count = group_count;
-        VkPhysicalDeviceGroupPropertiesKHR group_props{};
-        group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR;
-        ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, &group_props));
+        std::vector<VkPhysicalDeviceGroupPropertiesKHR> group_props{};
+        group_props.resize(group_count, VkPhysicalDeviceGroupPropertiesKHR{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR});
+        ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, group_props.data()));
         ASSERT_EQ(group_count, returned_group_count);
+
+        // Make sure each physical device shows up in a group, but only once
+        std::array<bool, max_physical_device_count> found{false};
+        for (uint32_t group = 0; group < group_count; ++group) {
+            for (uint32_t g_dev = 0; g_dev < group_props[group].physicalDeviceCount; ++g_dev) {
+                for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) {
+                    if (physical_devices[dev] == group_props[group].physicalDevices[g_dev]) {
+                        ASSERT_EQ(false, found[dev]);
+                        found[dev] = true;
+                        break;
+                    }
+                }
+            }
+        }
+        for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) {
+            ASSERT_EQ(true, found[dev]);
+        }
     }
 }
 
 TEST_F(EnumeratePhysicalDeviceGroups, TwoCall) {
-    auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
-    Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME};  // known instance extensions
-    env->reset_icd().add_instance_extension(first_ext);
-
-    // ICD contains 2 devices
-    driver.physical_devices.emplace_back("PhysicalDevice0", 12);
-    driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
-    driver.physical_devices.emplace_back("PhysicalDevice1", 24);
-    driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
-    // ICD contains 1 group, which contains both devices
-    driver.physical_device_groups.push_back({});
-    driver.physical_device_groups.back()
-        .use_physical_device(driver.physical_devices[0])
-        .use_physical_device(driver.physical_devices[1]);
-    uint32_t physical_device_count = 2;
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1);
+    driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME});
+
+    // ICD contains 3 devices in two groups
+    for (size_t i = 0; i < 3; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i), rand() % 50 + 3);
+        driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    }
+    driver.physical_device_groups.emplace_back(driver.physical_devices[0]);
+    driver.physical_device_groups.back().use_physical_device(driver.physical_devices[1]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[2]);
+    const uint32_t max_physical_device_count = 3;
 
     // Core function
     {
@@ -934,8 +1291,8 @@ TEST_F(EnumeratePhysicalDeviceGroups, TwoCall) {
         inst.create_info.set_api_version(VK_API_VERSION_1_1);
         inst.CheckCreate();
 
-        auto physical_devices = std::vector<VkPhysicalDevice>(physical_device_count);
-        uint32_t returned_phys_dev_count = physical_device_count;
+        auto physical_devices = std::vector<VkPhysicalDevice>(max_physical_device_count);
+        uint32_t returned_phys_dev_count = max_physical_device_count;
         ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data()));
         handle_assert_has_values(physical_devices);
 
@@ -944,20 +1301,37 @@ TEST_F(EnumeratePhysicalDeviceGroups, TwoCall) {
         ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, nullptr));
         ASSERT_EQ(group_count, returned_group_count);
 
-        VkPhysicalDeviceGroupProperties group_props{};
-        group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES;
-        ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, &group_props));
+        std::vector<VkPhysicalDeviceGroupProperties> group_props{};
+        group_props.resize(group_count, VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+        ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props.data()));
         ASSERT_EQ(group_count, returned_group_count);
+
+        // Make sure each physical device shows up in a group, but only once
+        std::array<bool, max_physical_device_count> found{false};
+        for (uint32_t group = 0; group < group_count; ++group) {
+            for (uint32_t g_dev = 0; g_dev < group_props[group].physicalDeviceCount; ++g_dev) {
+                for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) {
+                    if (physical_devices[dev] == group_props[group].physicalDevices[g_dev]) {
+                        ASSERT_EQ(false, found[dev]);
+                        found[dev] = true;
+                        break;
+                    }
+                }
+            }
+        }
+        for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) {
+            ASSERT_EQ(true, found[dev]);
+        }
     }
-    driver.add_instance_extension({"VK_KHR_device_group_creation"});
+    driver.add_instance_extension({VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME});
     // Extension
     {
         InstWrapper inst{env->vulkan_functions};
         inst.create_info.add_extension("VK_KHR_device_group_creation");
         inst.CheckCreate();
 
-        auto physical_devices = std::vector<VkPhysicalDevice>(physical_device_count);
-        uint32_t returned_phys_dev_count = physical_device_count;
+        auto physical_devices = std::vector<VkPhysicalDevice>(max_physical_device_count);
+        uint32_t returned_phys_dev_count = max_physical_device_count;
         ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data()));
         handle_assert_has_values(physical_devices);
 
@@ -969,28 +1343,42 @@ TEST_F(EnumeratePhysicalDeviceGroups, TwoCall) {
         ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, nullptr));
         ASSERT_EQ(group_count, returned_group_count);
 
-        VkPhysicalDeviceGroupPropertiesKHR group_props{};
-        group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR;
-        ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, &group_props));
+        std::vector<VkPhysicalDeviceGroupPropertiesKHR> group_props{};
+        group_props.resize(group_count, VkPhysicalDeviceGroupPropertiesKHR{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR});
+        ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, group_props.data()));
         ASSERT_EQ(group_count, returned_group_count);
+
+        // Make sure each physical device shows up in a group, but only once
+        std::array<bool, max_physical_device_count> found{false};
+        for (uint32_t group = 0; group < group_count; ++group) {
+            for (uint32_t g_dev = 0; g_dev < group_props[group].physicalDeviceCount; ++g_dev) {
+                for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) {
+                    if (physical_devices[dev] == group_props[group].physicalDevices[g_dev]) {
+                        ASSERT_EQ(false, found[dev]);
+                        found[dev] = true;
+                        break;
+                    }
+                }
+            }
+        }
+        for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) {
+            ASSERT_EQ(true, found[dev]);
+        }
     }
 }
 
 TEST_F(EnumeratePhysicalDeviceGroups, TwoCallIncomplete) {
-    auto& driver = env->get_test_icd().set_min_icd_interface_version(5);
-    Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME};  // known instance extensions
-    env->reset_icd().add_instance_extension(first_ext);
-
-    // ICD contains 2 devices
-    driver.physical_devices.emplace_back("PhysicalDevice0", 12);
-    driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
-    driver.physical_devices.emplace_back("PhysicalDevice1", 24);
-    driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
-    // ICD contains 1 group, which contains both devices
-    driver.physical_device_groups.push_back({});
-    driver.physical_device_groups.back()
-        .use_physical_device(driver.physical_devices[0])
-        .use_physical_device(driver.physical_devices[1]);
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1);
+    driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME});
+
+    // ICD contains 3 devices in two groups
+    for (size_t i = 0; i < 3; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i), rand() % 50 + 3);
+        driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    }
+    driver.physical_device_groups.emplace_back(driver.physical_devices[0]);
+    driver.physical_device_groups.back().use_physical_device(driver.physical_devices[1]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[2]);
 
     // Core function
     {
@@ -1002,15 +1390,32 @@ TEST_F(EnumeratePhysicalDeviceGroups, TwoCallIncomplete) {
         uint32_t returned_group_count = 0;
         ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, nullptr));
         ASSERT_EQ(group_count, returned_group_count);
-        returned_group_count = 0;
 
-        VkPhysicalDeviceGroupProperties group_props{};
-        group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES;
-        ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, &group_props));
-        ASSERT_EQ(0, returned_group_count);
-        handle_assert_no_values(returned_group_count, group_props.physicalDevices);
+        returned_group_count = 1;
+        std::array<VkPhysicalDeviceGroupProperties, 1> group_props{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES};
+        ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props.data()));
+        ASSERT_EQ(1, returned_group_count);
+
+        returned_group_count = 2;
+        std::array<VkPhysicalDeviceGroupProperties, 2> group_props_2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES};
+        ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_2.data()));
+        ASSERT_EQ(2, returned_group_count);
+
+        // Make sure the incomplete group items appear in the complete group
+        for (uint32_t inc_group = 0; inc_group < 1; ++inc_group) {
+            bool found = false;
+            for (uint32_t full_group = 0; full_group < 2; ++full_group) {
+                if (group_props[inc_group].physicalDeviceCount == group_props_2[full_group].physicalDeviceCount &&
+                    group_props[inc_group].physicalDevices[0] == group_props_2[full_group].physicalDevices[0] &&
+                    group_props[inc_group].physicalDevices[1] == group_props_2[full_group].physicalDevices[1]) {
+                    found = true;
+                    break;
+                }
+            }
+            ASSERT_EQ(true, found);
+        }
     }
-    driver.add_instance_extension({"VK_KHR_device_group_creation"});
+    driver.add_instance_extension({VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME});
     // Extension
     {
         InstWrapper inst{env->vulkan_functions};
@@ -1024,13 +1429,633 @@ TEST_F(EnumeratePhysicalDeviceGroups, TwoCallIncomplete) {
         uint32_t returned_group_count = 0;
         ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, nullptr));
         ASSERT_EQ(group_count, returned_group_count);
-        returned_group_count = 0;
 
-        VkPhysicalDeviceGroupPropertiesKHR group_props{};
-        group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR;
-        ASSERT_EQ(VK_INCOMPLETE, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, &group_props));
-        ASSERT_EQ(0, returned_group_count);
-        handle_assert_no_values(returned_group_count, group_props.physicalDevices);
+        returned_group_count = 1;
+        std::array<VkPhysicalDeviceGroupPropertiesKHR, 1> group_props{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR};
+        ASSERT_EQ(VK_INCOMPLETE, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, group_props.data()));
+        ASSERT_EQ(1, returned_group_count);
+
+        returned_group_count = 2;
+        std::array<VkPhysicalDeviceGroupPropertiesKHR, 2> group_props_2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR};
+        ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, group_props_2.data()));
+        ASSERT_EQ(2, returned_group_count);
+
+        // Make sure the incomplete group items appear in the complete group
+        for (uint32_t inc_group = 0; inc_group < 1; ++inc_group) {
+            bool found = false;
+            for (uint32_t full_group = 0; full_group < 2; ++full_group) {
+                if (group_props[inc_group].physicalDeviceCount == group_props_2[full_group].physicalDeviceCount &&
+                    group_props[inc_group].physicalDevices[0] == group_props_2[full_group].physicalDevices[0] &&
+                    group_props[inc_group].physicalDevices[1] == group_props_2[full_group].physicalDevices[1]) {
+                    found = true;
+                    break;
+                }
+            }
+            ASSERT_EQ(true, found);
+        }
+    }
+}
+
+// Call the core vkEnumeratePhysicalDeviceGroups and the extension
+// vkEnumeratePhysicalDeviceGroupsKHR, and make sure they return the same info.
+TEST_F(EnumeratePhysicalDeviceGroups, TestCoreVersusExtensionSameReturns) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1);
+    driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME});
+    driver.add_instance_extension({VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME});
+
+    // Generate the devices
+    for (size_t i = 0; i < 6; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+
+    // Generate the starting groups
+    driver.physical_device_groups.emplace_back(driver.physical_devices[0]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[1]);
+    driver.physical_device_groups.back()
+        .use_physical_device(driver.physical_devices[2])
+        .use_physical_device(driver.physical_devices[3]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[4]);
+    driver.physical_device_groups.back().use_physical_device(driver.physical_devices[5]);
+
+    uint32_t expected_counts[3] = {1, 3, 2};
+    uint32_t core_group_count = 0;
+    std::vector<VkPhysicalDeviceGroupProperties> core_group_props{};
+    uint32_t ext_group_count = 0;
+    std::vector<VkPhysicalDeviceGroupPropertiesKHR> ext_group_props{};
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.create_info.set_api_version(1, 1, 0);
+    inst.create_info.add_extension("VK_KHR_device_group_creation");
+    inst.CheckCreate();
+
+    // Core function
+    core_group_count = static_cast<uint32_t>(driver.physical_device_groups.size());
+    uint32_t returned_group_count = 0;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, nullptr));
+    ASSERT_EQ(core_group_count, returned_group_count);
+
+    core_group_props.resize(returned_group_count,
+                            VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, core_group_props.data()));
+    ASSERT_EQ(core_group_count, returned_group_count);
+
+    auto vkEnumeratePhysicalDeviceGroupsKHR = reinterpret_cast<PFN_vkEnumeratePhysicalDeviceGroupsKHR>(
+        env->vulkan_functions.vkGetInstanceProcAddr(inst.inst, "vkEnumeratePhysicalDeviceGroupsKHR"));
+
+    ext_group_count = static_cast<uint32_t>(driver.physical_device_groups.size());
+    returned_group_count = 0;
+    ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, nullptr));
+    ASSERT_EQ(ext_group_count, returned_group_count);
+
+    ext_group_props.resize(returned_group_count,
+                           VkPhysicalDeviceGroupPropertiesKHR{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR});
+    ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, ext_group_props.data()));
+    ASSERT_EQ(ext_group_count, returned_group_count);
+
+    // Make sure data from each matches
+    ASSERT_EQ(core_group_count, 3);
+    ASSERT_EQ(ext_group_count, 3);
+    for (uint32_t group = 0; group < core_group_count; ++group) {
+        ASSERT_EQ(core_group_props[group].physicalDeviceCount, expected_counts[group]);
+        ASSERT_EQ(ext_group_props[group].physicalDeviceCount, expected_counts[group]);
+        for (uint32_t dev = 0; dev < core_group_props[group].physicalDeviceCount; ++dev) {
+            ASSERT_EQ(core_group_props[group].physicalDevices[dev], ext_group_props[group].physicalDevices[dev]);
+        }
+    }
+    // Make sure no physical device appears in more than one group
+    for (uint32_t group1 = 0; group1 < core_group_count; ++group1) {
+        for (uint32_t group2 = group1 + 1; group2 < core_group_count; ++group2) {
+            for (uint32_t dev1 = 0; dev1 < core_group_props[group1].physicalDeviceCount; ++dev1) {
+                for (uint32_t dev2 = 0; dev2 < core_group_props[group1].physicalDeviceCount; ++dev2) {
+                    ASSERT_NE(core_group_props[group1].physicalDevices[dev1], core_group_props[group2].physicalDevices[dev2]);
+                }
+            }
+        }
+    }
+}
+
+// Start with 6 devices in 3 different groups, and then add a group,
+// querying vkEnumeratePhysicalDeviceGroups before and after the add.
+TEST_F(EnumeratePhysicalDeviceGroups, CallThriceAddGroupInBetween) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1);
+
+    // Generate the devices
+    for (size_t i = 0; i < 7; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+
+    // Generate the starting groups
+    driver.physical_device_groups.emplace_back(driver.physical_devices[0]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[1]);
+    driver.physical_device_groups.back()
+        .use_physical_device(driver.physical_devices[2])
+        .use_physical_device(driver.physical_devices[3]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[4]);
+    driver.physical_device_groups.back().use_physical_device(driver.physical_devices[5]);
+
+    uint32_t before_expected_counts[3] = {1, 3, 2};
+    uint32_t after_expected_counts[4] = {1, 3, 1, 2};
+    uint32_t before_group_count = 3;
+    uint32_t after_group_count = 4;
+    uint32_t returned_group_count = 0;
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.create_info.set_api_version(1, 1, 0);
+    inst.CheckCreate();
+
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_before{};
+    group_props_before.resize(before_group_count,
+                              VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    returned_group_count = before_group_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_before.data()));
+    ASSERT_EQ(before_group_count, returned_group_count);
+    for (uint32_t group = 0; group < before_group_count; ++group) {
+        ASSERT_EQ(group_props_before[group].physicalDeviceCount, before_expected_counts[group]);
+    }
+
+    // Insert new group after first two
+    driver.physical_device_groups.insert(driver.physical_device_groups.begin() + 2, driver.physical_devices[6]);
+
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_after{};
+    group_props_after.resize(before_group_count,
+                             VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after.data()));
+    ASSERT_EQ(before_group_count, returned_group_count);
+    for (uint32_t group = 0; group < before_group_count; ++group) {
+        ASSERT_EQ(group_props_after[group].physicalDeviceCount, after_expected_counts[group]);
+    }
+
+    group_props_after.resize(after_group_count,
+                             VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    returned_group_count = after_group_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after.data()));
+    ASSERT_EQ(after_group_count, returned_group_count);
+    for (uint32_t group = 0; group < after_group_count; ++group) {
+        ASSERT_EQ(group_props_after[group].physicalDeviceCount, after_expected_counts[group]);
+    }
+
+    // Make sure all devices in the old group info are still found in the new group info
+    for (uint32_t group1 = 0; group1 < group_props_before.size(); ++group1) {
+        for (uint32_t group2 = 0; group2 < group_props_after.size(); ++group2) {
+            if (group_props_before[group1].physicalDeviceCount == group_props_after[group2].physicalDeviceCount) {
+                uint32_t found_count = 0;
+                bool found;
+                for (uint32_t dev1 = 0; dev1 < group_props_before[group1].physicalDeviceCount; ++dev1) {
+                    found = false;
+                    for (uint32_t dev2 = 0; dev2 < group_props_after[group2].physicalDeviceCount; ++dev2) {
+                        if (group_props_before[group1].physicalDevices[dev1] == group_props_after[group2].physicalDevices[dev2]) {
+                            found_count++;
+                            found = true;
+                            break;
+                        }
+                    }
+                }
+                ASSERT_EQ(found, found_count == group_props_before[group1].physicalDeviceCount);
+            }
+        }
+    }
+}
+
+// Start with 7 devices in 4 different groups, and then remove a group,
+// querying vkEnumeratePhysicalDeviceGroups before and after the remove.
+TEST_F(EnumeratePhysicalDeviceGroups, CallTwiceRemoveGroupInBetween) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1);
+
+    // Generate the devices
+    for (size_t i = 0; i < 7; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+
+    // Generate the starting groups
+    driver.physical_device_groups.emplace_back(driver.physical_devices[0]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[1]);
+    driver.physical_device_groups.back()
+        .use_physical_device(driver.physical_devices[2])
+        .use_physical_device(driver.physical_devices[3]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[4]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[5]);
+    driver.physical_device_groups.back().use_physical_device(driver.physical_devices[6]);
+
+    uint32_t before_expected_counts[4] = {1, 3, 1, 2};
+    uint32_t after_expected_counts[3] = {1, 3, 2};
+    uint32_t before_group_count = 4;
+    uint32_t after_group_count = 3;
+    uint32_t returned_group_count = 0;
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.create_info.set_api_version(1, 1, 0);
+    inst.CheckCreate();
+
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_before{};
+    group_props_before.resize(before_group_count,
+                              VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    returned_group_count = before_group_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_before.data()));
+    ASSERT_EQ(before_group_count, returned_group_count);
+    for (uint32_t group = 0; group < before_group_count; ++group) {
+        ASSERT_EQ(group_props_before[group].physicalDeviceCount, before_expected_counts[group]);
+    }
+
+    // Insert new group after first two
+    driver.physical_device_groups.erase(driver.physical_device_groups.begin() + 2);
+
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_after{};
+    group_props_after.resize(after_group_count,
+                             VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after.data()));
+    ASSERT_EQ(after_group_count, returned_group_count);
+    for (uint32_t group = 0; group < after_group_count; ++group) {
+        ASSERT_EQ(group_props_after[group].physicalDeviceCount, after_expected_counts[group]);
+    }
+
+    // Make sure all devices in the new group info are found in the old group info
+    for (uint32_t group1 = 0; group1 < group_props_after.size(); ++group1) {
+        for (uint32_t group2 = 0; group2 < group_props_before.size(); ++group2) {
+            if (group_props_after[group1].physicalDeviceCount == group_props_before[group2].physicalDeviceCount) {
+                uint32_t found_count = 0;
+                bool found;
+                for (uint32_t dev1 = 0; dev1 < group_props_after[group1].physicalDeviceCount; ++dev1) {
+                    found = false;
+                    for (uint32_t dev2 = 0; dev2 < group_props_before[group2].physicalDeviceCount; ++dev2) {
+                        if (group_props_after[group1].physicalDevices[dev1] == group_props_before[group2].physicalDevices[dev2]) {
+                            found_count++;
+                            found = true;
+                            break;
+                        }
+                    }
+                }
+                ASSERT_EQ(found, found_count == group_props_after[group1].physicalDeviceCount);
+            }
+        }
+    }
+}
+
+// Start with 6 devices in 3 different groups, and then add a device to the middle group,
+// querying vkEnumeratePhysicalDeviceGroups before and after the add.
+TEST_F(EnumeratePhysicalDeviceGroups, CallTwiceAddDeviceInBetween) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1);
+
+    // Generate the devices
+    for (size_t i = 0; i < 7; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+
+    // Generate the starting groups
+    driver.physical_device_groups.emplace_back(driver.physical_devices[0]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[1]);
+    driver.physical_device_groups.back()
+        .use_physical_device(driver.physical_devices[2])
+        .use_physical_device(driver.physical_devices[3]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[4]);
+    driver.physical_device_groups.back().use_physical_device(driver.physical_devices[5]);
+
+    uint32_t expected_group_count = 3;
+    uint32_t before_expected_counts[3] = {1, 3, 2};
+    uint32_t after_expected_counts[3] = {1, 4, 2};
+    uint32_t returned_group_count = 0;
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.create_info.set_api_version(1, 1, 0);
+    inst.CheckCreate();
+
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_before{};
+    group_props_before.resize(expected_group_count,
+                              VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    returned_group_count = expected_group_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_before.data()));
+    ASSERT_EQ(expected_group_count, returned_group_count);
+    for (uint32_t group = 0; group < expected_group_count; ++group) {
+        ASSERT_EQ(group_props_before[group].physicalDeviceCount, before_expected_counts[group]);
+    }
+
+    // Insert new device to 2nd group
+    driver.physical_device_groups[1].use_physical_device(driver.physical_devices[6]);
+
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_after{};
+    group_props_after.resize(expected_group_count,
+                             VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after.data()));
+    ASSERT_EQ(expected_group_count, returned_group_count);
+    for (uint32_t group = 0; group < expected_group_count; ++group) {
+        ASSERT_EQ(group_props_after[group].physicalDeviceCount, after_expected_counts[group]);
+    }
+
+    // Make sure all devices in the old group info are still found in the new group info
+    for (uint32_t group1 = 0; group1 < group_props_before.size(); ++group1) {
+        for (uint32_t group2 = 0; group2 < group_props_after.size(); ++group2) {
+            uint32_t found_count = 0;
+            bool found;
+            for (uint32_t dev1 = 0; dev1 < group_props_before[group1].physicalDeviceCount; ++dev1) {
+                found = false;
+                for (uint32_t dev2 = 0; dev2 < group_props_after[group2].physicalDeviceCount; ++dev2) {
+                    if (group_props_before[group1].physicalDevices[dev1] == group_props_after[group2].physicalDevices[dev2]) {
+                        found_count++;
+                        found = true;
+                        break;
+                    }
+                }
+            }
+            ASSERT_EQ(found, found_count != 0 && found_count == before_expected_counts[group1]);
+            if (found) {
+                break;
+            }
+        }
+    }
+}
+
+// Start with 6 devices in 3 different groups, and then remove a device to the middle group,
+// querying vkEnumeratePhysicalDeviceGroups before and after the remove.
+TEST_F(EnumeratePhysicalDeviceGroups, CallTwiceRemoveDeviceInBetween) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1);
+
+    // Generate the devices
+    for (size_t i = 0; i < 6; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+
+    // Generate the starting groups
+    driver.physical_device_groups.emplace_back(driver.physical_devices[0]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[1]);
+    driver.physical_device_groups.back()
+        .use_physical_device(driver.physical_devices[2])
+        .use_physical_device(driver.physical_devices[3]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[4]);
+    driver.physical_device_groups.back().use_physical_device(driver.physical_devices[5]);
+
+    uint32_t before_expected_counts[3] = {1, 3, 2};
+    uint32_t after_expected_counts[3] = {1, 2, 2};
+    uint32_t expected_group_count = 3;
+    uint32_t returned_group_count = 0;
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.create_info.set_api_version(1, 1, 0);
+    inst.CheckCreate();
+
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_before{};
+    group_props_before.resize(expected_group_count,
+                              VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    returned_group_count = expected_group_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_before.data()));
+    ASSERT_EQ(expected_group_count, returned_group_count);
+    printf("Before:\n");
+    for (uint32_t group = 0; group < expected_group_count; ++group) {
+        printf("  Group %u:\n", group);
+        ASSERT_EQ(group_props_before[group].physicalDeviceCount, before_expected_counts[group]);
+        for (uint32_t dev = 0; dev < group_props_before[group].physicalDeviceCount; ++dev) {
+            printf("    Dev %u: %p\n", dev, group_props_before[group].physicalDevices[dev]);
+        }
+    }
+
+    // Remove middle device in middle group
+    driver.physical_device_groups[1].physical_device_handles.erase(
+        driver.physical_device_groups[1].physical_device_handles.begin() + 1);
+
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_after{};
+    group_props_after.resize(expected_group_count,
+                             VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after.data()));
+    ASSERT_EQ(expected_group_count, returned_group_count);
+    printf("After:\n");
+    for (uint32_t group = 0; group < expected_group_count; ++group) {
+        printf("  Group %u:\n", group);
+        ASSERT_EQ(group_props_after[group].physicalDeviceCount, after_expected_counts[group]);
+        for (uint32_t dev = 0; dev < group_props_after[group].physicalDeviceCount; ++dev) {
+            printf("    Dev %u: %p\n", dev, group_props_after[group].physicalDevices[dev]);
+        }
+    }
+
+    // Make sure all devices in the new group info are found in the old group info
+    for (uint32_t group1 = 0; group1 < group_props_after.size(); ++group1) {
+        for (uint32_t group2 = 0; group2 < group_props_before.size(); ++group2) {
+            uint32_t found_count = 0;
+            bool found;
+            for (uint32_t dev1 = 0; dev1 < group_props_after[group1].physicalDeviceCount; ++dev1) {
+                found = false;
+                for (uint32_t dev2 = 0; dev2 < group_props_before[group2].physicalDeviceCount; ++dev2) {
+                    if (group_props_after[group1].physicalDevices[dev1] == group_props_before[group2].physicalDevices[dev2]) {
+                        found_count++;
+                        found = true;
+                        break;
+                    }
+                }
+            }
+            ASSERT_EQ(found, found_count != 0 && found_count == after_expected_counts[group1]);
+            if (found) {
+                break;
+            }
+        }
+    }
+}
+
+// Start with 9 devices but only some in 3 different groups, add and remove
+// various devices and groups while querying in between.
+TEST_F(EnumeratePhysicalDeviceGroups, MultipleAddRemoves) {
+    auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1);
+
+    // Generate the devices
+    for (size_t i = 0; i < 9; i++) {
+        driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i));
+    }
+
+    // Generate the starting groups
+    driver.physical_device_groups.emplace_back(driver.physical_devices[0]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[1]);
+    driver.physical_device_groups.back()
+        .use_physical_device(driver.physical_devices[2])
+        .use_physical_device(driver.physical_devices[3]);
+    driver.physical_device_groups.emplace_back(driver.physical_devices[4]);
+    driver.physical_device_groups.back().use_physical_device(driver.physical_devices[5]);
+
+    uint32_t before_expected_counts[3] = {1, 3, 2};
+    uint32_t after_add_group_expected_counts[4] = {1, 3, 1, 2};
+    uint32_t after_remove_dev_expected_counts[4] = {1, 2, 1, 2};
+    uint32_t after_remove_group_expected_counts[3] = {2, 1, 2};
+    uint32_t after_add_dev_expected_counts[3] = {2, 1, 4};
+    uint32_t before_group_count = 3;
+    uint32_t after_group_count = 4;
+    uint32_t returned_group_count = 0;
+
+    InstWrapper inst{env->vulkan_functions};
+    inst.create_info.set_api_version(1, 1, 0);
+    inst.CheckCreate();
+
+    // Should be: 3 Groups { { 0 }, { 1, 2, 3 }, { 4, 5 } }
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_before{};
+    group_props_before.resize(before_group_count,
+                              VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    returned_group_count = before_group_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_before.data()));
+    ASSERT_EQ(before_group_count, returned_group_count);
+    for (uint32_t group = 0; group < before_group_count; ++group) {
+        ASSERT_EQ(group_props_before[group].physicalDeviceCount, before_expected_counts[group]);
+    }
+
+    // Insert new group after first two
+    driver.physical_device_groups.insert(driver.physical_device_groups.begin() + 2, driver.physical_devices[6]);
+
+    // Should be: 4 Groups { { 0 }, { 1, 2, 3 }, { 6 }, { 4, 5 } }
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_after_add_group{};
+    group_props_after_add_group.resize(after_group_count,
+                                       VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    returned_group_count = after_group_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after_add_group.data()));
+    ASSERT_EQ(after_group_count, returned_group_count);
+    for (uint32_t group = 0; group < before_group_count; ++group) {
+        ASSERT_EQ(group_props_after_add_group[group].physicalDeviceCount, after_add_group_expected_counts[group]);
+    }
+
+    // Remove first device in 2nd group
+    driver.physical_device_groups[1].physical_device_handles.erase(
+        driver.physical_device_groups[1].physical_device_handles.begin());
+
+    // Should be: 4 Groups { { 0 }, { 2, 3 }, { 6 }, { 4, 5 } }
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_after_remove_device{};
+    group_props_after_remove_device.resize(after_group_count,
+                                           VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    returned_group_count = after_group_count;
+    ASSERT_EQ(VK_SUCCESS,
+              inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after_remove_device.data()));
+    ASSERT_EQ(after_group_count, returned_group_count);
+    for (uint32_t group = 0; group < before_group_count; ++group) {
+        ASSERT_EQ(group_props_after_remove_device[group].physicalDeviceCount, after_remove_dev_expected_counts[group]);
+    }
+
+    // Remove first group
+    driver.physical_device_groups.erase(driver.physical_device_groups.begin());
+
+    // Should be: 3 Groups { { 2, 3 }, { 6 }, { 4, 5 } }
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_after_remove_group{};
+    group_props_after_remove_group.resize(before_group_count,
+                                          VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    returned_group_count = before_group_count;
+    ASSERT_EQ(VK_SUCCESS,
+              inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after_remove_group.data()));
+    ASSERT_EQ(before_group_count, returned_group_count);
+    for (uint32_t group = 0; group < before_group_count; ++group) {
+        ASSERT_EQ(group_props_after_remove_group[group].physicalDeviceCount, after_remove_group_expected_counts[group]);
+    }
+
+    // Add two devices to last group
+    driver.physical_device_groups.back()
+        .use_physical_device(driver.physical_devices[7])
+        .use_physical_device(driver.physical_devices[8]);
+
+    // Should be: 3 Groups { { 2, 3 }, { 6 }, { 4, 5, 7, 8 } }
+    std::vector<VkPhysicalDeviceGroupProperties> group_props_after_add_device{};
+    group_props_after_add_device.resize(before_group_count,
+                                        VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    returned_group_count = before_group_count;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after_add_device.data()));
+    ASSERT_EQ(before_group_count, returned_group_count);
+    for (uint32_t group = 0; group < before_group_count; ++group) {
+        ASSERT_EQ(group_props_after_add_device[group].physicalDeviceCount, after_add_dev_expected_counts[group]);
+    }
+}
+
+// Fill in random but valid data into the device properties struct for the current physical device
+static void FillInRandomDeviceProps(VkPhysicalDeviceProperties& props, VkPhysicalDeviceType dev_type, uint32_t api_vers,
+                                    uint32_t vendor, uint32_t device) {
+    props.apiVersion = api_vers;
+    props.vendorID = vendor;
+    props.deviceID = device;
+    props.deviceType = dev_type;
+    for (uint8_t idx = 0; idx < VK_UUID_SIZE; ++idx) {
+        props.pipelineCacheUUID[idx] = static_cast<uint8_t>(rand() % 255);
+    }
+}
+
+// Pass in a PNext that the fake ICD will fill in some data for.
+TEST_F(EnumeratePhysicalDeviceGroups, FakePNext) {
+    FrameworkEnvironment env{};
+
+    // ICD 0: Vulkan 1.1
+    //   PhysDev 0: pd0, Discrete, Vulkan 1.1, Bus 7
+    //   PhysDev 1: pd1, Integrated, Vulkan 1.1, Bus 3
+    //   PhysDev 2: pd2, Discrete, Vulkan 1.1, Bus 6
+    //   Group 0: PhysDev 0, PhysDev 2
+    //   Group 1: PhysDev 1
+    // ICD 1: Vulkan 1.1
+    //   PhysDev 4: pd4, Discrete, Vulkan 1.1, Bus 1
+    //   PhysDev 5: pd5, Discrete, Vulkan 1.1, Bus 4
+    //   PhysDev 6: pd6, Discrete, Vulkan 1.1, Bus 2
+    //   Group 0: PhysDev 5, PhysDev 6
+    //   Group 1: PhysDev 4
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_6));
+    auto& cur_icd_0 = env.get_test_icd(0);
+    cur_icd_0.set_icd_api_version(VK_API_VERSION_1_1);
+    cur_icd_0.physical_devices.push_back({"pd0", 7});
+    cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+                            888, 0xAAA001);
+    cur_icd_0.physical_devices.push_back({"pd1", 3});
+    cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU,
+                            VK_API_VERSION_1_1, 888, 0xAAA002);
+    cur_icd_0.physical_devices.push_back({"pd2", 6});
+    cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+                            888, 0xAAA003);
+    cur_icd_0.physical_device_groups.push_back({});
+    cur_icd_0.physical_device_groups.back()
+        .use_physical_device(cur_icd_0.physical_devices[0])
+        .use_physical_device(cur_icd_0.physical_devices[2]);
+    cur_icd_0.physical_device_groups.push_back({cur_icd_0.physical_devices[1]});
+
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_6));
+    auto& cur_icd_1 = env.get_test_icd(1);
+    cur_icd_1.set_icd_api_version(VK_API_VERSION_1_1);
+    cur_icd_1.physical_devices.push_back({"pd4", 1});
+    cur_icd_1.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_1.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+                            75, 0xCCCC001);
+    cur_icd_1.physical_devices.push_back({"pd5", 4});
+    cur_icd_1.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_1.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+                            75, 0xCCCC002);
+    cur_icd_1.physical_devices.push_back({"pd6", 2});
+    cur_icd_1.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_1.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+                            75, 0xCCCC003);
+    cur_icd_1.physical_device_groups.push_back({});
+    cur_icd_1.physical_device_groups.back()
+        .use_physical_device(cur_icd_1.physical_devices[1])
+        .use_physical_device(cur_icd_1.physical_devices[2]);
+    cur_icd_1.physical_device_groups.push_back({cur_icd_1.physical_devices[0]});
+
+    InstWrapper inst(env.vulkan_functions);
+    inst.create_info.set_api_version(VK_API_VERSION_1_1);
+    inst.CheckCreate();
+
+    auto GetPhysDevProps2 = reinterpret_cast<PFN_vkGetPhysicalDeviceProperties2>(
+        inst.functions->vkGetInstanceProcAddr(inst, "vkGetPhysicalDeviceProperties2"));
+    ASSERT_NE(GetPhysDevProps2, nullptr);
+
+    // NOTE: This is a fake struct to make sure the pNext chain is properly passed down to the ICD
+    //       vkEnumeratePhysicalDeviceGroups.
+    //       The two versions must match:
+    //           "FakePNext" test in loader_regresion_tests.cpp
+    //           "test_vkEnumeratePhysicalDeviceGroups" in test_icd.cpp
+    struct FakePnextSharedWithICD {
+        VkStructureType sType;
+        void* pNext;
+        uint32_t value;
+    };
+
+    const uint32_t max_phys_dev_groups = 4;
+    uint32_t group_count = max_phys_dev_groups;
+    std::array<FakePnextSharedWithICD, max_phys_dev_groups> fake_structs;
+    std::array<VkPhysicalDeviceGroupProperties, max_phys_dev_groups> physical_device_groups{
+        VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES};
+    for (uint32_t group = 0; group < max_phys_dev_groups; ++group) {
+        fake_structs[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTI_DRAW_PROPERTIES_EXT;
+        physical_device_groups[group].pNext = &fake_structs[group];
+    }
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &group_count, physical_device_groups.data()));
+    ASSERT_EQ(group_count, max_phys_dev_groups);
+
+    // Value should get written to 0xDECAFBADD by the fake ICD
+    for (uint32_t group = 0; group < max_phys_dev_groups; ++group) {
+        ASSERT_EQ(fake_structs[group].value, 0xDECAFBAD);
     }
 }
 
@@ -1212,18 +2237,6 @@ TEST_F(CreateInstance, InstanceNullExtensionPtr) {
     ASSERT_EQ(env->vulkan_functions.vkCreateInstance(&info, VK_NULL_HANDLE, &inst), VK_ERROR_EXTENSION_NOT_PRESENT);
 }
 
-// Fill in random but valid data into the device properties struct for the current physical device
-static void FillInRandomDeviceProps(VkPhysicalDeviceProperties& props, VkPhysicalDeviceType dev_type, uint32_t api_vers,
-                                    uint32_t vendor, uint32_t device) {
-    props.apiVersion = api_vers;
-    props.vendorID = vendor;
-    props.deviceID = device;
-    props.deviceType = dev_type;
-    for (uint8_t idx = 0; idx < VK_UUID_SIZE; ++idx) {
-        props.pipelineCacheUUID[idx] = static_cast<uint8_t>(rand() % 255);
-    }
-}
-
 #if defined(__linux__) || defined(__FreeBSD__)
 // NOTE: Sort order only affects Linux
 TEST(SortedPhysicalDevices, DevicesSortEnabled) {
@@ -1436,6 +2449,9 @@ TEST(SortedPhysicalDevices, DevicesSortedDisabled) {
             default:
                 ASSERT_EQ(false, true);
         }
+        if (!sorted) {
+            break;
+        }
     }
     ASSERT_EQ(false, sorted);
 
@@ -1450,61 +2466,81 @@ TEST(SortedPhysicalDevices, DevicesSortedDisabled) {
     remove_env_var("VK_LOADER_DISABLE_SELECT");
 }
 
-#if 0  // TODO: Enable check on physical device group sorting to make sure proper order returned.
-       //       This is disabled because the test framework needs a little more work for this.
 TEST(SortedPhysicalDevices, DeviceGroupsSortedEnabled) {
     FrameworkEnvironment env{};
 
+    // ICD 0: Vulkan 1.1
+    //   PhysDev 0: pd0, Discrete, Vulkan 1.1, Bus 7
+    //   PhysDev 1: pd1, Integrated, Vulkan 1.1, Bus 3
+    //   PhysDev 2: pd2, Discrete, Vulkan 1.1, Bus 6
+    //   Group 0: PhysDev 0, PhysDev 2
+    //   Group 1: PhysDev 1
+    // ICD 1: Vulkan 1.1
+    //   PhysDev 3: pd3, CPU, Vulkan 1.1, Bus 0
+    // ICD 2: Vulkan 1.1
+    //   PhysDev 4: pd4, Discrete, Vulkan 1.1, Bus 1
+    //   PhysDev 5: pd5, Discrete, Vulkan 1.1, Bus 4
+    //   PhysDev 6: pd6, Discrete, Vulkan 1.1, Bus 2
+    //   Group 0: PhysDev 5, PhysDev 6
+    //   Group 1: PhysDev 4
+    // ICD 3: Vulkan 1.1
+    //   PhysDev 7: pd7, Virtual, Vulkan 1.1, Bus 0
     env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1));
     auto& cur_icd_0 = env.get_test_icd(0);
     cur_icd_0.set_icd_api_version(VK_API_VERSION_1_1);
     cur_icd_0.physical_devices.push_back({"pd0", 7});
+    cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
     FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
                             888, 0xAAA001);
     cur_icd_0.physical_devices.push_back({"pd1", 3});
+    cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
     FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU,
                             VK_API_VERSION_1_1, 888, 0xAAA002);
     cur_icd_0.physical_devices.push_back({"pd2", 6});
+    cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
     FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
                             888, 0xAAA003);
     cur_icd_0.physical_device_groups.push_back({});
     cur_icd_0.physical_device_groups.back()
         .use_physical_device(cur_icd_0.physical_devices[0])
         .use_physical_device(cur_icd_0.physical_devices[2]);
-    cur_icd_0.physical_device_groups.push_back({});
-    cur_icd_0.physical_device_groups.back().use_physical_device(cur_icd_0.physical_devices[1]);
+    cur_icd_0.physical_device_groups.push_back({cur_icd_0.physical_devices[1]});
 
     env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1));
     auto& cur_icd_1 = env.get_test_icd(1);
     cur_icd_1.set_icd_api_version(VK_API_VERSION_1_1);
     cur_icd_1.physical_devices.push_back({"pd3", 0});
-    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_CPU, VK_API_VERSION_1_1, 1,
+    cur_icd_1.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_1.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_CPU, VK_API_VERSION_1_1, 1,
                             0xBBBB001);
 
     env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1));
     auto& cur_icd_2 = env.get_test_icd(2);
     cur_icd_2.set_icd_api_version(VK_API_VERSION_1_1);
     cur_icd_2.physical_devices.push_back({"pd4", 1});
-    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+    cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
                             75, 0xCCCC001);
     cur_icd_2.physical_devices.push_back({"pd5", 4});
-    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+    cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
                             75, 0xCCCC002);
     cur_icd_2.physical_devices.push_back({"pd6", 2});
-    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+    cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
                             75, 0xCCCC003);
     cur_icd_2.physical_device_groups.push_back({});
     cur_icd_2.physical_device_groups.back()
         .use_physical_device(cur_icd_2.physical_devices[1])
         .use_physical_device(cur_icd_2.physical_devices[2]);
-    cur_icd_2.physical_device_groups.push_back({});
-    cur_icd_2.physical_device_groups.back().use_physical_device(cur_icd_2.physical_devices[0]);
+    cur_icd_2.physical_device_groups.push_back({cur_icd_2.physical_devices[0]});
 
     env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1));
     auto& cur_icd_3 = env.get_test_icd(3);
     cur_icd_3.set_icd_api_version(VK_API_VERSION_1_1);
     cur_icd_3.physical_devices.push_back({"pd7", 0});
-    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU, VK_API_VERSION_1_1,
+    cur_icd_3.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_3.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU, VK_API_VERSION_1_1,
                             6940, 0xDDDD001);
 
     InstWrapper inst(env.vulkan_functions);
@@ -1521,7 +2557,7 @@ TEST(SortedPhysicalDevices, DeviceGroupsSortedEnabled) {
     ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &device_count, physical_devices.data()));
     ASSERT_EQ(device_count, max_phys_devs);
 
-    const uint32_t max_phys_dev_groups = 8;
+    const uint32_t max_phys_dev_groups = 6;
     uint32_t group_count = max_phys_dev_groups;
     std::array<VkPhysicalDeviceGroupProperties, max_phys_dev_groups> physical_device_groups{
         VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES};
@@ -1532,71 +2568,259 @@ TEST(SortedPhysicalDevices, DeviceGroupsSortedEnabled) {
     for (uint32_t group = 0; group < max_phys_dev_groups; ++group) {
         for (uint32_t dev = 0; dev < physical_device_groups[group].physicalDeviceCount; ++dev) {
             VkPhysicalDeviceProperties props{};
-            inst->vkGetPhysicalDeviceProperties(physical_devices[dev], &props);
+            inst->vkGetPhysicalDeviceProperties(physical_device_groups[group].physicalDevices[dev], &props);
             VkPhysicalDeviceProperties2 props2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2};
             VkPhysicalDevicePCIBusInfoPropertiesEXT pci_bus_info{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT};
             props2.pNext = &pci_bus_info;
-            GetPhysDevProps2(physical_devices[dev], &props2);
-            /*
-                        switch (++cur_dev) {
-                            case 0:
-                                ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU);
-                                ASSERT_EQ(true, !strcmp("pd3", props.deviceName));
-                                ASSERT_EQ(props.vendorID, 75);
-                                ASSERT_EQ(props.deviceID, 0xCCCC001);
-                                ASSERT_EQ(pci_bus_info.pciBus, 1);
-                                break;
-                            case 1:
-                                ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU);
-                                ASSERT_EQ(true, !strcmp("pd4", props.deviceName));
-                                ASSERT_EQ(props.vendorID, 75);
-                                ASSERT_EQ(props.deviceID, 0xCCCC002);
-                                ASSERT_EQ(pci_bus_info.pciBus, 4);
-                                break;
-                            case 2:
-                                ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU);
-                                ASSERT_EQ(true, !strcmp("pd0", props.deviceName));
-                                ASSERT_EQ(props.vendorID, 888);
-                                ASSERT_EQ(props.deviceID, 0xAAA001);
-                                ASSERT_EQ(pci_bus_info.pciBus, 7);
-                                break;
-                            case 3:
-                                ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU);
-                                ASSERT_EQ(true, !strcmp("pd1", props.deviceName));
-                                ASSERT_EQ(props.vendorID, 888);
-                                ASSERT_EQ(props.deviceID, 0xAAA002);
-                                ASSERT_EQ(pci_bus_info.pciBus, 3);
-                                break;
-                            case 4:
-                                ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU);
-                                ASSERT_EQ(true, !strcmp("pd5", props.deviceName));
-                                ASSERT_EQ(props.vendorID, 6940);
-                                ASSERT_EQ(props.deviceID, 0xDDDD001);
-                                ASSERT_EQ(pci_bus_info.pciBus, 0);
-                                break;
-                            case 5:
-                                ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_CPU);
-                                ASSERT_EQ(true, !strcmp("pd2", props.deviceName));
-                                ASSERT_EQ(props.vendorID, 1);
-                                ASSERT_EQ(props.deviceID, 0xBBBB001);
-                                ASSERT_EQ(pci_bus_info.pciBus, 0);
-                                break;
-                            default:
-                                ASSERT_EQ(false, true);
-                        }
-            */
+            GetPhysDevProps2(physical_device_groups[group].physicalDevices[dev], &props2);
+            switch (cur_dev++) {
+                case 0:
+                    ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU);
+                    ASSERT_EQ(true, !strcmp("pd4", props.deviceName));
+                    ASSERT_EQ(props.vendorID, 75);
+                    ASSERT_EQ(props.deviceID, 0xCCCC001);
+                    ASSERT_EQ(pci_bus_info.pciBus, 1);
+                    break;
+                case 1:
+                    ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU);
+                    ASSERT_EQ(true, !strcmp("pd6", props.deviceName));
+                    ASSERT_EQ(props.vendorID, 75);
+                    ASSERT_EQ(props.deviceID, 0xCCCC003);
+                    ASSERT_EQ(pci_bus_info.pciBus, 2);
+                    break;
+                case 2:
+                    ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU);
+                    ASSERT_EQ(true, !strcmp("pd5", props.deviceName));
+                    ASSERT_EQ(props.vendorID, 75);
+                    ASSERT_EQ(props.deviceID, 0xCCCC002);
+                    ASSERT_EQ(pci_bus_info.pciBus, 4);
+                    break;
+                case 3:
+                    ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU);
+                    ASSERT_EQ(true, !strcmp("pd2", props.deviceName));
+                    ASSERT_EQ(props.vendorID, 888);
+                    ASSERT_EQ(props.deviceID, 0xAAA003);
+                    ASSERT_EQ(pci_bus_info.pciBus, 6);
+                    break;
+                case 4:
+                    ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU);
+                    ASSERT_EQ(true, !strcmp("pd0", props.deviceName));
+                    ASSERT_EQ(props.vendorID, 888);
+                    ASSERT_EQ(props.deviceID, 0xAAA001);
+                    ASSERT_EQ(pci_bus_info.pciBus, 7);
+                    break;
+                case 5:
+                    ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU);
+                    ASSERT_EQ(true, !strcmp("pd1", props.deviceName));
+                    ASSERT_EQ(props.vendorID, 888);
+                    ASSERT_EQ(props.deviceID, 0xAAA002);
+                    ASSERT_EQ(pci_bus_info.pciBus, 3);
+                    break;
+                case 6:
+                    ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU);
+                    ASSERT_EQ(true, !strcmp("pd7", props.deviceName));
+                    ASSERT_EQ(props.vendorID, 6940);
+                    ASSERT_EQ(props.deviceID, 0xDDDD001);
+                    ASSERT_EQ(pci_bus_info.pciBus, 0);
+                    break;
+                case 7:
+                    ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_CPU);
+                    ASSERT_EQ(true, !strcmp("pd3", props.deviceName));
+                    ASSERT_EQ(props.vendorID, 1);
+                    ASSERT_EQ(props.deviceID, 0xBBBB001);
+                    ASSERT_EQ(pci_bus_info.pciBus, 0);
+                    break;
+                default:
+                    ASSERT_EQ(false, true);
+            }
         }
     }
 
+    // Make sure if we call enumerate again, the information is the same
     std::array<VkPhysicalDeviceGroupProperties, max_phys_dev_groups> physical_device_groups_again{
         VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES};
     ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &group_count, physical_device_groups_again.data()));
     ASSERT_EQ(group_count, max_phys_dev_groups);
     for (uint32_t group = 0; group < max_phys_dev_groups; ++group) {
+        ASSERT_EQ(physical_device_groups[group].physicalDeviceCount, physical_device_groups_again[group].physicalDeviceCount);
         for (uint32_t dev = 0; dev < physical_device_groups[group].physicalDeviceCount; ++dev) {
             ASSERT_EQ(physical_device_groups[group].physicalDevices[dev], physical_device_groups_again[group].physicalDevices[dev]);
         }
     }
 }
-#endif
-#endif  // __linux__ || __FreeBSD__
\ No newline at end of file
+
+TEST(SortedPhysicalDevices, DeviceGroupsSortedDisabled) {
+    FrameworkEnvironment env{};
+
+    set_env_var("VK_LOADER_DISABLE_SELECT", "1");
+
+    // ICD 0: Vulkan 1.1
+    //   PhysDev 0: pd0, Discrete, Vulkan 1.1, Bus 7
+    //   PhysDev 1: pd1, Integrated, Vulkan 1.1, Bus 3
+    //   PhysDev 2: pd2, Discrete, Vulkan 1.1, Bus 6
+    //   Group 0: PhysDev 0, PhysDev 2
+    //   Group 1: PhysDev 1
+    // ICD 1: Vulkan 1.1
+    //   PhysDev 3: pd3, CPU, Vulkan 1.1, Bus 0
+    // ICD 2: Vulkan 1.1
+    //   PhysDev 4: pd4, Discrete, Vulkan 1.1, Bus 1
+    //   PhysDev 5: pd5, Discrete, Vulkan 1.1, Bus 4
+    //   PhysDev 6: pd6, Discrete, Vulkan 1.1, Bus 2
+    //   Group 0: PhysDev 5, PhysDev 6
+    //   Group 1: PhysDev 4
+    // ICD 3: Vulkan 1.1
+    //   PhysDev 7: pd7, Virtual, Vulkan 1.1, Bus 0
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1));
+    auto& cur_icd_0 = env.get_test_icd(0);
+    cur_icd_0.set_icd_api_version(VK_API_VERSION_1_1);
+    cur_icd_0.physical_devices.push_back({"pd0", 7});
+    cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+                            888, 0xAAA001);
+    cur_icd_0.physical_devices.push_back({"pd1", 3});
+    cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU,
+                            VK_API_VERSION_1_1, 888, 0xAAA002);
+    cur_icd_0.physical_devices.push_back({"pd2", 6});
+    cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+                            888, 0xAAA003);
+    cur_icd_0.physical_device_groups.push_back({});
+    cur_icd_0.physical_device_groups.back()
+        .use_physical_device(cur_icd_0.physical_devices[0])
+        .use_physical_device(cur_icd_0.physical_devices[2]);
+    cur_icd_0.physical_device_groups.push_back({cur_icd_0.physical_devices[1]});
+
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1));
+    auto& cur_icd_1 = env.get_test_icd(1);
+    cur_icd_1.set_icd_api_version(VK_API_VERSION_1_1);
+    cur_icd_1.physical_devices.push_back({"pd3", 0});
+    cur_icd_1.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_1.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_CPU, VK_API_VERSION_1_1, 1,
+                            0xBBBB001);
+
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1));
+    auto& cur_icd_2 = env.get_test_icd(2);
+    cur_icd_2.set_icd_api_version(VK_API_VERSION_1_1);
+    cur_icd_2.physical_devices.push_back({"pd4", 1});
+    cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+                            75, 0xCCCC001);
+    cur_icd_2.physical_devices.push_back({"pd5", 4});
+    cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+                            75, 0xCCCC002);
+    cur_icd_2.physical_devices.push_back({"pd6", 2});
+    cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1,
+                            75, 0xCCCC003);
+    cur_icd_2.physical_device_groups.push_back({});
+    cur_icd_2.physical_device_groups.back()
+        .use_physical_device(cur_icd_2.physical_devices[1])
+        .use_physical_device(cur_icd_2.physical_devices[2]);
+    cur_icd_2.physical_device_groups.push_back({cur_icd_2.physical_devices[0]});
+
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1));
+    auto& cur_icd_3 = env.get_test_icd(3);
+    cur_icd_3.set_icd_api_version(VK_API_VERSION_1_1);
+    cur_icd_3.physical_devices.push_back({"pd7", 0});
+    cur_icd_3.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0});
+    FillInRandomDeviceProps(cur_icd_3.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU, VK_API_VERSION_1_1,
+                            6940, 0xDDDD001);
+
+    InstWrapper inst(env.vulkan_functions);
+    inst.create_info.set_api_version(VK_API_VERSION_1_1);
+    inst.CheckCreate();
+
+    auto GetPhysDevProps2 = reinterpret_cast<PFN_vkGetPhysicalDeviceProperties2>(
+        inst.functions->vkGetInstanceProcAddr(inst, "vkGetPhysicalDeviceProperties2"));
+    ASSERT_NE(GetPhysDevProps2, nullptr);
+
+    const uint32_t max_phys_devs = 8;
+    uint32_t device_count = max_phys_devs;
+    std::array<VkPhysicalDevice, max_phys_devs> physical_devices;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &device_count, physical_devices.data()));
+    ASSERT_EQ(device_count, max_phys_devs);
+
+    const uint32_t max_phys_dev_groups = 6;
+    uint32_t group_count = max_phys_dev_groups;
+    std::array<VkPhysicalDeviceGroupProperties, max_phys_dev_groups> physical_device_groups{
+        VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES};
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &group_count, physical_device_groups.data()));
+    ASSERT_EQ(group_count, max_phys_dev_groups);
+
+    // Make sure the devices are not in the sorted order.  The order is really undefined, but the chances of
+    // it being exactly the expected sorted is very low.
+    bool sorted = true;
+    uint32_t cur_dev = 0;
+    for (uint32_t group = 0; group < max_phys_dev_groups; ++group) {
+        for (uint32_t dev = 0; dev < physical_device_groups[group].physicalDeviceCount; ++dev) {
+            VkPhysicalDeviceProperties props{};
+            inst->vkGetPhysicalDeviceProperties(physical_device_groups[group].physicalDevices[dev], &props);
+            switch (cur_dev++) {
+                case 0:
+                    if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || strcmp("pd4", props.deviceName)) {
+                        sorted = false;
+                    }
+                    break;
+                case 1:
+                    if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || strcmp("pd6", props.deviceName)) {
+                        sorted = false;
+                    }
+                    break;
+                case 2:
+                    if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || strcmp("pd5", props.deviceName)) {
+                        sorted = false;
+                    }
+                    break;
+                case 3:
+                    if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || strcmp("pd2", props.deviceName)) {
+                        sorted = false;
+                    }
+                    break;
+                case 4:
+                    if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || strcmp("pd0", props.deviceName)) {
+                        sorted = false;
+                    }
+                    break;
+                case 5:
+                    if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU || strcmp("pd1", props.deviceName)) {
+                        sorted = false;
+                    }
+                    break;
+                case 6:
+                    if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU || strcmp("pd7", props.deviceName)) {
+                        sorted = false;
+                    }
+                    break;
+                case 7:
+                    if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_CPU || strcmp("pd3", props.deviceName)) {
+                        sorted = false;
+                    }
+                    break;
+                default:
+                    ASSERT_EQ(false, true);
+            }
+        }
+        if (!sorted) {
+            break;
+        }
+    }
+    ASSERT_EQ(false, sorted);
+
+    // Make sure if we call enumerate again, the information is the same
+    std::array<VkPhysicalDeviceGroupProperties, max_phys_dev_groups> physical_device_groups_again{
+        VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES};
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &group_count, physical_device_groups_again.data()));
+    ASSERT_EQ(group_count, max_phys_dev_groups);
+    for (uint32_t group = 0; group < max_phys_dev_groups; ++group) {
+        ASSERT_EQ(physical_device_groups[group].physicalDeviceCount, physical_device_groups_again[group].physicalDeviceCount);
+        for (uint32_t dev = 0; dev < physical_device_groups[group].physicalDeviceCount; ++dev) {
+            ASSERT_EQ(physical_device_groups[group].physicalDevices[dev], physical_device_groups_again[group].physicalDevices[dev]);
+        }
+    }
+
+    remove_env_var("VK_LOADER_DISABLE_SELECT");
+}
+
+#endif  // __linux__ || __FreeBSD__
index d89a907820fec8aa269327b71b4eaa24e2e10d77..a4fd5cf71a0275dbfc2dfb2287981adf96d53eb2 100644 (file)
@@ -320,6 +320,94 @@ TEST(MultipleDriverConfig, DifferentICDInterfaceVersions) {
     ASSERT_EQ(env.vulkan_functions.vkEnumeratePhysicalDevices(inst, &phys_dev_count, phys_devs_array.data()), VK_SUCCESS);
     ASSERT_EQ(phys_dev_count, 2);
 }
+
+TEST(MultipleDriverConfig, DifferentICDsWithDevices) {
+    FrameworkEnvironment env{};
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_EXPORT_ICD_GIPA));
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2));
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2_EXPORT_ICD_GPDPA));
+
+    // Make sure the loader returns all devices from all active ICDs.  Many of the other
+    // tests add multiple devices to a single ICD, this just makes sure the loader combines
+    // device info across multiple drivers properly.
+    TestICD& icd0 = env.get_test_icd(0);
+    icd0.physical_devices.emplace_back("physical_device_0");
+    icd0.min_icd_interface_version = 5;
+    icd0.max_icd_interface_version = 5;
+
+    TestICD& icd1 = env.get_test_icd(1);
+    icd1.physical_devices.emplace_back("physical_device_1");
+    icd1.physical_devices.emplace_back("physical_device_2");
+    icd1.min_icd_interface_version = 5;
+    icd1.max_icd_interface_version = 5;
+
+    TestICD& icd2 = env.get_test_icd(2);
+    icd2.physical_devices.emplace_back("physical_device_3");
+    icd2.min_icd_interface_version = 5;
+    icd2.max_icd_interface_version = 5;
+
+    InstWrapper inst{env.vulkan_functions};
+    inst.CheckCreate();
+
+    std::array<VkPhysicalDevice, 4> phys_devs_array;
+    uint32_t phys_dev_count = 4;
+    ASSERT_EQ(env.vulkan_functions.vkEnumeratePhysicalDevices(inst, &phys_dev_count, phys_devs_array.data()), VK_SUCCESS);
+    ASSERT_EQ(phys_dev_count, 4);
+}
+
+TEST(MultipleDriverConfig, DifferentICDsWithDevicesAndGroups) {
+    FrameworkEnvironment env{};
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_EXPORT_ICD_GIPA));
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2));
+    env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2_EXPORT_ICD_GPDPA));
+
+    // The loader has to be able to handle drivers that support device groups in combination
+    // with drivers that don't support device groups.  When this is the case, the loader needs
+    // to take every driver that doesn't support device groups and put each of its devices in
+    // a separate group.  Then it combines that information with the drivers that support
+    // device groups returned info.
+
+    // ICD 0 :  No 1.1 support (so 1 device will become 1 group in loader)
+    TestICD& icd0 = env.get_test_icd(0);
+    icd0.physical_devices.emplace_back("physical_device_0");
+    icd0.min_icd_interface_version = 5;
+    icd0.max_icd_interface_version = 5;
+    icd0.set_icd_api_version(VK_API_VERSION_1_0);
+
+    // ICD 1 :  1.1 support (with 1 group with 2 devices)
+    TestICD& icd1 = env.get_test_icd(1);
+    icd1.physical_devices.emplace_back("physical_device_1");
+    icd1.physical_devices.emplace_back("physical_device_2");
+    icd1.physical_device_groups.emplace_back(icd1.physical_devices[0]);
+    icd1.physical_device_groups.back().use_physical_device(icd1.physical_devices[1]);
+    icd1.min_icd_interface_version = 5;
+    icd1.max_icd_interface_version = 5;
+    icd1.set_icd_api_version(VK_API_VERSION_1_1);
+
+    // ICD 2 :  No 1.1 support (so 3 devices will become 3 groups in loader)
+    TestICD& icd2 = env.get_test_icd(2);
+    icd2.physical_devices.emplace_back("physical_device_3");
+    icd2.physical_devices.emplace_back("physical_device_4");
+    icd2.physical_devices.emplace_back("physical_device_5");
+    icd2.min_icd_interface_version = 5;
+    icd2.max_icd_interface_version = 5;
+    icd2.set_icd_api_version(VK_API_VERSION_1_0);
+
+    InstWrapper inst{env.vulkan_functions};
+    inst.create_info.set_api_version(1, 1, 0);
+    inst.CheckCreate();
+
+    uint32_t group_count = static_cast<uint32_t>(5);
+    uint32_t returned_group_count = 0;
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, nullptr));
+    ASSERT_EQ(group_count, returned_group_count);
+
+    std::vector<VkPhysicalDeviceGroupProperties> group_props{};
+    group_props.resize(group_count, VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES});
+    ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props.data()));
+    ASSERT_EQ(group_count, returned_group_count);
+}
+
 // shim function pointers for 1.3
 // Should use autogen for this - it generates 'shim' functions for validation layers, maybe that could be used here.
 void test_vkCmdBeginRendering(VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo* pRenderPassBegin,
@@ -668,4 +756,4 @@ TEST(LayerManifest, ExplicitNonVulkanVariant) {
     inst.CheckCreate(VK_ERROR_LAYER_NOT_PRESENT);
     ASSERT_TRUE(log.find(std::string("Layer ") + explicit_layer_name +
                          " has an \'api_version\' field which contains a non-zero variant value of 1.  Skipping Layer."));
-}
\ No newline at end of file
+}