From: Mark Young Date: Thu, 2 Dec 2021 23:55:13 +0000 (-0700) Subject: Loader single EnumPhysDev call through layers X-Git-Tag: upstream/v1.3.207~17 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=0a19663fef75d5ccada5c57d5ce60cb3f448299d;p=platform%2Fupstream%2FVulkan-Loader.git Loader single EnumPhysDev call through layers The loader trampoline previously would query all devices every time vkEnumeratePhysicalDevices was called. To do this, it would make two calls every time: - First, it would ignore the passed in user values - Second, it would query the total number of available devices. - Third, it would query the values for every available device This resulted in layers reporting 2 vkEnumeratePhysicalDevices call for every 1 the application made which could get very polluted in output. It didn't break any functionality, just made things messy. This change removes that behavior and adds a bunch of test cases to verify nothing broke in the move. --- diff --git a/loader/loader.c b/loader/loader.c index 3e563a0f..38df6b37 100644 --- a/loader/loader.c +++ b/loader/loader.c @@ -6033,85 +6033,132 @@ out: return res; } -VkResult setup_loader_tramp_phys_devs(struct loader_instance *inst) { +// Update the trampoline physical devices with the wrapped version. +// We always want to re-use previous physical device pointers since they may be used by an application +// after returning previously. +VkResult setup_loader_tramp_phys_devs(struct loader_instance *inst, uint32_t phys_dev_count, VkPhysicalDevice *phys_devs) { VkResult res = VK_SUCCESS; - VkPhysicalDevice *local_phys_devs = NULL; - uint32_t total_count = 0; + uint32_t cur_idx; + uint32_t new_idx; + uint32_t found_count = 0; + uint32_t old_count = inst->phys_dev_count_tramp; + uint32_t new_count = inst->total_gpu_count; struct loader_physical_device_tramp **new_phys_devs = NULL; - // Query how many GPUs there - res = inst->disp->layer_inst_disp.EnumeratePhysicalDevices(inst->instance, &total_count, NULL); - if (res != VK_SUCCESS) { - loader_log( - inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_tramp_phys_devs: Failed during dispatch call of \'vkEnumeratePhysicalDevices\' to lower layers or " - "loader to get count."); - goto out; + if (0 == phys_dev_count) { + return VK_SUCCESS; } - - // Really use what the total GPU count is since Optimus and other layers may mess - // the count up. - total_count = inst->total_gpu_count; - - // Create an array for the new physical devices, which will be stored - // in the instance for the trampoline code. - new_phys_devs = (struct loader_physical_device_tramp **)loader_instance_heap_alloc( - inst, total_count * sizeof(struct loader_physical_device_tramp *), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); - if (NULL == new_phys_devs) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_tramp_phys_devs: Failed to allocate new physical device array of size %d", total_count); - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; + if (phys_dev_count > new_count) { + new_count = phys_dev_count; } - memset(new_phys_devs, 0, total_count * sizeof(struct loader_physical_device_tramp *)); - // Create a temporary array (on the stack) to keep track of the - // returned VkPhysicalDevice values. - local_phys_devs = loader_stack_alloc(sizeof(VkPhysicalDevice) * total_count); - if (NULL == local_phys_devs) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_tramp_phys_devs: Failed to allocate local physical device array of size %d", total_count); - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; + // We want an old to new index array and a new to old index array + int32_t *old_to_new_index = (int32_t *)loader_stack_alloc(sizeof(int32_t) * old_count); + int32_t *new_to_old_index = (int32_t *)loader_stack_alloc(sizeof(int32_t) * new_count); + if (NULL == old_to_new_index || NULL == new_to_old_index) { + return VK_ERROR_OUT_OF_HOST_MEMORY; } - memset(local_phys_devs, 0, sizeof(VkPhysicalDevice) * total_count); - res = inst->disp->layer_inst_disp.EnumeratePhysicalDevices(inst->instance, &total_count, local_phys_devs); - if (VK_SUCCESS != res) { - loader_log( - inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_tramp_phys_devs: Failed during dispatch call of \'vkEnumeratePhysicalDevices\' to lower layers or " - "loader to get content."); - goto out; + // Initialize both + for (cur_idx = 0; cur_idx < old_count; ++cur_idx) { + old_to_new_index[cur_idx] = -1; + } + for (cur_idx = 0; cur_idx < new_count; ++cur_idx) { + new_to_old_index[cur_idx] = -1; } - // Copy or create everything to fill the new array of physical devices - for (uint32_t new_idx = 0; new_idx < total_count; new_idx++) { - // Check if this physical device is already in the old buffer - for (uint32_t old_idx = 0; old_idx < inst->phys_dev_count_tramp; old_idx++) { - if (local_phys_devs[new_idx] == inst->phys_devs_tramp[old_idx]->phys_dev) { - new_phys_devs[new_idx] = inst->phys_devs_tramp[old_idx]; + // Figure out the old->new and new->old indices + for (cur_idx = 0; cur_idx < old_count; ++cur_idx) { + for (new_idx = 0; new_idx < phys_dev_count; ++new_idx) { + if (inst->phys_devs_tramp[cur_idx]->phys_dev == phys_devs[new_idx]) { + old_to_new_index[cur_idx] = (int32_t)new_idx; + new_to_old_index[new_idx] = (int32_t)cur_idx; + found_count++; break; } } + } - // If this physical device isn't in the old buffer, create it - if (NULL == new_phys_devs[new_idx]) { - new_phys_devs[new_idx] = (struct loader_physical_device_tramp *)loader_instance_heap_alloc( - inst, sizeof(struct loader_physical_device_tramp), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); - if (NULL == new_phys_devs[new_idx]) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_tramp_phys_devs: Failed to allocate physical device trampoline object %d", new_idx); - total_count = new_idx; - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; + // If we found exactly the number of items we were looking for as we had before. Then everything + // we already have is good enough and we just need to update the array that was passed in with + // the loader values. + if (found_count == phys_dev_count && 0 != old_count && old_count == new_count) { + for (new_idx = 0; new_idx < phys_dev_count; ++new_idx) { + for (cur_idx = 0; cur_idx < old_count; ++cur_idx) { + if (old_to_new_index[cur_idx] == (int32_t)new_idx) { + phys_devs[new_idx] = (VkPhysicalDevice)inst->phys_devs_tramp[cur_idx]; + break; + } + } + } + // Nothing else to do for this path + return VK_SUCCESS; + } else { + // Something is different, so do the full path of checking every device and creating a new array to use. + // This can happen if a device was added, or removed, or we hadn't previously queried all the data and we + // have more to store. + new_phys_devs = loader_instance_heap_alloc(inst, sizeof(struct loader_physical_device_tramp *) * new_count, + VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); + if (NULL == new_phys_devs) { + loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, + "setup_loader_tramp_phys_devs: Failed to allocate new physical device array of size %d", new_count); + res = VK_ERROR_OUT_OF_HOST_MEMORY; + goto out; + } + memset(new_phys_devs, 0, sizeof(struct loader_physical_device_tramp *) * new_count); + + if (new_count > phys_dev_count) { + found_count = phys_dev_count; + } else { + found_count = new_count; + } + + // First try to see if an old item exists that matches the new item. If so, just copy it over. + for (new_idx = 0; new_idx < found_count; ++new_idx) { + bool old_item_found = false; + for (cur_idx = 0; cur_idx < old_count; ++cur_idx) { + if (old_to_new_index[cur_idx] == (int32_t)new_idx) { + // Copy over old item to correct spot in the new array + new_phys_devs[new_idx] = inst->phys_devs_tramp[cur_idx]; + inst->phys_devs_tramp[cur_idx] = NULL; + old_item_found = true; + break; + } + } + // Something wasn't found, so it's new so add it to the new list + if (!old_item_found) { + new_phys_devs[new_idx] = loader_instance_heap_alloc(inst, sizeof(struct loader_physical_device_tramp), + VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); + if (NULL == new_phys_devs[new_idx]) { + loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, + "setup_loader_tramp_phys_devs: Failed to allocate new trampoline physical device"); + res = VK_ERROR_OUT_OF_HOST_MEMORY; + goto out; + } + + // Initialize the new physicalDevice object + loader_set_dispatch((void *)new_phys_devs[new_idx], inst->disp); + new_phys_devs[new_idx]->this_instance = inst; + new_phys_devs[new_idx]->phys_dev = phys_devs[new_idx]; + new_phys_devs[new_idx]->magic = PHYS_TRAMP_MAGIC_NUMBER; } - // Initialize the new physicalDevice object - loader_set_dispatch((void *)new_phys_devs[new_idx], inst->disp); - new_phys_devs[new_idx]->this_instance = inst; - new_phys_devs[new_idx]->phys_dev = local_phys_devs[new_idx]; - new_phys_devs[new_idx]->magic = PHYS_TRAMP_MAGIC_NUMBER; + phys_devs[new_idx] = (VkPhysicalDevice)new_phys_devs[new_idx]; + } + + // We usually get here if the user array is smaller than the total number of devices, so copy the + // remaining devices we have over to the new array. + uint32_t start = found_count; + for (new_idx = start; new_idx < new_count; ++new_idx) { + for (cur_idx = 0; cur_idx < old_count; ++cur_idx) { + if (old_to_new_index[cur_idx] == -1) { + new_phys_devs[new_idx] = inst->phys_devs_tramp[cur_idx]; + inst->phys_devs_tramp[cur_idx] = NULL; + old_to_new_index[cur_idx] = new_idx; + found_count++; + break; + } + } } } @@ -6119,47 +6166,41 @@ out: if (VK_SUCCESS != res) { if (NULL != new_phys_devs) { - for (uint32_t i = 0; i < total_count; i++) { + for (new_idx = 0; new_idx < found_count; ++new_idx) { // If an OOM occurred inside the copying of the new physical devices into the existing array // will leave some of the old physical devices in the array which may have been copied into // the new array, leading to them being freed twice. To avoid this we just make sure to not // delete physical devices which were copied. bool found = false; - for (uint32_t old_idx = 0; old_idx < inst->phys_dev_count_tramp; old_idx++) { - if (new_phys_devs[i] == inst->phys_devs_tramp[old_idx]) { + for (cur_idx = 0; cur_idx < inst->phys_dev_count_tramp; cur_idx++) { + if (new_phys_devs[new_idx] == inst->phys_devs_tramp[cur_idx]) { found = true; break; } } if (!found) { - loader_instance_heap_free(inst, new_phys_devs[i]); + loader_instance_heap_free(inst, new_phys_devs[new_idx]); } } loader_instance_heap_free(inst, new_phys_devs); } - total_count = 0; + inst->total_gpu_count = 0; } else { - // Free everything that didn't carry over to the new array of - // physical devices + if (new_count > inst->total_gpu_count) { + inst->total_gpu_count = new_count; + } + // Look for any items that were not used this time. if (NULL != inst->phys_devs_tramp) { - for (uint32_t i = 0; i < inst->phys_dev_count_tramp; i++) { - bool found = false; - for (uint32_t j = 0; j < total_count; j++) { - if (inst->phys_devs_tramp[i] == new_phys_devs[j]) { - found = true; - break; - } - } - if (!found) { - loader_instance_heap_free(inst, inst->phys_devs_tramp[i]); + for (cur_idx = 0; cur_idx < inst->phys_dev_count_tramp; ++cur_idx) { + if (NULL != inst->phys_devs_tramp[cur_idx]) { + loader_instance_heap_free(inst, inst->phys_devs_tramp[cur_idx]); + break; } } loader_instance_heap_free(inst, inst->phys_devs_tramp); } - - // Swap in the new physical device list - inst->phys_dev_count_tramp = total_count; inst->phys_devs_tramp = new_phys_devs; + inst->phys_dev_count_tramp = found_count; } return res; @@ -6443,6 +6484,50 @@ out: return res; } +VkResult setup_loader_tramp_phys_dev_groups(struct loader_instance *inst, uint32_t group_count, + VkPhysicalDeviceGroupProperties *groups) { + VkResult res = VK_SUCCESS; + uint32_t cur_idx; + uint32_t dev_idx; + + if (0 == group_count) { + return VK_SUCCESS; + } + + // Generate a list of all the devices and convert them to the loader ID + uint32_t phys_dev_count = 0; + for (cur_idx = 0; cur_idx < group_count; ++cur_idx) { + phys_dev_count += groups[cur_idx].physicalDeviceCount; + } + VkPhysicalDevice *devices = (VkPhysicalDevice *)loader_stack_alloc(sizeof(VkPhysicalDevice) * phys_dev_count); + if (NULL == devices) { + return VK_ERROR_OUT_OF_HOST_MEMORY; + } + + uint32_t cur_device = 0; + for (cur_idx = 0; cur_idx < group_count; ++cur_idx) { + for (dev_idx = 0; dev_idx < groups[cur_idx].physicalDeviceCount; ++dev_idx) { + devices[cur_device++] = groups[cur_idx].physicalDevices[dev_idx]; + } + } + + // Update the devices based on the loader physical device values. + res = setup_loader_tramp_phys_devs(inst, phys_dev_count, devices); + if (VK_SUCCESS != res) { + return res; + } + + // Update the devices in the group structures now + cur_device = 0; + for (cur_idx = 0; cur_idx < group_count; ++cur_idx) { + for (dev_idx = 0; dev_idx < groups[cur_idx].physicalDeviceCount; ++dev_idx) { + groups[cur_idx].physicalDevices[dev_idx] = devices[cur_device++]; + } + } + + return res; +} + VKAPI_ATTR VkResult VKAPI_CALL terminator_EnumeratePhysicalDevices(VkInstance instance, uint32_t *pPhysicalDeviceCount, VkPhysicalDevice *pPhysicalDevices) { struct loader_instance *inst = (struct loader_instance *)instance; @@ -6802,7 +6887,10 @@ out: // ---- Vulkan Core 1.1 terminators -VkResult setup_loader_term_phys_dev_groups(struct loader_instance *inst) { +VKAPI_ATTR VkResult VKAPI_CALL terminator_EnumeratePhysicalDeviceGroups( + VkInstance instance, uint32_t *pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties *pPhysicalDeviceGroupProperties) { + struct loader_instance *inst = (struct loader_instance *)instance; + VkResult res = VK_SUCCESS; struct loader_icd_term *icd_term; uint32_t total_count = 0; @@ -6815,15 +6903,6 @@ VkResult setup_loader_term_phys_dev_groups(struct loader_instance *inst) { uint32_t sorted_count = 0; uint32_t icd_idx = 0; - if (0 == inst->phys_dev_count_term) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_term_phys_dev_groups: Loader failed to setup physical device terminator info before calling " - "\'EnumeratePhysicalDeviceGroups\'."); - assert(false); - res = VK_ERROR_INITIALIZATION_FAILED; - goto out; - } - // For each ICD, query the number of physical device groups, and then get an // internal value for those physical devices. icd_term = inst->icd_terms; @@ -6860,356 +6939,380 @@ VkResult setup_loader_term_phys_dev_groups(struct loader_instance *inst) { total_count += cur_icd_group_count; } - if (total_count == 0) { - loader_log(inst, VULKAN_LOADER_INFO_BIT, 0, - "setupLoaderTermPhysDevGroups: Did not detect any GPU Groups" - " in the current config"); - goto out; + // If GPUs not sorted yet, look through them and generate list of all available GPUs + if (0 == total_count || 0 == inst->total_gpu_count) { + if (VK_SUCCESS != setup_loader_term_phys_devs(inst)) { + res = VK_ERROR_INITIALIZATION_FAILED; + loader_log(inst, VULKAN_LOADER_INFO_BIT, 0, + "setupLoaderTermPhysDevGroups: Did not detect any GPU Groups" + " in the current config"); + goto out; + } } - // Create an array for the new physical device groups, which will be stored - // in the instance for the Terminator code. - new_phys_dev_groups = (VkPhysicalDeviceGroupProperties **)loader_instance_heap_alloc( - inst, total_count * sizeof(VkPhysicalDeviceGroupProperties *), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); - if (NULL == new_phys_dev_groups) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_term_phys_dev_groups: Failed to allocate new physical device group array of size %d", - total_count); - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; - } - memset(new_phys_dev_groups, 0, total_count * sizeof(VkPhysicalDeviceGroupProperties *)); + if (NULL != pPhysicalDeviceGroupProperties) { + // Create an array for the new physical device groups, which will be stored + // in the instance for the Terminator code. + new_phys_dev_groups = (VkPhysicalDeviceGroupProperties **)loader_instance_heap_alloc( + inst, total_count * sizeof(VkPhysicalDeviceGroupProperties *), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); + if (NULL == new_phys_dev_groups) { + loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, + "setup_loader_term_phys_dev_groups: Failed to allocate new physical device group array of size %d", + total_count); + res = VK_ERROR_OUT_OF_HOST_MEMORY; + goto out; + } + memset(new_phys_dev_groups, 0, total_count * sizeof(VkPhysicalDeviceGroupProperties *)); - // Create a temporary array (on the stack) to keep track of the - // returned VkPhysicalDevice values. - local_phys_dev_groups = loader_stack_alloc(sizeof(struct loader_physical_device_group_term) * total_count); - local_phys_dev_group_sorted = loader_stack_alloc(sizeof(bool) * total_count); - if (NULL == local_phys_dev_groups || NULL == local_phys_dev_group_sorted) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_term_phys_dev_groups: Failed to allocate local physical device group array of size %d", - total_count); - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; - } - // Initialize the memory to something valid - memset(local_phys_dev_groups, 0, sizeof(struct loader_physical_device_group_term) * total_count); - memset(local_phys_dev_group_sorted, 0, sizeof(bool) * total_count); - for (uint32_t group = 0; group < total_count; group++) { - local_phys_dev_groups[group].group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR; - local_phys_dev_groups[group].group_props.pNext = NULL; - local_phys_dev_groups[group].group_props.subsetAllocation = false; - } + // Create a temporary array (on the stack) to keep track of the + // returned VkPhysicalDevice values. + local_phys_dev_groups = loader_stack_alloc(sizeof(struct loader_physical_device_group_term) * total_count); + local_phys_dev_group_sorted = loader_stack_alloc(sizeof(bool) * total_count); + if (NULL == local_phys_dev_groups || NULL == local_phys_dev_group_sorted) { + loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, + "setup_loader_term_phys_dev_groups: Failed to allocate local physical device group array of size %d", + total_count); + res = VK_ERROR_OUT_OF_HOST_MEMORY; + goto out; + } + // Initialize the memory to something valid + memset(local_phys_dev_groups, 0, sizeof(struct loader_physical_device_group_term) * total_count); + memset(local_phys_dev_group_sorted, 0, sizeof(bool) * total_count); #if defined(_WIN32) - // Get the physical devices supported by platform sorting mechanism into a separate list - res = windows_read_sorted_physical_devices(inst, &sorted_phys_dev_array, &sorted_count); - if (VK_SUCCESS != res) { - goto out; - } + // Get the physical devices supported by platform sorting mechanism into a separate list + res = windows_read_sorted_physical_devices(inst, &sorted_phys_dev_array, &sorted_count); + if (VK_SUCCESS != res) { + goto out; + } #endif - cur_icd_group_count = 0; - icd_term = inst->icd_terms; - for (icd_idx = 0; NULL != icd_term; icd_term = icd_term->next, icd_idx++) { - uint32_t count_this_time = total_count - cur_icd_group_count; + cur_icd_group_count = 0; + icd_term = inst->icd_terms; + for (icd_idx = 0; NULL != icd_term; icd_term = icd_term->next, icd_idx++) { + uint32_t count_this_time = total_count - cur_icd_group_count; - // Check if this group can be sorted + // Check if this group can be sorted #if defined(VK_USE_PLATFORM_WIN32_KHR) - bool icd_sorted = sorted_count && (icd_term->scanned_icd->EnumerateAdapterPhysicalDevices != NULL); + bool icd_sorted = sorted_count && (icd_term->scanned_icd->EnumerateAdapterPhysicalDevices != NULL); #else - bool icd_sorted = false; + bool icd_sorted = false; #endif - // Get the function pointer to use to call into the ICD. This could be the core or KHR version - if (inst->enabled_known_extensions.khr_device_group_creation) { - fpEnumeratePhysicalDeviceGroups = icd_term->dispatch.EnumeratePhysicalDeviceGroupsKHR; - } else { - fpEnumeratePhysicalDeviceGroups = icd_term->dispatch.EnumeratePhysicalDeviceGroups; - } - - if (NULL == fpEnumeratePhysicalDeviceGroups) { - VkPhysicalDevice *phys_dev_array = loader_stack_alloc(sizeof(VkPhysicalDevice) * count_this_time); - if (NULL == phys_dev_array) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_term_phys_dev_groups: Failed to allocate local physical device array of size %d", - count_this_time); - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; + // Get the function pointer to use to call into the ICD. This could be the core or KHR version + if (inst->enabled_known_extensions.khr_device_group_creation) { + fpEnumeratePhysicalDeviceGroups = icd_term->dispatch.EnumeratePhysicalDeviceGroupsKHR; + } else { + fpEnumeratePhysicalDeviceGroups = icd_term->dispatch.EnumeratePhysicalDeviceGroups; } - res = icd_term->dispatch.EnumeratePhysicalDevices(icd_term->instance, &count_this_time, phys_dev_array); - if (res != VK_SUCCESS) { - loader_log( - inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_term_phys_dev_groups: Failed during dispatch call of \'EnumeratePhysicalDevices\' to ICD %d " - "to get plain phys dev count.", - icd_idx); - goto out; - } + if (NULL == fpEnumeratePhysicalDeviceGroups) { + icd_term->dispatch.EnumeratePhysicalDevices(icd_term->instance, &count_this_time, NULL); - // Add each GPU as it's own group - for (uint32_t indiv_gpu = 0; indiv_gpu < count_this_time; indiv_gpu++) { - local_phys_dev_groups[indiv_gpu + cur_icd_group_count].this_icd_term = icd_term; - local_phys_dev_groups[indiv_gpu + cur_icd_group_count].icd_index = icd_idx; - local_phys_dev_groups[indiv_gpu + cur_icd_group_count].group_props.physicalDeviceCount = 1; - local_phys_dev_groups[indiv_gpu + cur_icd_group_count].group_props.physicalDevices[0] = phys_dev_array[indiv_gpu]; - local_phys_dev_group_sorted[indiv_gpu + cur_icd_group_count] = icd_sorted; - } + VkPhysicalDevice *phys_dev_array = loader_stack_alloc(sizeof(VkPhysicalDevice) * count_this_time); + if (NULL == phys_dev_array) { + loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, + "setup_loader_term_phys_dev_groups: Failed to allocate local physical device array of size %d", + count_this_time); + res = VK_ERROR_OUT_OF_HOST_MEMORY; + goto out; + } - } else { - res = fpEnumeratePhysicalDeviceGroups(icd_term->instance, &count_this_time, - &local_phys_dev_groups[cur_icd_group_count].group_props); - for (uint32_t group = 0; group < count_this_time; ++group) { - local_phys_dev_group_sorted[group + cur_icd_group_count] = icd_sorted; - local_phys_dev_groups[group + cur_icd_group_count].this_icd_term = icd_term; - local_phys_dev_groups[group + cur_icd_group_count].icd_index = icd_idx; - } - if (VK_SUCCESS != res) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_term_phys_dev_groups: Failed during dispatch call of " - "\'EnumeratePhysicalDeviceGroups\' to ICD %d to get content.", - icd_idx); - goto out; + res = icd_term->dispatch.EnumeratePhysicalDevices(icd_term->instance, &count_this_time, phys_dev_array); + if (res != VK_SUCCESS) { + loader_log( + inst, VULKAN_LOADER_ERROR_BIT, 0, + "setup_loader_term_phys_dev_groups: Failed during dispatch call of \'EnumeratePhysicalDevices\' to ICD %d " + "to get plain phys dev count.", + icd_idx); + goto out; + } + + // Add each GPU as it's own group + for (uint32_t indiv_gpu = 0; indiv_gpu < count_this_time; indiv_gpu++) { + uint32_t cur_index = indiv_gpu + cur_icd_group_count; + local_phys_dev_groups[cur_index].this_icd_term = icd_term; + local_phys_dev_groups[cur_index].icd_index = icd_idx; + local_phys_dev_groups[cur_index].group_props.physicalDeviceCount = 1; + local_phys_dev_groups[cur_index].group_props.physicalDevices[0] = phys_dev_array[indiv_gpu]; + local_phys_dev_group_sorted[cur_index] = icd_sorted; + } + + } else { + fpEnumeratePhysicalDeviceGroups(icd_term->instance, &count_this_time, NULL); + if (cur_icd_group_count + count_this_time < *pPhysicalDeviceGroupCount) { + // Can just use passed in structs + res = fpEnumeratePhysicalDeviceGroups(icd_term->instance, &count_this_time, + &pPhysicalDeviceGroupProperties[cur_icd_group_count]); + for (uint32_t group = 0; group < count_this_time; ++group) { + uint32_t cur_index = group + cur_icd_group_count; + local_phys_dev_groups[cur_index].group_props = pPhysicalDeviceGroupProperties[cur_index]; + local_phys_dev_group_sorted[cur_index] = icd_sorted; + local_phys_dev_groups[cur_index].this_icd_term = icd_term; + local_phys_dev_groups[cur_index].icd_index = icd_idx; + } + } else { + // Have to use a temporary copy + VkPhysicalDeviceGroupProperties *tmp_group_props = + loader_stack_alloc(count_this_time * sizeof(VkPhysicalDeviceGroupProperties)); + for (uint32_t group = 0; group < count_this_time; group++) { + tmp_group_props[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR; + uint32_t cur_index = group + cur_icd_group_count; + if (*pPhysicalDeviceGroupCount > cur_index) { + tmp_group_props[group].pNext = pPhysicalDeviceGroupProperties[cur_index].pNext; + } else { + tmp_group_props[group].pNext = NULL; + } + tmp_group_props[group].subsetAllocation = false; + } + + res = fpEnumeratePhysicalDeviceGroups(icd_term->instance, &count_this_time, tmp_group_props); + for (uint32_t group = 0; group < count_this_time; ++group) { + uint32_t cur_index = group + cur_icd_group_count; + local_phys_dev_groups[cur_index].group_props = tmp_group_props[group]; + local_phys_dev_group_sorted[cur_index] = icd_sorted; + local_phys_dev_groups[cur_index].this_icd_term = icd_term; + local_phys_dev_groups[cur_index].icd_index = icd_idx; + } + } + if (VK_SUCCESS != res) { + loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, + "setup_loader_term_phys_dev_groups: Failed during dispatch call of " + "\'EnumeratePhysicalDeviceGroups\' to ICD %d to get content.", + icd_idx); + goto out; + } } - } - cur_icd_group_count += count_this_time; - } + cur_icd_group_count += count_this_time; + } #ifdef LOADER_ENABLE_LINUX_SORT - if (is_linux_sort_enabled(inst)) { - // Get the physical devices supported by platform sorting mechanism into a separate list - res = linux_read_sorted_physical_device_groups(inst, total_count, local_phys_dev_groups); - } + if (is_linux_sort_enabled(inst)) { + // Get the physical devices supported by platform sorting mechanism into a separate list + res = linux_read_sorted_physical_device_groups(inst, total_count, local_phys_dev_groups); + } #endif // LOADER_ENABLE_LINUX_SORT - // Replace all the physical device IDs with the proper loader values - for (uint32_t group = 0; group < total_count; group++) { - for (uint32_t group_gpu = 0; group_gpu < local_phys_dev_groups[group].group_props.physicalDeviceCount; group_gpu++) { - bool found = false; - for (uint32_t term_gpu = 0; term_gpu < inst->phys_dev_count_term; term_gpu++) { - if (local_phys_dev_groups[group].group_props.physicalDevices[group_gpu] == - inst->phys_devs_term[term_gpu]->phys_dev) { - local_phys_dev_groups[group].group_props.physicalDevices[group_gpu] = - (VkPhysicalDevice)inst->phys_devs_term[term_gpu]; - found = true; - break; + // Replace all the physical device IDs with the proper loader values + if (NULL != inst->phys_devs_term) { + for (uint32_t group = 0; group < total_count; group++) { + for (uint32_t group_gpu = 0; group_gpu < local_phys_dev_groups[group].group_props.physicalDeviceCount; + group_gpu++) { + bool found = false; + for (uint32_t term_gpu = 0; term_gpu < inst->phys_dev_count_term; term_gpu++) { + if (local_phys_dev_groups[group].group_props.physicalDevices[group_gpu] == + inst->phys_devs_term[term_gpu]->phys_dev) { + local_phys_dev_groups[group].group_props.physicalDevices[group_gpu] = + (VkPhysicalDevice)inst->phys_devs_term[term_gpu]; + found = true; + break; + } + } + if (!found) { + loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, + "setup_loader_term_phys_dev_groups: Failed to find GPU %d in group %d returned by " + "\'EnumeratePhysicalDeviceGroups\' in list returned by \'EnumeratePhysicalDevices\'", + group_gpu, group); + res = VK_ERROR_INITIALIZATION_FAILED; + goto out; + } } } - if (!found) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_term_phys_dev_groups: Failed to find GPU %d in group %d returned by " - "\'EnumeratePhysicalDeviceGroups\' in list returned by \'EnumeratePhysicalDevices\'", - group_gpu, group); - res = VK_ERROR_INITIALIZATION_FAILED; - goto out; - } } - } - uint32_t idx = 0; + uint32_t idx = 0; #if defined(_WIN32) - // Copy over everything found through sorted enumeration - for (uint32_t i = 0; i < sorted_count; ++i) { - // Find the VkPhysicalDeviceGroupProperties object in local_phys_dev_groups - VkPhysicalDeviceGroupProperties *group_properties = NULL; - for (uint32_t group = 0; group < total_count; group++) { - if (sorted_phys_dev_array[i].device_count != local_phys_dev_groups[group].group_props.physicalDeviceCount) { - continue; - } + // Copy over everything found through sorted enumeration + for (uint32_t i = 0; i < sorted_count; ++i) { + // Find the VkPhysicalDeviceGroupProperties object in local_phys_dev_groups + VkPhysicalDeviceGroupProperties *group_properties = NULL; + for (uint32_t group = 0; group < total_count; group++) { + if (sorted_phys_dev_array[i].device_count != local_phys_dev_groups[group].group_props.physicalDeviceCount) { + continue; + } - bool match = true; - for (uint32_t group_gpu = 0; group_gpu < local_phys_dev_groups[group].group_props.physicalDeviceCount; group_gpu++) { - if (sorted_phys_dev_array[i].physical_devices[group_gpu] != - ((struct loader_physical_device_term *)local_phys_dev_groups[group].group_props.physicalDevices[group_gpu]) - ->phys_dev) { - match = false; - break; + bool match = true; + for (uint32_t group_gpu = 0; group_gpu < local_phys_dev_groups[group].group_props.physicalDeviceCount; + group_gpu++) { + if (sorted_phys_dev_array[i].physical_devices[group_gpu] != + ((struct loader_physical_device_term *)local_phys_dev_groups[group].group_props.physicalDevices[group_gpu]) + ->phys_dev) { + match = false; + break; + } } - } - if (match) { - group_properties = &local_phys_dev_groups[group].group_props; + if (match) { + group_properties = &local_phys_dev_groups[group].group_props; + } } - } - // Check if this physical device group with the same contents is already in the old buffer - for (uint32_t old_idx = 0; old_idx < inst->phys_dev_group_count_term; old_idx++) { - if (NULL != group_properties && - group_properties->physicalDeviceCount == inst->phys_dev_groups_term[old_idx]->physicalDeviceCount) { - bool found_all_gpus = true; - for (uint32_t old_gpu = 0; old_gpu < inst->phys_dev_groups_term[old_idx]->physicalDeviceCount; old_gpu++) { - bool found_gpu = false; - for (uint32_t new_gpu = 0; new_gpu < group_properties->physicalDeviceCount; new_gpu++) { - if (group_properties->physicalDevices[new_gpu] == - inst->phys_dev_groups_term[old_idx]->physicalDevices[old_gpu]) { - found_gpu = true; + // Check if this physical device group with the same contents is already in the old buffer + for (uint32_t old_idx = 0; old_idx < inst->phys_dev_group_count_term; old_idx++) { + if (NULL != group_properties && + group_properties->physicalDeviceCount == inst->phys_dev_groups_term[old_idx]->physicalDeviceCount) { + bool found_all_gpus = true; + for (uint32_t old_gpu = 0; old_gpu < inst->phys_dev_groups_term[old_idx]->physicalDeviceCount; old_gpu++) { + bool found_gpu = false; + for (uint32_t new_gpu = 0; new_gpu < group_properties->physicalDeviceCount; new_gpu++) { + if (group_properties->physicalDevices[new_gpu] == + inst->phys_dev_groups_term[old_idx]->physicalDevices[old_gpu]) { + found_gpu = true; + break; + } + } + + if (!found_gpu) { + found_all_gpus = false; break; } } - - if (!found_gpu) { - found_all_gpus = false; + if (!found_all_gpus) { + continue; + } else { + new_phys_dev_groups[idx] = inst->phys_dev_groups_term[old_idx]; break; } } - if (!found_all_gpus) { - continue; - } else { - new_phys_dev_groups[idx] = inst->phys_dev_groups_term[old_idx]; - break; - } } - } - // If this physical device group isn't in the old buffer, create it - if (group_properties != NULL && NULL == new_phys_dev_groups[idx]) { - new_phys_dev_groups[idx] = (VkPhysicalDeviceGroupPropertiesKHR *)loader_instance_heap_alloc( - inst, sizeof(VkPhysicalDeviceGroupPropertiesKHR), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); - if (NULL == new_phys_dev_groups[idx]) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_term_phys_dev_groups: Failed to allocate physical device group Terminator object %d", - idx); - total_count = idx; - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; + // If this physical device group isn't in the old buffer, create it + if (group_properties != NULL && NULL == new_phys_dev_groups[idx]) { + new_phys_dev_groups[idx] = (VkPhysicalDeviceGroupPropertiesKHR *)loader_instance_heap_alloc( + inst, sizeof(VkPhysicalDeviceGroupPropertiesKHR), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); + if (NULL == new_phys_dev_groups[idx]) { + loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, + "setup_loader_term_phys_dev_groups: Failed to allocate physical device group Terminator object %d", + idx); + total_count = idx; + res = VK_ERROR_OUT_OF_HOST_MEMORY; + goto out; + } + memcpy(new_phys_dev_groups[idx], group_properties, sizeof(VkPhysicalDeviceGroupPropertiesKHR)); } - memcpy(new_phys_dev_groups[idx], group_properties, sizeof(VkPhysicalDeviceGroupPropertiesKHR)); - } - ++idx; - } + ++idx; + } #endif - // Copy or create everything to fill the new array of physical device groups - for (uint32_t new_idx = 0; new_idx < total_count; new_idx++) { - // Skip groups which have been included through sorting - if (local_phys_dev_group_sorted[new_idx] || local_phys_dev_groups[new_idx].group_props.physicalDeviceCount == 0) { - continue; - } + // Copy or create everything to fill the new array of physical device groups + for (uint32_t new_idx = 0; new_idx < total_count; new_idx++) { + // Skip groups which have been included through sorting + if (local_phys_dev_group_sorted[new_idx] || local_phys_dev_groups[new_idx].group_props.physicalDeviceCount == 0) { + continue; + } + + // Check if this physical device group with the same contents is already in the old buffer + for (uint32_t old_idx = 0; old_idx < inst->phys_dev_group_count_term; old_idx++) { + if (local_phys_dev_groups[new_idx].group_props.physicalDeviceCount == + inst->phys_dev_groups_term[old_idx]->physicalDeviceCount) { + bool found_all_gpus = true; + for (uint32_t old_gpu = 0; old_gpu < inst->phys_dev_groups_term[old_idx]->physicalDeviceCount; old_gpu++) { + bool found_gpu = false; + for (uint32_t new_gpu = 0; new_gpu < local_phys_dev_groups[new_idx].group_props.physicalDeviceCount; + new_gpu++) { + if (local_phys_dev_groups[new_idx].group_props.physicalDevices[new_gpu] == + inst->phys_dev_groups_term[old_idx]->physicalDevices[old_gpu]) { + found_gpu = true; + break; + } + } - // Check if this physical device group with the same contents is already in the old buffer - for (uint32_t old_idx = 0; old_idx < inst->phys_dev_group_count_term; old_idx++) { - if (local_phys_dev_groups[new_idx].group_props.physicalDeviceCount == - inst->phys_dev_groups_term[old_idx]->physicalDeviceCount) { - bool found_all_gpus = true; - for (uint32_t old_gpu = 0; old_gpu < inst->phys_dev_groups_term[old_idx]->physicalDeviceCount; old_gpu++) { - bool found_gpu = false; - for (uint32_t new_gpu = 0; new_gpu < local_phys_dev_groups[new_idx].group_props.physicalDeviceCount; - new_gpu++) { - if (local_phys_dev_groups[new_idx].group_props.physicalDevices[new_gpu] == - inst->phys_dev_groups_term[old_idx]->physicalDevices[old_gpu]) { - found_gpu = true; + if (!found_gpu) { + found_all_gpus = false; break; } } - - if (!found_gpu) { - found_all_gpus = false; + if (!found_all_gpus) { + continue; + } else { + new_phys_dev_groups[idx] = inst->phys_dev_groups_term[old_idx]; break; } } - if (!found_all_gpus) { - continue; - } else { - new_phys_dev_groups[idx] = inst->phys_dev_groups_term[old_idx]; - break; - } } - } - // If this physical device group isn't in the old buffer, create it - if (NULL == new_phys_dev_groups[idx]) { - new_phys_dev_groups[idx] = (VkPhysicalDeviceGroupPropertiesKHR *)loader_instance_heap_alloc( - inst, sizeof(VkPhysicalDeviceGroupPropertiesKHR), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); + // If this physical device group isn't in the old buffer, create it if (NULL == new_phys_dev_groups[idx]) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setup_loader_term_phys_dev_groups: Failed to allocate physical device group Terminator object %d", - idx); - total_count = idx; - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; + new_phys_dev_groups[idx] = (VkPhysicalDeviceGroupPropertiesKHR *)loader_instance_heap_alloc( + inst, sizeof(VkPhysicalDeviceGroupPropertiesKHR), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); + if (NULL == new_phys_dev_groups[idx]) { + loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, + "setup_loader_term_phys_dev_groups: Failed to allocate physical device group Terminator object %d", + idx); + total_count = idx; + res = VK_ERROR_OUT_OF_HOST_MEMORY; + goto out; + } + memcpy(new_phys_dev_groups[idx], &local_phys_dev_groups[new_idx].group_props, + sizeof(VkPhysicalDeviceGroupPropertiesKHR)); } - memcpy(new_phys_dev_groups[idx], &local_phys_dev_groups[new_idx].group_props, - sizeof(VkPhysicalDeviceGroupPropertiesKHR)); - } - ++idx; + ++idx; + } } out: - if (VK_SUCCESS != res) { - if (NULL != new_phys_dev_groups) { - for (uint32_t i = 0; i < total_count; i++) { - loader_instance_heap_free(inst, new_phys_dev_groups[i]); + if (NULL != pPhysicalDeviceGroupProperties) { + if (VK_SUCCESS != res) { + if (NULL != new_phys_dev_groups) { + for (uint32_t i = 0; i < total_count; i++) { + loader_instance_heap_free(inst, new_phys_dev_groups[i]); + } + loader_instance_heap_free(inst, new_phys_dev_groups); } - loader_instance_heap_free(inst, new_phys_dev_groups); - } - } else { - // Free everything that didn't carry over to the new array of - // physical device groups - if (NULL != inst->phys_dev_groups_term) { - for (uint32_t i = 0; i < inst->phys_dev_group_count_term; i++) { - bool found = false; - for (uint32_t j = 0; j < total_count; j++) { - if (inst->phys_dev_groups_term[i] == new_phys_dev_groups[j]) { - found = true; - break; + } else { + // Free everything that didn't carry over to the new array of + // physical device groups + if (NULL != inst->phys_dev_groups_term) { + for (uint32_t i = 0; i < inst->phys_dev_group_count_term; i++) { + bool found = false; + for (uint32_t j = 0; j < total_count; j++) { + if (inst->phys_dev_groups_term[i] == new_phys_dev_groups[j]) { + found = true; + break; + } + } + if (!found) { + loader_instance_heap_free(inst, inst->phys_dev_groups_term[i]); } } - if (!found) { - loader_instance_heap_free(inst, inst->phys_dev_groups_term[i]); - } + loader_instance_heap_free(inst, inst->phys_dev_groups_term); } - loader_instance_heap_free(inst, inst->phys_dev_groups_term); - } - // Swap in the new physical device group list - inst->phys_dev_group_count_term = total_count; - inst->phys_dev_groups_term = new_phys_dev_groups; - } + // Swap in the new physical device group list + inst->phys_dev_group_count_term = total_count; + inst->phys_dev_groups_term = new_phys_dev_groups; + } - if (sorted_phys_dev_array != NULL) { - for (uint32_t i = 0; i < sorted_count; ++i) { - if (sorted_phys_dev_array[i].device_count > 0 && sorted_phys_dev_array[i].physical_devices != NULL) { - loader_instance_heap_free(inst, sorted_phys_dev_array[i].physical_devices); + if (sorted_phys_dev_array != NULL) { + for (uint32_t i = 0; i < sorted_count; ++i) { + if (sorted_phys_dev_array[i].device_count > 0 && sorted_phys_dev_array[i].physical_devices != NULL) { + loader_instance_heap_free(inst, sorted_phys_dev_array[i].physical_devices); + } } + loader_instance_heap_free(inst, sorted_phys_dev_array); } - loader_instance_heap_free(inst, sorted_phys_dev_array); - } - - return res; -} - -VKAPI_ATTR VkResult VKAPI_CALL terminator_EnumeratePhysicalDeviceGroups( - VkInstance instance, uint32_t *pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties *pPhysicalDeviceGroupProperties) { - struct loader_instance *inst = (struct loader_instance *)instance; - VkResult res = VK_SUCCESS; - - // Always call the setup loader terminator physical device groups because they may - // have changed at any point. - res = setup_loader_term_phys_dev_groups(inst); - if (VK_SUCCESS != res) { - goto out; - } - uint32_t copy_count = inst->phys_dev_group_count_term; - if (NULL != pPhysicalDeviceGroupProperties) { - if (copy_count > *pPhysicalDeviceGroupCount) { - copy_count = *pPhysicalDeviceGroupCount; - res = VK_INCOMPLETE; - } + uint32_t copy_count = inst->phys_dev_group_count_term; + if (NULL != pPhysicalDeviceGroupProperties) { + if (copy_count > *pPhysicalDeviceGroupCount) { + copy_count = *pPhysicalDeviceGroupCount; + res = VK_INCOMPLETE; + } - for (uint32_t i = 0; i < copy_count; i++) { - memcpy(&pPhysicalDeviceGroupProperties[i], inst->phys_dev_groups_term[i], sizeof(VkPhysicalDeviceGroupPropertiesKHR)); + for (uint32_t i = 0; i < copy_count; i++) { + memcpy(&pPhysicalDeviceGroupProperties[i], inst->phys_dev_groups_term[i], sizeof(VkPhysicalDeviceGroupProperties)); + } } - } - *pPhysicalDeviceGroupCount = copy_count; - -out: + *pPhysicalDeviceGroupCount = copy_count; + } else { + *pPhysicalDeviceGroupCount = total_count; + } return res; } diff --git a/loader/loader.h b/loader/loader.h index 5700d157..4ef69966 100644 --- a/loader/loader.h +++ b/loader/loader.h @@ -1,8 +1,8 @@ /* * - * Copyright (c) 2014-2021 The Khronos Group Inc. - * Copyright (c) 2014-2021 Valve Corporation - * Copyright (c) 2014-2021 LunarG, Inc. + * Copyright (c) 2014-2022 The Khronos Group Inc. + * Copyright (c) 2014-2022 Valve Corporation + * Copyright (c) 2014-2022 LunarG, Inc. * Copyright (C) 2015 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -164,8 +164,8 @@ VkResult loader_validate_device_extensions(struct loader_instance *this_instance const struct loader_layer_list *activated_device_layers, const struct loader_extension_list *icd_exts, const VkDeviceCreateInfo *pCreateInfo); -VkResult setup_loader_tramp_phys_devs(struct loader_instance *inst); -VkResult setup_loader_term_phys_devs(struct loader_instance *inst); +VkResult setup_loader_tramp_phys_devs(struct loader_instance *inst, uint32_t phys_dev_count, VkPhysicalDevice *phys_devs); +VkResult setup_loader_tramp_phys_dev_groups(struct loader_instance *inst, uint32_t group_count, VkPhysicalDeviceGroupProperties *groups); VkStringErrorFlags vk_string_validate(const int max_length, const char *char_array); char *loader_get_next_path(char *path); diff --git a/loader/loader_common.h b/loader/loader_common.h index 5f923434..de9aa1a9 100644 --- a/loader/loader_common.h +++ b/loader/loader_common.h @@ -271,8 +271,6 @@ struct loader_instance { // device stored internal to the public structures. uint32_t phys_dev_group_count_term; struct VkPhysicalDeviceGroupProperties **phys_dev_groups_term; - uint32_t phys_dev_group_count_tramp; - struct VkPhysicalDeviceGroupProperties **phys_dev_groups_tramp; struct loader_instance *next; diff --git a/loader/loader_linux.c b/loader/loader_linux.c index ff82c9c3..6f914b07 100644 --- a/loader/loader_linux.c +++ b/loader/loader_linux.c @@ -219,7 +219,8 @@ static void linux_env_var_default_device(struct loader_instance *inst, uint32_t for (int32_t i = 0; i < (int32_t)device_count; ++i) { if (sorted_device_info[i].vendor_id == vendor_id && sorted_device_info[i].device_id == device_id) { loader_log(inst, VULKAN_LOADER_INFO_BIT | VULKAN_LOADER_DRIVER_BIT, 0, - "linux_env_var_default_device: Found default at index %u \'%s\'", i, sorted_device_info[i].device_name); + "linux_env_var_default_device: Found default at index %u \'%s\'", i, + sorted_device_info[i].device_name); sorted_device_info[i].default_device = true; break; } @@ -424,6 +425,11 @@ VkResult linux_read_sorted_physical_device_groups(struct loader_instance *inst, // Sort GPUs in each group qsort(sorted_group_term[group].internal_device_info, sorted_group_term[group].group_props.physicalDeviceCount, sizeof(struct LinuxSortedDeviceInfo), compare_devices); + + // Match the externally used physical device list with the sorted physical device list for this group. + for (uint32_t dev = 0; dev < sorted_group_term[group].group_props.physicalDeviceCount; ++dev) { + sorted_group_term[group].group_props.physicalDevices[dev] = sorted_group_term[group].internal_device_info[dev].physical_device; + } } // Sort device groups by PCI info @@ -435,8 +441,9 @@ VkResult linux_read_sorted_physical_device_groups(struct loader_instance *inst, for (uint32_t group = 0; group < group_count; ++group) { loader_log(inst, VULKAN_LOADER_INFO_BIT | VULKAN_LOADER_DRIVER_BIT, 0, " Group %u", group); for (uint32_t gpu = 0; gpu < sorted_group_term[group].group_props.physicalDeviceCount; ++gpu) { - loader_log(inst, VULKAN_LOADER_INFO_BIT | VULKAN_LOADER_DRIVER_BIT, 0, " [%u] %s %s", gpu, + loader_log(inst, VULKAN_LOADER_INFO_BIT | VULKAN_LOADER_DRIVER_BIT, 0, " [%u] %s %p %s", gpu, sorted_group_term[group].internal_device_info[gpu].device_name, + sorted_group_term[group].internal_device_info[gpu].physical_device, (sorted_group_term[group].internal_device_info[gpu].default_device ? "[default]" : "")); } } diff --git a/loader/trampoline.c b/loader/trampoline.c index be06f57e..69723b53 100644 --- a/loader/trampoline.c +++ b/loader/trampoline.c @@ -724,13 +724,6 @@ LOADER_EXPORT VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(VkInstance instance, loader_instance_heap_free(ptr_instance, ptr_instance->phys_devs_tramp); } - if (ptr_instance->phys_dev_groups_tramp) { - for (uint32_t i = 0; i < ptr_instance->phys_dev_group_count_tramp; i++) { - loader_instance_heap_free(ptr_instance, ptr_instance->phys_dev_groups_tramp[i]); - } - loader_instance_heap_free(ptr_instance, ptr_instance->phys_dev_groups_tramp); - } - if (messenger_setup) { loader_log(ptr_instance, VULKAN_LOADER_INFO_BIT, 0, "vkDestroyInstance: destroying temporary instance debug util messenger"); @@ -760,8 +753,6 @@ LOADER_EXPORT VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(VkInstance instance, LOADER_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDevices(VkInstance instance, uint32_t *pPhysicalDeviceCount, VkPhysicalDevice *pPhysicalDevices) { VkResult res = VK_SUCCESS; - uint32_t count; - uint32_t i; struct loader_instance *inst; loader_platform_thread_lock_mutex(&loader_lock); @@ -784,38 +775,16 @@ LOADER_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDevices(VkInstan // Setup the trampoline loader physical devices. This will actually // call down and setup the terminator loader physical devices during the // process. - VkResult setup_res = setup_loader_tramp_phys_devs(inst); - if (setup_res != VK_SUCCESS && setup_res != VK_INCOMPLETE) { - res = setup_res; - goto out; - } - - count = inst->phys_dev_count_tramp; - - if (inst->phys_dev_count_tramp != inst->total_gpu_count) { - loader_log(inst, VULKAN_LOADER_WARN_BIT | VULKAN_LOADER_LAYER_BIT, 0, - "vkEnumeratePhysicalDevices: One or more layers modified physical devices!" - "Count returned by ICDs = %d, count returned above layers = %d", - inst->total_gpu_count, inst->phys_dev_count_tramp); - } + res = inst->disp->layer_inst_disp.EnumeratePhysicalDevices(inst->instance, pPhysicalDeviceCount, pPhysicalDevices); // Wrap the PhysDev object for loader usage, return wrapped objects - if (NULL != pPhysicalDevices) { - if (inst->phys_dev_count_tramp > *pPhysicalDeviceCount) { - loader_log(inst, VULKAN_LOADER_INFO_BIT, 0, - "vkEnumeratePhysicalDevices: Trimming device count down" - " by application request from %d to %d physical devices", - inst->phys_dev_count_tramp, *pPhysicalDeviceCount); - count = *pPhysicalDeviceCount; - res = VK_INCOMPLETE; - } - for (i = 0; i < count; i++) { - pPhysicalDevices[i] = (VkPhysicalDevice)inst->phys_devs_tramp[i]; + if (NULL != pPhysicalDevices && (VK_SUCCESS == res || VK_INCOMPLETE == res)) { + VkResult update_res = setup_loader_tramp_phys_devs(inst, *pPhysicalDeviceCount, pPhysicalDevices); + if (VK_SUCCESS != update_res) { + res = update_res; } } - *pPhysicalDeviceCount = count; - out: loader_platform_thread_unlock_mutex(&loader_lock); @@ -2516,193 +2485,9 @@ LOADER_EXPORT VKAPI_ATTR void VKAPI_CALL vkCmdExecuteCommands(VkCommandBuffer co // ---- Vulkan core 1.1 trampolines -VkResult setupLoaderTrampPhysDevGroups(VkInstance instance, struct loader_instance *inst) { - VkResult res = VK_SUCCESS; - uint32_t total_count = 0; - VkPhysicalDeviceGroupPropertiesKHR **new_phys_dev_groups = NULL; - VkPhysicalDeviceGroupPropertiesKHR *local_phys_dev_groups = NULL; - PFN_vkEnumeratePhysicalDeviceGroups fpEnumeratePhysicalDeviceGroups = NULL; - - // Get the function pointer to use to call into the ICD. This could be the core or KHR version - if (inst->enabled_known_extensions.khr_device_group_creation) { - fpEnumeratePhysicalDeviceGroups = inst->disp->layer_inst_disp.EnumeratePhysicalDeviceGroupsKHR; - } else { - fpEnumeratePhysicalDeviceGroups = inst->disp->layer_inst_disp.EnumeratePhysicalDeviceGroups; - } - - // Setup the trampoline loader physical devices. This will actually - // call down and setup the terminator loader physical devices during the - // process. - VkResult setup_res = setup_loader_tramp_phys_devs(inst); - if (setup_res != VK_SUCCESS && setup_res != VK_INCOMPLETE) { - res = setup_res; - goto out; - } - - // Query how many physical device groups there - res = fpEnumeratePhysicalDeviceGroups(inst->instance, &total_count, NULL); - if (res != VK_SUCCESS) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setupLoaderTrampPhysDevGroups: Failed during dispatch call of " - "\'EnumeratePhysicalDeviceGroupsKHR\' to lower layers or " - "loader to get count."); - goto out; - } - - // Create an array for the new physical device groups, which will be stored - // in the instance for the trampoline code. - new_phys_dev_groups = (VkPhysicalDeviceGroupPropertiesKHR **)loader_instance_heap_alloc( - inst, total_count * sizeof(VkPhysicalDeviceGroupPropertiesKHR *), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); - if (NULL == new_phys_dev_groups) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setupLoaderTrampPhysDevGroups: Failed to allocate new physical device" - " group array of size %d", - total_count); - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; - } - memset(new_phys_dev_groups, 0, total_count * sizeof(VkPhysicalDeviceGroupPropertiesKHR *)); - - // Create a temporary array (on the stack) to keep track of the - // returned VkPhysicalDevice values. - local_phys_dev_groups = loader_stack_alloc(sizeof(VkPhysicalDeviceGroupPropertiesKHR) * total_count); - if (NULL == local_phys_dev_groups) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setupLoaderTrampPhysDevGroups: Failed to allocate local " - "physical device group array of size %d", - total_count); - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; - } - // Initialize the memory to something valid - memset(local_phys_dev_groups, 0, sizeof(VkPhysicalDeviceGroupPropertiesKHR) * total_count); - for (uint32_t group = 0; group < total_count; group++) { - local_phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR; - local_phys_dev_groups[group].pNext = NULL; - local_phys_dev_groups[group].subsetAllocation = false; - } - - // Call down and get the content - fpEnumeratePhysicalDeviceGroups(inst->instance, &total_count, local_phys_dev_groups); - if (VK_SUCCESS != res) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setupLoaderTrampPhysDevGroups: Failed during dispatch call of " - "\'EnumeratePhysicalDeviceGroupsKHR\' to lower layers or " - "loader to get content."); - goto out; - } - - // Replace all the physical device IDs with the proper loader values - for (uint32_t group = 0; group < total_count; group++) { - for (uint32_t group_gpu = 0; group_gpu < local_phys_dev_groups[group].physicalDeviceCount; group_gpu++) { - bool found = false; - for (uint32_t tramp_gpu = 0; tramp_gpu < inst->phys_dev_count_tramp; tramp_gpu++) { - if (local_phys_dev_groups[group].physicalDevices[group_gpu] == inst->phys_devs_tramp[tramp_gpu]->phys_dev) { - local_phys_dev_groups[group].physicalDevices[group_gpu] = (VkPhysicalDevice)inst->phys_devs_tramp[tramp_gpu]; - found = true; - break; - } - } - if (!found) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setupLoaderTrampPhysDevGroups: Failed to find GPU %d in group %d" - " returned by \'EnumeratePhysicalDeviceGroupsKHR\' in list returned" - " by \'EnumeratePhysicalDevices\'", - group_gpu, group); - res = VK_ERROR_INITIALIZATION_FAILED; - goto out; - } - } - } - - // Copy or create everything to fill the new array of physical device groups - for (uint32_t new_idx = 0; new_idx < total_count; new_idx++) { - // Check if this physical device group with the same contents is already in the old buffer - for (uint32_t old_idx = 0; old_idx < inst->phys_dev_group_count_tramp; old_idx++) { - if (local_phys_dev_groups[new_idx].physicalDeviceCount == inst->phys_dev_groups_tramp[old_idx]->physicalDeviceCount) { - bool found_all_gpus = true; - for (uint32_t old_gpu = 0; old_gpu < inst->phys_dev_groups_tramp[old_idx]->physicalDeviceCount; old_gpu++) { - bool found_gpu = false; - for (uint32_t new_gpu = 0; new_gpu < local_phys_dev_groups[new_idx].physicalDeviceCount; new_gpu++) { - if (local_phys_dev_groups[new_idx].physicalDevices[new_gpu] == - inst->phys_dev_groups_tramp[old_idx]->physicalDevices[old_gpu]) { - found_gpu = true; - break; - } - } - - if (!found_gpu) { - found_all_gpus = false; - break; - } - } - if (!found_all_gpus) { - continue; - } else { - new_phys_dev_groups[new_idx] = inst->phys_dev_groups_tramp[old_idx]; - break; - } - } - } - - // If this physical device group isn't in the old buffer, create it - if (NULL == new_phys_dev_groups[new_idx]) { - new_phys_dev_groups[new_idx] = (VkPhysicalDeviceGroupPropertiesKHR *)loader_instance_heap_alloc( - inst, sizeof(VkPhysicalDeviceGroupPropertiesKHR), VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); - if (NULL == new_phys_dev_groups[new_idx]) { - loader_log(inst, VULKAN_LOADER_ERROR_BIT, 0, - "setupLoaderTrampPhysDevGroups: Failed to allocate " - "physical device group trampoline object %d", - new_idx); - total_count = new_idx; - res = VK_ERROR_OUT_OF_HOST_MEMORY; - goto out; - } - memcpy(new_phys_dev_groups[new_idx], &local_phys_dev_groups[new_idx], sizeof(VkPhysicalDeviceGroupPropertiesKHR)); - } - } - -out: - if (VK_SUCCESS != res) { - if (NULL != new_phys_dev_groups) { - for (uint32_t i = 0; i < total_count; i++) { - loader_instance_heap_free(inst, new_phys_dev_groups[i]); - } - loader_instance_heap_free(inst, new_phys_dev_groups); - } - total_count = 0; - } else { - // Free everything that didn't carry over to the new array of - // physical device groups - if (NULL != inst->phys_dev_groups_tramp) { - for (uint32_t i = 0; i < inst->phys_dev_group_count_tramp; i++) { - bool found = false; - for (uint32_t j = 0; j < total_count; j++) { - if (inst->phys_dev_groups_tramp[i] == new_phys_dev_groups[j]) { - found = true; - break; - } - } - if (!found) { - loader_instance_heap_free(inst, inst->phys_dev_groups_tramp[i]); - } - } - loader_instance_heap_free(inst, inst->phys_dev_groups_tramp); - } - - // Swap in the new physical device group list - inst->phys_dev_group_count_tramp = total_count; - inst->phys_dev_groups_tramp = new_phys_dev_groups; - } - - return res; -} - LOADER_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDeviceGroups( VkInstance instance, uint32_t *pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties *pPhysicalDeviceGroupProperties) { VkResult res = VK_SUCCESS; - uint32_t count; - uint32_t i; struct loader_instance *inst = NULL; loader_platform_thread_lock_mutex(&loader_lock); @@ -2723,31 +2508,19 @@ LOADER_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDeviceGroups( goto out; } - VkResult setup_res = setupLoaderTrampPhysDevGroups(instance, inst); - if (VK_SUCCESS != setup_res) { - res = setup_res; - goto out; - } - - count = inst->phys_dev_group_count_tramp; - + // Setup the trampoline loader physical devices. This will actually + // call down and setup the terminator loader physical devices during the + // process. + res = inst->disp->layer_inst_disp.EnumeratePhysicalDeviceGroups(inst->instance, pPhysicalDeviceGroupCount, + pPhysicalDeviceGroupProperties); // Wrap the PhysDev object for loader usage, return wrapped objects - if (NULL != pPhysicalDeviceGroupProperties) { - if (inst->phys_dev_group_count_tramp > *pPhysicalDeviceGroupCount) { - loader_log(inst, VULKAN_LOADER_INFO_BIT, 0, - "vkEnumeratePhysicalDeviceGroupsKHR: Trimming device group count down" - " by application request from %d to %d physical device groups", - inst->phys_dev_group_count_tramp, *pPhysicalDeviceGroupCount); - count = *pPhysicalDeviceGroupCount; - res = VK_INCOMPLETE; - } - for (i = 0; i < count; i++) { - memcpy(&pPhysicalDeviceGroupProperties[i], inst->phys_dev_groups_tramp[i], sizeof(VkPhysicalDeviceGroupPropertiesKHR)); + if (NULL != pPhysicalDeviceGroupProperties && (VK_SUCCESS == res || VK_INCOMPLETE == res)) { + VkResult update_res = setup_loader_tramp_phys_dev_groups(inst, *pPhysicalDeviceGroupCount, pPhysicalDeviceGroupProperties); + if (VK_SUCCESS != update_res) { + res = update_res; } } - *pPhysicalDeviceGroupCount = count; - out: loader_platform_thread_unlock_mutex(&loader_lock); diff --git a/tests/framework/framework_config.h.in b/tests/framework/framework_config.h.in index 56b218e4..88b28677 100644 Binary files a/tests/framework/framework_config.h.in and b/tests/framework/framework_config.h.in differ diff --git a/tests/framework/icd/physical_device.h b/tests/framework/icd/physical_device.h index 8ee79030..b2916ce7 100644 --- a/tests/framework/icd/physical_device.h +++ b/tests/framework/icd/physical_device.h @@ -76,6 +76,8 @@ struct PhysicalDevice { }; struct PhysicalDeviceGroup { + PhysicalDeviceGroup() {} + PhysicalDeviceGroup(PhysicalDevice const& physical_device) { physical_device_handles.push_back(&physical_device); } PhysicalDeviceGroup& use_physical_device(PhysicalDevice const& physical_device) { physical_device_handles.push_back(&physical_device); return *this; diff --git a/tests/framework/icd/test_icd.cpp b/tests/framework/icd/test_icd.cpp index 389bddc6..cee7a03e 100644 --- a/tests/framework/icd/test_icd.cpp +++ b/tests/framework/icd/test_icd.cpp @@ -209,25 +209,73 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumeratePhysicalDevices(VkInstance instan // VK_SUCCESS,VK_INCOMPLETE, VK_ERROR_INITIALIZATION_FAILED VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumeratePhysicalDeviceGroups( VkInstance instance, uint32_t* pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties* pPhysicalDeviceGroupProperties) { + VkResult result = VK_SUCCESS; + if (pPhysicalDeviceGroupProperties == nullptr) { - *pPhysicalDeviceGroupCount = static_cast(icd.physical_device_groups.size()); + if (0 == icd.physical_device_groups.size()) { + *pPhysicalDeviceGroupCount = static_cast(icd.physical_devices.size()); + } else { + *pPhysicalDeviceGroupCount = static_cast(icd.physical_device_groups.size()); + } } else { - for (size_t device_group = 0; device_group < icd.physical_device_groups.size(); device_group++) { - if (device_group >= *pPhysicalDeviceGroupCount) { - return VK_INCOMPLETE; + // NOTE: This is a fake struct to make sure the pNext chain is properly passed down to the ICD + // vkEnumeratePhysicalDeviceGroups. + // The two versions must match: + // "FakePNext" test in loader_regresion_tests.cpp + // "test_vkEnumeratePhysicalDeviceGroups" in test_icd.cpp + struct FakePnextSharedWithICD { + VkStructureType sType; + void* pNext; + uint32_t value; + }; + + uint32_t group_count = 0; + if (0 == icd.physical_device_groups.size()) { + group_count = icd.physical_devices.size(); + for (size_t device_group = 0; device_group < icd.physical_devices.size(); device_group++) { + if (device_group >= *pPhysicalDeviceGroupCount) { + group_count = *pPhysicalDeviceGroupCount; + result = VK_INCOMPLETE; + break; + } + pPhysicalDeviceGroupProperties[device_group].subsetAllocation = false; + pPhysicalDeviceGroupProperties[device_group].physicalDeviceCount = 1; + pPhysicalDeviceGroupProperties[device_group].physicalDevices[0] = + icd.physical_devices[device_group].vk_physical_device.handle; } - pPhysicalDeviceGroupProperties[device_group].subsetAllocation = - icd.physical_device_groups[device_group].subset_allocation; - uint32_t handles_written = 0; - for (size_t i = 0; i < icd.physical_device_groups[device_group].physical_device_handles.size(); i++) { - handles_written++; - pPhysicalDeviceGroupProperties[device_group].physicalDevices[i] = - icd.physical_device_groups[device_group].physical_device_handles[i]->vk_physical_device.handle; + } else { + group_count = icd.physical_device_groups.size(); + for (size_t device_group = 0; device_group < icd.physical_device_groups.size(); device_group++) { + if (device_group >= *pPhysicalDeviceGroupCount) { + group_count = *pPhysicalDeviceGroupCount; + result = VK_INCOMPLETE; + break; + } + pPhysicalDeviceGroupProperties[device_group].subsetAllocation = + icd.physical_device_groups[device_group].subset_allocation; + uint32_t handles_written = 0; + for (size_t i = 0; i < icd.physical_device_groups[device_group].physical_device_handles.size(); i++) { + handles_written++; + pPhysicalDeviceGroupProperties[device_group].physicalDevices[i] = + icd.physical_device_groups[device_group].physical_device_handles[i]->vk_physical_device.handle; + } + pPhysicalDeviceGroupProperties[device_group].physicalDeviceCount = handles_written; } - pPhysicalDeviceGroupProperties[device_group].physicalDeviceCount = handles_written; } + // NOTE: The following code is purely to test pNext passing in vkEnumeratePhysicalDevice groups + // and includes normally invalid information. + for (size_t device_group = 0; device_group < group_count; device_group++) { + if (nullptr != pPhysicalDeviceGroupProperties[device_group].pNext) { + VkBaseInStructure* base = reinterpret_cast(pPhysicalDeviceGroupProperties[device_group].pNext); + if (base->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTI_DRAW_PROPERTIES_EXT) { + FakePnextSharedWithICD* fake = reinterpret_cast(base); + fake->value = 0xDECAFBAD; + } + } + } + *pPhysicalDeviceGroupCount = static_cast(group_count); } - return VK_SUCCESS; + return result; } VKAPI_ATTR VkResult VKAPI_CALL test_vkCreateDebugUtilsMessengerEXT(VkInstance instance, @@ -962,6 +1010,9 @@ PFN_vkVoidFunction get_instance_func_ver_1_1(VkInstance instance, const char* pN if (string_eq(pName, "test_vkEnumerateInstanceVersion")) { return to_vkVoidFunction(test_vkEnumerateInstanceVersion); } + if (string_eq(pName, "vkEnumeratePhysicalDeviceGroups")) { + return to_vkVoidFunction(test_vkEnumeratePhysicalDeviceGroups); + } } return nullptr; } @@ -1225,8 +1276,10 @@ PFN_vkVoidFunction get_instance_func(VkInstance instance, const char* pName) { if (string_eq(pName, "vkCreateInstance")) return to_vkVoidFunction(test_vkCreateInstance); if (string_eq(pName, "vkDestroyInstance")) return to_vkVoidFunction(test_vkDestroyInstance); if (string_eq(pName, "vkEnumeratePhysicalDevices")) return to_vkVoidFunction(test_vkEnumeratePhysicalDevices); - if (string_eq(pName, "vkEnumeratePhysicalDeviceGroups") || string_eq(pName, "vkEnumeratePhysicalDeviceGroupsKHR")) - return to_vkVoidFunction(test_vkEnumeratePhysicalDeviceGroups); + + if (IsInstanceExtensionEnabled(VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME)) { + if (string_eq(pName, "vkEnumeratePhysicalDeviceGroupsKHR")) return to_vkVoidFunction(test_vkEnumeratePhysicalDeviceGroups); + } PFN_vkVoidFunction ret_phys_dev = get_physical_device_func(instance, pName); if (ret_phys_dev != nullptr) return ret_phys_dev; diff --git a/tests/framework/icd/test_icd.h b/tests/framework/icd/test_icd.h index dbcbb5f3..a55d1520 100644 --- a/tests/framework/icd/test_icd.h +++ b/tests/framework/icd/test_icd.h @@ -1,7 +1,7 @@ /* - * Copyright (c) 2021 The Khronos Group Inc. - * Copyright (c) 2021 Valve Corporation - * Copyright (c) 2021 LunarG, Inc. + * Copyright (c) 2021-2022 The Khronos Group Inc. + * Copyright (c) 2021-2022 Valve Corporation + * Copyright (c) 2021-2022 LunarG, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and/or associated documentation files (the "Materials"), to diff --git a/tests/framework/layer/CMakeLists.txt b/tests/framework/layer/CMakeLists.txt index d5af270a..b65e2e35 100644 --- a/tests/framework/layer/CMakeLists.txt +++ b/tests/framework/layer/CMakeLists.txt @@ -1,6 +1,6 @@ # ~~~ -# Copyright (c) 2021 Valve Corporation -# Copyright (c) 2021 LunarG, Inc. +# Copyright (c) 2021-2022 Valve Corporation +# Copyright (c) 2021-2022 LunarG, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ target_compile_definitions(test_layer_deps INTERFACE target_link_libraries(test_layer_deps INTERFACE testing_framework_util) +# Interface testing layer set(TEST_LAYER_SOURCES test_layer.cpp test_layer.h) set(TEST_LAYER_BASE_EXPORTS TEST_LAYER_EXPORT_ENUMERATE_FUNCTIONS=1) @@ -53,6 +54,48 @@ target_link_libraries(test_layer_export_version_2 PRIVATE test_layer_deps) target_compile_definitions(test_layer_export_version_2 PRIVATE ${TEST_LAYER_VERSION_0_EXPORTS} ${TEST_LAYER_VERSION_1_EXPORTS} ${TEST_LAYER_VERSION_2_EXPORTS} ) +# Physical device list altering layer (add phys dev, remove phys dev, re-order phys devs) +set(PHYSDEV_LAYER_ADD_EXPORTS + ${TEST_LAYER_VERSION_0_EXPORTS} ${TEST_LAYER_VERSION_1_EXPORTS} ${TEST_LAYER_VERSION_2_EXPORTS} + TEST_LAYER_NAME="VkLayer_LunarG_add_phys_dev" + TEST_PHYSDEV_LAYER_ADD=1 + ) +set(PHYSDEV_REMOVE_EXPORTS + ${TEST_LAYER_VERSION_0_EXPORTS} ${TEST_LAYER_VERSION_1_EXPORTS} ${TEST_LAYER_VERSION_2_EXPORTS} + TEST_LAYER_NAME="VkLayer_LunarG_remove_phys_dev" + TEST_PHYSDEV_LAYER_REMOVE=1 + ) +set(PHYSDEV_LAYER_REORDER_EXPORTS + ${TEST_LAYER_VERSION_0_EXPORTS} ${TEST_LAYER_VERSION_1_EXPORTS} ${TEST_LAYER_VERSION_2_EXPORTS} + TEST_LAYER_NAME="VkLayer_LunarG_reorder_phys_dev" + TEST_PHYSDEV_LAYER_REORDER=1 + ) +set(PHYSDEV_LAYER_ALL_EXPORTS + ${TEST_LAYER_VERSION_0_EXPORTS} ${TEST_LAYER_VERSION_1_EXPORTS} ${TEST_LAYER_VERSION_2_EXPORTS} + TEST_LAYER_NAME="VkLayer_LunarG_all_phys_dev" + TEST_PHYSDEV_LAYER_ADD=1 + TEST_PHYSDEV_LAYER_REMOVE=1 + TEST_PHYSDEV_LAYER_REORDER=1 + ) + +add_library(test_layer_physdev_add SHARED ${TEST_LAYER_SOURCES}) +target_link_libraries(test_layer_physdev_add PRIVATE test_layer_deps) +target_compile_definitions(test_layer_physdev_add PRIVATE ${PHYSDEV_LAYER_ADD_EXPORTS}) + +add_library(test_layer_physdev_remove SHARED ${TEST_LAYER_SOURCES}) +target_link_libraries(test_layer_physdev_remove PRIVATE test_layer_deps) +target_compile_definitions(test_layer_physdev_remove PRIVATE ${PHYSDEV_REMOVE_EXPORTS}) + +add_library(test_layer_physdev_reorder SHARED ${TEST_LAYER_SOURCES}) +target_link_libraries(test_layer_physdev_reorder PRIVATE test_layer_deps) +target_compile_definitions(test_layer_physdev_reorder PRIVATE ${PHYSDEV_LAYER_REORDER_EXPORTS}) + +add_library(test_layer_physdev_all SHARED ${TEST_LAYER_SOURCES}) +target_link_libraries(test_layer_physdev_all PRIVATE test_layer_deps) +target_compile_definitions(test_layer_physdev_all PRIVATE ${PHYSDEV_LAYER_ALL_EXPORTS}) + +# Wrap Objects layer which wraps dispatchable handles to test that the loader will properly +# work in the case where handles in the terminator don't match handles in the trampoline portion. set(WRAP_LAYER_VERSION_1_EXPORTS TEST_LAYER_EXPORT_DIRECT_DISP=1 TEST_LAYER_EXPORT_MAINT_1=1 @@ -83,11 +126,16 @@ add_library(test_layer_wrap_objects_3 SHARED wrap_objects) target_link_libraries(test_layer_wrap_objects_3 PRIVATE test_layer_deps) target_compile_definitions(test_layer_wrap_objects_3 PRIVATE ${WRAP_LAYER_VERSION_3_EXPORTS}) +# Windows requires export definitions for these layers if(WIN32) target_sources(test_layer_export_base PRIVATE export_definitions/test_layer_base.def) target_sources(test_layer_export_version_0 PRIVATE export_definitions/test_layer_0.def) target_sources(test_layer_export_version_1 PRIVATE export_definitions/test_layer_1.def) target_sources(test_layer_export_version_2 PRIVATE export_definitions/test_layer_2.def) + target_sources(test_layer_physdev_add PRIVATE export_definitions/test_layer_2.def) + target_sources(test_layer_physdev_remove PRIVATE export_definitions/test_layer_2.def) + target_sources(test_layer_physdev_reorder PRIVATE export_definitions/test_layer_2.def) + target_sources(test_layer_physdev_all PRIVATE export_definitions/test_layer_2.def) target_sources(test_layer_wrap_objects PRIVATE export_definitions/test_layer_wrap_objects.def) target_sources(test_layer_wrap_objects_1 PRIVATE export_definitions/test_layer_wrap_objects.def) target_sources(test_layer_wrap_objects_2 PRIVATE export_definitions/test_layer_wrap_objects.def) diff --git a/tests/framework/layer/test_layer.cpp b/tests/framework/layer/test_layer.cpp index 1efa6a4b..409da350 100644 --- a/tests/framework/layer/test_layer.cpp +++ b/tests/framework/layer/test_layer.cpp @@ -1,7 +1,7 @@ /* - * Copyright (c) 2021 The Khronos Group Inc. - * Copyright (c) 2021 Valve Corporation - * Copyright (c) 2021 LunarG, Inc. + * Copyright (c) 2021-2022 The Khronos Group Inc. + * Copyright (c) 2021-2022 Valve Corporation + * Copyright (c) 2021-2022 LunarG, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and/or associated documentation files (the "Materials"), to @@ -74,6 +74,10 @@ #define LAYER_EXPORT_NEGOTIATE_LOADER_LAYER_INTERFACE_VERSION 0 #endif +#ifndef TEST_LAYER_NAME +#define TEST_LAYER_NAME "VkLayer_LunarG_test_layer" +#endif + TestLayer layer; extern "C" { FRAMEWORK_EXPORT TestLayer* get_test_layer_func() { return &layer; } @@ -107,7 +111,7 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumerateInstanceLayerProperties(uint32_t* VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumerateInstanceExtensionProperties(const char* pLayerName, uint32_t* pPropertyCount, VkExtensionProperties* pProperties) { - if (pLayerName && string_eq(pLayerName, layer.unique_name.c_str())) { + if (pLayerName && string_eq(pLayerName, TEST_LAYER_NAME)) { *pPropertyCount = 0; return VK_SUCCESS; } @@ -122,7 +126,7 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumerateDeviceLayerProperties(VkPhysicalD VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumerateDeviceExtensionProperties(VkPhysicalDevice physicalDevice, const char* pLayerName, uint32_t* pPropertyCount, VkExtensionProperties* pProperties) { - if (pLayerName && string_eq(pLayerName, layer.unique_name.c_str())) { + if (pLayerName && string_eq(pLayerName, TEST_LAYER_NAME)) { *pPropertyCount = 0; return VK_SUCCESS; } @@ -171,7 +175,7 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vkCreateInstance(const VkInstanceCreateInfo* } VKAPI_ATTR VkResult VKAPI_CALL test_override_vkCreateInstance(const VkInstanceCreateInfo* pCreateInfo, - const VkAllocationCallbacks* pAllocator, VkInstance* pInstance) { + const VkAllocationCallbacks* pAllocator, VkInstance* pInstance) { return VK_ERROR_INVALID_SHADER_NV; } @@ -214,6 +218,195 @@ VKAPI_ATTR VkResult VKAPI_CALL test_vkCreateDevice(VkPhysicalDevice physicalDevi return result; } +VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumeratePhysicalDevices(VkInstance instance, uint32_t* pPhysicalDeviceCount, + VkPhysicalDevice* pPhysicalDevices) { +#if !TEST_PHYSDEV_LAYER_REMOVE && !TEST_PHYSDEV_LAYER_ADD && !TEST_PHYSDEV_LAYER_REORDER + return layer.instance_dispatch_table.EnumeratePhysicalDevices(instance, pPhysicalDeviceCount, pPhysicalDevices); +#else // TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER + VkResult res = VK_SUCCESS; + + if (layer.complete_physical_devices.size() == 0) { + // Get list of all physical devices from lower down + // NOTE: This only works if we don't test changing the number of devices + // underneath us when using this test. + uint32_t icd_count = 0; + layer.instance_dispatch_table.EnumeratePhysicalDevices(instance, &icd_count, nullptr); + std::vector tmp_vector; + tmp_vector.resize(icd_count); + layer.instance_dispatch_table.EnumeratePhysicalDevices(instance, &icd_count, tmp_vector.data()); + layer.complete_physical_devices.clear(); + +#if TEST_PHYSDEV_LAYER_REMOVE + // Erase the 3rd and 4th items + layer.removed_physical_devices.push_back(tmp_vector[3]); + layer.removed_physical_devices.push_back(tmp_vector[4]); + tmp_vector.erase(tmp_vector.begin() + 3); + tmp_vector.erase(tmp_vector.begin() + 3); +#endif // TEST_PHYSDEV_LAYER_REMOVE + +#if TEST_PHYSDEV_LAYER_ADD + // Insert a new device in the beginning, middle, and end + uint32_t middle = tmp_vector.size() / 2; + VkPhysicalDevice new_phys_dev = reinterpret_cast((size_t)(0xABCD0000)); + layer.added_physical_devices.push_back(new_phys_dev); + tmp_vector.insert(tmp_vector.begin(), new_phys_dev); + new_phys_dev = reinterpret_cast((size_t)(0xBADC0000)); + layer.added_physical_devices.push_back(new_phys_dev); + tmp_vector.insert(tmp_vector.begin() + middle, new_phys_dev); + new_phys_dev = reinterpret_cast((size_t)(0xDCBA0000)); + layer.added_physical_devices.push_back(new_phys_dev); + tmp_vector.push_back(new_phys_dev); +#endif // TEST_PHYSDEV_LAYER_ADD + +#if TEST_PHYSDEV_LAYER_REORDER + // Flip the order of items + for (int32_t dev = tmp_vector.size() - 1; dev >= 0; --dev) { + layer.complete_physical_devices.push_back(tmp_vector[dev]); + } +#else // !TEST_PHYSDEV_LAYER_REORDER + // Otherwise, keep the order the same + for (uint32_t dev = 0; dev < tmp_vector.size(); ++dev) { + layer.complete_physical_devices.push_back(tmp_vector[dev]); + } +#endif // !TEST_PHYSDEV_LAYER_REORDER + } + + if (nullptr == pPhysicalDevices) { + *pPhysicalDeviceCount = layer.complete_physical_devices.size(); + } else { + uint32_t adj_count = layer.complete_physical_devices.size(); + if (*pPhysicalDeviceCount < adj_count) { + adj_count = *pPhysicalDeviceCount; + res = VK_INCOMPLETE; + } + for (uint32_t dev = 0; dev < adj_count; ++dev) { + pPhysicalDevices[dev] = layer.complete_physical_devices[dev]; + } + *pPhysicalDeviceCount = adj_count; + } + + return res; +#endif // TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER +} + +VKAPI_ATTR void VKAPI_CALL test_vkGetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice, + VkPhysicalDeviceProperties* pProperties) { +#if TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER + if (std::find(layer.removed_physical_devices.begin(), layer.removed_physical_devices.end(), physicalDevice) != + layer.removed_physical_devices.end()) { + // Should not get here since the application should not know about those devices + assert(false); + } else if (std::find(layer.added_physical_devices.begin(), layer.added_physical_devices.end(), physicalDevice) != + layer.added_physical_devices.end()) { + // Added device so put in some placeholder info we can test against + pProperties->apiVersion = VK_API_VERSION_1_2; + pProperties->driverVersion = VK_MAKE_API_VERSION(0, 12, 14, 196); + pProperties->vendorID = 0xDECAFBAD; + pProperties->deviceID = 0xDEADBADD; +#if defined(_WIN32) + strncpy_s(pProperties->deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, "physdev_added_xx", 17); +#else + strncpy(pProperties->deviceName, "physdev_added_xx", VK_MAX_PHYSICAL_DEVICE_NAME_SIZE); +#endif + } else { +#else // !TEST_PHYSDEV_LAYER_REMOVE && !TEST_PHYSDEV_LAYER_ADD && !TEST_PHYSDEV_LAYER_REORDER + { +#endif + // Not an affected device so just return + layer.instance_dispatch_table.GetPhysicalDeviceProperties(physicalDevice, pProperties); + } +} + +VKAPI_ATTR VkResult VKAPI_CALL test_vkEnumeratePhysicalDeviceGroups( + VkInstance instance, uint32_t* pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties* pPhysicalDeviceGroupProperties) { +#if !TEST_PHYSDEV_LAYER_REMOVE && !TEST_PHYSDEV_LAYER_ADD && !TEST_PHYSDEV_LAYER_REORDER + return layer.instance_dispatch_table.EnumeratePhysicalDeviceGroups(instance, pPhysicalDeviceGroupCount, + pPhysicalDeviceGroupProperties); +#else // TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER + VkResult res = VK_SUCCESS; + + if (layer.complete_physical_device_groups.size() == 0) { + uint32_t fake_count = 1000; + // Call EnumerateDevices to add remove devices as needed + test_vkEnumeratePhysicalDevices(instance, &fake_count, nullptr); + + // Get list of all physical devices from lower down + // NOTE: This only works if we don't test changing the number of devices + // underneath us when using this test. + uint32_t icd_group_count = 0; + layer.instance_dispatch_table.EnumeratePhysicalDeviceGroups(instance, &icd_group_count, nullptr); + std::vector tmp_vector(icd_group_count); + for (uint32_t group = 0; group < icd_group_count; ++group) { + tmp_vector[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES; + } + layer.instance_dispatch_table.EnumeratePhysicalDeviceGroups(instance, &icd_group_count, tmp_vector.data()); + layer.complete_physical_device_groups.clear(); + +#if TEST_PHYSDEV_LAYER_REMOVE + // Now, if a device has been removed, and it was the only group, we need to remove the group as well. + for (uint32_t rem_dev = 0; rem_dev < layer.removed_physical_devices.size(); ++rem_dev) { + for (uint32_t group = 0; group < icd_group_count; ++group) { + for (uint32_t grp_dev = 0; grp_dev < tmp_vector[group].physicalDeviceCount; ++grp_dev) { + if (tmp_vector[group].physicalDevices[grp_dev] == layer.removed_physical_devices[rem_dev]) { + for (uint32_t cp_item = grp_dev + 1; cp_item < tmp_vector[group].physicalDeviceCount; ++cp_item) { + tmp_vector[group].physicalDevices[grp_dev] = tmp_vector[group].physicalDevices[cp_item]; + } + tmp_vector[group].physicalDeviceCount--; + } + } + } + } + for (uint32_t group = 0; group < tmp_vector.size(); ++group) { + if (tmp_vector[group].physicalDeviceCount == 0) { + layer.removed_physical_device_groups.push_back(tmp_vector[group]); + tmp_vector.erase(tmp_vector.begin() + group); + --group; + } + } +#endif // TEST_PHYSDEV_LAYER_REMOVE + +#if TEST_PHYSDEV_LAYER_ADD + // Add a new group for each physical device not associated with a current group + for (uint32_t dev = 0; dev < layer.added_physical_devices.size(); ++dev) { + VkPhysicalDeviceGroupProperties props{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}; + props.physicalDeviceCount = 1; + props.physicalDevices[0] = layer.added_physical_devices[dev]; + tmp_vector.push_back(props); + layer.added_physical_device_groups.push_back(props); + } +#endif // TEST_PHYSDEV_LAYER_ADD + +#if TEST_PHYSDEV_LAYER_REORDER + // Flip the order of items + for (int32_t dev = tmp_vector.size() - 1; dev >= 0; --dev) { + layer.complete_physical_device_groups.push_back(tmp_vector[dev]); + } +#else // !TEST_PHYSDEV_LAYER_REORDER + // Otherwise, keep the order the same + for (uint32_t dev = 0; dev < tmp_vector.size(); ++dev) { + layer.complete_physical_device_groups.push_back(tmp_vector[dev]); + } +#endif // !TEST_PHYSDEV_LAYER_REORDER + } + + if (nullptr == pPhysicalDeviceGroupProperties) { + *pPhysicalDeviceGroupCount = layer.complete_physical_device_groups.size(); + } else { + uint32_t adj_count = layer.complete_physical_device_groups.size(); + if (*pPhysicalDeviceGroupCount < adj_count) { + adj_count = *pPhysicalDeviceGroupCount; + res = VK_INCOMPLETE; + } + for (uint32_t dev = 0; dev < adj_count; ++dev) { + pPhysicalDeviceGroupProperties[dev] = layer.complete_physical_device_groups[dev]; + } + *pPhysicalDeviceGroupCount = adj_count; + } + + return res; +#endif // TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER +} + // device functions VKAPI_ATTR void VKAPI_CALL test_vkDestroyDevice(VkDevice device, const VkAllocationCallbacks* pAllocator) { @@ -244,6 +437,9 @@ VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL get_instance_func(VkInstance instance, if (string_eq(pName, "vkEnumerateDeviceLayerProperties")) return to_vkVoidFunction(test_vkEnumerateDeviceLayerProperties); if (string_eq(pName, "vkEnumerateDeviceExtensionProperties")) return to_vkVoidFunction(test_vkEnumerateDeviceExtensionProperties); + if (string_eq(pName, "vkEnumeratePhysicalDevices")) return to_vkVoidFunction(test_vkEnumeratePhysicalDevices); + if (string_eq(pName, "vkEnumeratePhysicalDeviceGroups")) return to_vkVoidFunction(test_vkEnumeratePhysicalDeviceGroups); + if (string_eq(pName, "vkGetPhysicalDeviceProperties")) return to_vkVoidFunction(test_vkGetPhysicalDeviceProperties); if (string_eq(pName, "vkCreateInstance")) return to_vkVoidFunction(test_vkCreateInstance); if (string_eq(pName, "vkDestroyInstance")) return to_vkVoidFunction(test_vkDestroyInstance); if (string_eq(pName, "vkCreateDevice")) return to_vkVoidFunction(test_vkCreateDevice); @@ -326,13 +522,27 @@ FRAMEWORK_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumerateDeviceExtensionProper VkExtensionProperties* pProperties) { return test_vkEnumerateDeviceExtensionProperties(physicalDevice, pLayerName, pPropertyCount, pProperties); } +FRAMEWORK_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDevices(VkInstance instance, uint32_t* pPhysicalDeviceCount, + VkPhysicalDevice* pPhysicalDevices) { + return test_vkEnumeratePhysicalDevices(instance, pPhysicalDeviceCount, pPhysicalDevices); +} +FRAMEWORK_EXPORT VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice, + VkPhysicalDeviceProperties* pProperties) { + return test_vkGetPhysicalDeviceProperties(physicalDevice, pProperties); +} +FRAMEWORK_EXPORT VKAPI_ATTR VkResult VKAPI_CALL vkEnumeratePhysicalDeviceGroups( + VkInstance instance, uint32_t* pPhysicalDeviceGroupCount, VkPhysicalDeviceGroupProperties* pPhysicalDeviceGroupProperties) { + return test_vkEnumeratePhysicalDeviceGroups(instance, pPhysicalDeviceGroupCount, pPhysicalDeviceGroupProperties); +} + #endif #if TEST_LAYER_EXPORT_LAYER_NAMED_GIPA FRAMEWORK_EXPORT VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL test_layer_GetInstanceProcAddr(VkInstance instance, const char* pName) { return get_instance_func(instance, pName); } -FRAMEWORK_EXPORT VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL test_override_vkGetInstanceProcAddr(VkInstance instance, const char* pName) { +FRAMEWORK_EXPORT VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL test_override_vkGetInstanceProcAddr(VkInstance instance, + const char* pName) { if (string_eq(pName, "vkCreateInstance")) return to_vkVoidFunction(test_override_vkCreateInstance); return get_instance_func(instance, pName); } diff --git a/tests/framework/layer/test_layer.h b/tests/framework/layer/test_layer.h index 6c421f4d..2944a55d 100644 --- a/tests/framework/layer/test_layer.h +++ b/tests/framework/layer/test_layer.h @@ -1,7 +1,7 @@ /* - * Copyright (c) 2021 The Khronos Group Inc. - * Copyright (c) 2021 Valve Corporation - * Copyright (c) 2021 LunarG, Inc. + * Copyright (c) 2021-2022 The Khronos Group Inc. + * Copyright (c) 2021-2022 Valve Corporation + * Copyright (c) 2021-2022 LunarG, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and/or associated documentation files (the "Materials"), to @@ -80,6 +80,21 @@ Interface Version 2 // Added manifest version 1.1.0 +// vkEnumeratePhysicalDevices and vkEnumeratePhysicalDeviceGroups add a new item +#ifndef TEST_PHYSDEV_LAYER_ADD +#define TEST_PHYSDEV_LAYER_ADD 0 +#endif + +// vkEnumeratePhysicalDevices and vkEnumeratePhysicalDeviceGroups remove an item +#ifndef TEST_PHYSDEV_LAYER_REMOVE +#define TEST_PHYSDEV_LAYER_REMOVE 0 +#endif + +// vkEnumeratePhysicalDevices and vkEnumeratePhysicalDeviceGroups reorders items +#ifndef TEST_PHYSDEV_LAYER_REORDER +#define TEST_PHYSDEV_LAYER_REORDER 0 +#endif + struct TestLayer; // Callbacks allow tests to implement custom functionality without modifying the layer binary @@ -94,7 +109,6 @@ struct TestLayer { BUILDER_VALUE(TestLayer, bool, is_meta_layer, false) - BUILDER_VALUE(TestLayer, std::string, unique_name, {}) BUILDER_VALUE(TestLayer, uint32_t, api_version, VK_API_VERSION_1_0) BUILDER_VALUE(TestLayer, uint32_t, reported_layer_props, 1) BUILDER_VALUE(TestLayer, uint32_t, reported_extension_props, 1) @@ -119,6 +133,14 @@ struct TestLayer { BUILDER_VALUE(TestLayer, std::function, create_instance_callback, {}) // Called in vkCreateDevice after calling down the chain & returning BUILDER_VALUE(TestLayer, std::function, create_device_callback, {}) +#if TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER + BUILDER_VECTOR(TestLayer, VkPhysicalDevice, complete_physical_devices, complete_physical_device) + BUILDER_VECTOR(TestLayer, VkPhysicalDevice, removed_physical_devices, removed_physical_device) + BUILDER_VECTOR(TestLayer, VkPhysicalDevice, added_physical_devices, added_physical_device) + BUILDER_VECTOR(TestLayer, VkPhysicalDeviceGroupProperties, complete_physical_device_groups, complete_physical_device_group) + BUILDER_VECTOR(TestLayer, VkPhysicalDeviceGroupProperties, removed_physical_device_groups, removed_physical_device_group) + BUILDER_VECTOR(TestLayer, VkPhysicalDeviceGroupProperties, added_physical_device_groups, added_physical_device_group) +#endif // TEST_PHYSDEV_LAYER_REMOVE || TEST_PHYSDEV_LAYER_ADD || TEST_PHYSDEV_LAYER_REORDER PFN_vkGetInstanceProcAddr next_vkGetInstanceProcAddr = VK_NULL_HANDLE; PFN_GetPhysicalDeviceProcAddr next_GetPhysicalDeviceProcAddr = VK_NULL_HANDLE; diff --git a/tests/framework/layer/wrap_objects.cpp b/tests/framework/layer/wrap_objects.cpp index a8b99fe2..f78fab1a 100644 --- a/tests/framework/layer/wrap_objects.cpp +++ b/tests/framework/layer/wrap_objects.cpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2015-2021 Valve Corporation - * Copyright (c) 2015-2021 LunarG, Inc. + * Copyright (c) 2015-2022 Valve Corporation + * Copyright (c) 2015-2022 LunarG, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -425,14 +425,25 @@ VKAPI_ATTR VkResult VKAPI_CALL wrap_vkEnumerateDeviceExtensionProperties(VkPhysi ext_count = 0; #if TEST_LAYER_EXPORT_MAINT_1 if (ext_count < count) { +#if defined(_WIN32) + strncpy_s(pProperties[ext_count].extensionName, VK_MAX_EXTENSION_NAME_SIZE, VK_KHR_MAINTENANCE1_EXTENSION_NAME, + strlen(VK_KHR_MAINTENANCE1_EXTENSION_NAME) + 1); +#else strcpy(pProperties[ext_count].extensionName, VK_KHR_MAINTENANCE1_EXTENSION_NAME); +#endif pProperties[ext_count].specVersion = 2; ext_count++; } #endif #if TEST_LAYER_EXPORT_PRESENT_IMAGE if (ext_count < count) { +#if defined(_WIN32) + strncpy_s(pProperties[ext_count].extensionName, VK_MAX_EXTENSION_NAME_SIZE, + VK_KHR_SHARED_PRESENTABLE_IMAGE_EXTENSION_NAME, + strlen(VK_KHR_SHARED_PRESENTABLE_IMAGE_EXTENSION_NAME) + 1); +#else strcpy(pProperties[ext_count].extensionName, VK_KHR_SHARED_PRESENTABLE_IMAGE_EXTENSION_NAME); +#endif pProperties[ext_count].specVersion = 1; ext_count++; } diff --git a/tests/loader_debug_ext_tests.cpp b/tests/loader_debug_ext_tests.cpp index 411b13b7..e9c297c3 100644 --- a/tests/loader_debug_ext_tests.cpp +++ b/tests/loader_debug_ext_tests.cpp @@ -782,7 +782,7 @@ TEST_F(SeparateMessenger, InfoInEnumDevsIgnoredSeverity) { } // Test debug utils created outside of vkCreateInstance with info in vkEnumeratePhysicalDevices. -TEST_F(SeparateMessenger, DebugUtilsInfoInEnumDevs) { +TEST_F(SeparateMessenger, InfoInEnumDevs) { expected_message = "Trimming device count down by application request"; expected_object_type = VK_OBJECT_TYPE_INSTANCE; expected_message_flags = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT; diff --git a/tests/loader_layer_tests.cpp b/tests/loader_layer_tests.cpp index 969508ba..8db22808 100644 --- a/tests/loader_layer_tests.cpp +++ b/tests/loader_layer_tests.cpp @@ -1,7 +1,7 @@ /* - * Copyright (c) 2021 The Khronos Group Inc. - * Copyright (c) 2021 Valve Corporation - * Copyright (c) 2021 LunarG, Inc. + * Copyright (c) 2021-2022 The Khronos Group Inc. + * Copyright (c) 2021-2022 Valve Corporation + * Copyright (c) 2021-2022 LunarG, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and/or associated documentation files (the "Materials"), to @@ -45,6 +45,7 @@ class LayerExtensions : public LayerTests {}; class MetaLayers : public LayerTests {}; class OverrideMetaLayer : public LayerTests {}; class LayerCreateInstance : public LayerTests {}; +class LayerPhysDeviceMod : public LayerTests {}; void CheckLogForLayerString(FrameworkEnvironment& env, const char* implicit_layer_name, bool check_for_enable) { { @@ -2776,3 +2777,544 @@ TEST(TestLayers, DeviceLayerNotPresent) { dev.create_info.add_layer(explicit_layer_name); dev.CheckCreate(phys_dev); } + +TEST_F(LayerPhysDeviceMod, AddPhysicalDevices) { + FrameworkEnvironment env; + env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{} + .set_name("VkLayer_LunarG_add_phys_dev") + .set_lib_path(TEST_LAYER_PHYSDEV_ADD) + .set_api_version(VK_API_VERSION_1_1) + .set_disable_environment("TEST_DISABLE_ADD_PHYS_DEV")), + "test_layer_remove.json"); + + for (uint32_t icd = 0; icd < 2; ++icd) { + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2)); + auto& cur_icd = env.get_test_icd(icd); + cur_icd.icd_api_version = VK_API_VERSION_1_2; + VkPhysicalDeviceProperties properties{}; + properties.apiVersion = VK_API_VERSION_1_2; + properties.vendorID = 0x11000000 + (icd << 6); + char vendor_char = 'a' + icd; + for (uint32_t dev = 0; dev < 3; ++dev) { + properties.deviceID = properties.vendorID + dev; + properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + char dev_char = '0' + dev; + std::string dev_name = "physdev_"; + dev_name += vendor_char; + dev_name += "_"; + dev_name += dev_char; +#if defined(_WIN32) + strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1); +#else + strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE); +#endif + cur_icd.add_physical_device({}); + cur_icd.physical_devices.back().set_properties(properties); + } + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]); + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]); + cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]); + } + const uint32_t icd_devices = 6; + + InstWrapper inst{env.vulkan_functions}; + inst.create_info.set_api_version(VK_API_VERSION_1_1); + inst.CheckCreate(); + + uint32_t dev_count = 0; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &dev_count, nullptr)); + ASSERT_GT(dev_count, icd_devices); + + auto not_exp_physical_devices = std::vector(icd_devices); + uint32_t returned_phys_dev_count = icd_devices; + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, not_exp_physical_devices.data())); + ASSERT_EQ(icd_devices, returned_phys_dev_count); + + auto physical_devices = std::vector(dev_count); + returned_phys_dev_count = dev_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data())); + ASSERT_EQ(dev_count, returned_phys_dev_count); + + uint32_t diff_count = dev_count - icd_devices; + uint32_t found_incomplete = 0; + uint32_t found_added_count = 0; + for (uint32_t dev = 0; dev < dev_count; ++dev) { + VkPhysicalDeviceProperties props{}; + inst->vkGetPhysicalDeviceProperties(physical_devices[dev], &props); + if (string_eq(props.deviceName, "physdev_added_xx")) { + found_added_count++; + } + for (uint32_t incomp = 0; incomp < icd_devices; ++incomp) { + if (not_exp_physical_devices[incomp] == physical_devices[dev]) { + found_incomplete++; + break; + } + } + } + + // We should have added the number of diff items, and the incomplete count should match the number of + // original physical devices. + ASSERT_EQ(found_added_count, diff_count); + ASSERT_EQ(found_incomplete, icd_devices); +} + +TEST_F(LayerPhysDeviceMod, RemovePhysicalDevices) { + FrameworkEnvironment env; + env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{} + .set_name("VkLayer_LunarG_remove_phys_dev") + .set_lib_path(TEST_LAYER_PHYSDEV_REMOVE) + .set_api_version(VK_API_VERSION_1_1) + .set_disable_environment("TEST_DISABLE_REMOVE_PHYS_DEV")), + "test_layer_remove.json"); + + for (uint32_t icd = 0; icd < 2; ++icd) { + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2)); + auto& cur_icd = env.get_test_icd(icd); + cur_icd.icd_api_version = VK_API_VERSION_1_2; + VkPhysicalDeviceProperties properties{}; + properties.apiVersion = VK_API_VERSION_1_2; + properties.vendorID = 0x11000000 + (icd << 6); + char vendor_char = 'a' + icd; + for (uint32_t dev = 0; dev < 3; ++dev) { + properties.deviceID = properties.vendorID + dev; + properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + char dev_char = '0' + dev; + std::string dev_name = "physdev_"; + dev_name += vendor_char; + dev_name += "_"; + dev_name += dev_char; +#if defined(_WIN32) + strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1); +#else + strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE); +#endif + cur_icd.add_physical_device({}); + cur_icd.physical_devices.back().set_properties(properties); + } + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]); + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]); + cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]); + } + const uint32_t icd_devices = 6; + + InstWrapper inst{env.vulkan_functions}; + inst.create_info.set_api_version(VK_API_VERSION_1_1); + inst.CheckCreate(); + + uint32_t dev_count = 0; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &dev_count, nullptr)); + ASSERT_LT(dev_count, icd_devices); + + auto physical_devices = std::vector(dev_count); + uint32_t returned_phys_dev_count = dev_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data())); + ASSERT_EQ(dev_count, returned_phys_dev_count); +} + +TEST_F(LayerPhysDeviceMod, ReorderPhysicalDevices) { + FrameworkEnvironment env; + env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{} + .set_name("VkLayer_LunarG_reorder_phys_dev") + .set_lib_path(TEST_LAYER_PHYSDEV_REORDER) + .set_api_version(VK_API_VERSION_1_1) + .set_disable_environment("TEST_DISABLE_REORDER_PHYS_DEV")), + "test_layer_reorder.json"); + + for (uint32_t icd = 0; icd < 2; ++icd) { + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2)); + auto& cur_icd = env.get_test_icd(icd); + cur_icd.icd_api_version = VK_API_VERSION_1_2; + VkPhysicalDeviceProperties properties{}; + properties.apiVersion = VK_API_VERSION_1_2; + properties.vendorID = 0x11000000 + (icd << 6); + char vendor_char = 'a' + icd; + for (uint32_t dev = 0; dev < 3; ++dev) { + properties.deviceID = properties.vendorID + dev; + properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + char dev_char = '0' + dev; + std::string dev_name = "physdev_"; + dev_name += vendor_char; + dev_name += "_"; + dev_name += dev_char; +#if defined(_WIN32) + strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1); +#else + strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE); +#endif + cur_icd.add_physical_device({}); + cur_icd.physical_devices.back().set_properties(properties); + } + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]); + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]); + cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]); + } + const uint32_t icd_devices = 6; + + InstWrapper inst{env.vulkan_functions}; + inst.create_info.set_api_version(VK_API_VERSION_1_1); + inst.CheckCreate(); + + uint32_t dev_count = 0; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &dev_count, nullptr)); + ASSERT_EQ(dev_count, icd_devices); + + auto physical_devices = std::vector(dev_count); + uint32_t returned_phys_dev_count = dev_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data())); + ASSERT_EQ(dev_count, returned_phys_dev_count); +} + +TEST_F(LayerPhysDeviceMod, AddRemoveAndReorderPhysicalDevices) { + FrameworkEnvironment env; + env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{} + .set_name("VkLayer_LunarG_all_phys_dev") + .set_lib_path(TEST_LAYER_PHYSDEV_ALL) + .set_api_version(VK_API_VERSION_1_1) + .set_disable_environment("TEST_DISABLE_ALL_PHYS_DEV")), + "test_layer_all.json"); + + for (uint32_t icd = 0; icd < 2; ++icd) { + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2)); + auto& cur_icd = env.get_test_icd(icd); + cur_icd.icd_api_version = VK_API_VERSION_1_2; + VkPhysicalDeviceProperties properties{}; + properties.apiVersion = VK_API_VERSION_1_2; + properties.vendorID = 0x11000000 + (icd << 6); + char vendor_char = 'a' + icd; + for (uint32_t dev = 0; dev < 3; ++dev) { + properties.deviceID = properties.vendorID + dev; + properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + char dev_char = '0' + dev; + std::string dev_name = "physdev_"; + dev_name += vendor_char; + dev_name += "_"; + dev_name += dev_char; +#if defined(_WIN32) + strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1); +#else + strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE); +#endif + cur_icd.add_physical_device({}); + cur_icd.physical_devices.back().set_properties(properties); + } + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]); + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]); + cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]); + } + const uint32_t icd_devices = 6; + + InstWrapper inst{env.vulkan_functions}; + inst.create_info.set_api_version(VK_API_VERSION_1_1); + inst.CheckCreate(); + + uint32_t dev_count = 0; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &dev_count, nullptr)); + ASSERT_GT(dev_count, icd_devices); + + auto physical_devices = std::vector(dev_count); + uint32_t returned_phys_dev_count = dev_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data())); + ASSERT_EQ(dev_count, returned_phys_dev_count); + + uint32_t found_added_count = 0; + for (uint32_t dev = 0; dev < dev_count; ++dev) { + VkPhysicalDeviceProperties props{}; + inst->vkGetPhysicalDeviceProperties(physical_devices[dev], &props); + if (string_eq(props.deviceName, "physdev_added_xx")) { + found_added_count++; + } + } + + // Should see 2 removed, but 3 added so a diff count of 1 + uint32_t diff_count = dev_count - icd_devices; + ASSERT_EQ(1, diff_count); + ASSERT_EQ(found_added_count, 3); +} + +static bool GroupsAreTheSame(VkPhysicalDeviceGroupProperties a, VkPhysicalDeviceGroupProperties b) { + if (a.physicalDeviceCount != b.physicalDeviceCount) { + return false; + } + for (uint32_t dev = 0; dev < a.physicalDeviceCount; ++dev) { + if (a.physicalDevices[dev] != b.physicalDevices[dev]) { + return false; + } + } + return true; +} + +TEST_F(LayerPhysDeviceMod, AddPhysicalDeviceGroups) { + FrameworkEnvironment env; + env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{} + .set_name("VkLayer_LunarG_add_phys_dev") + .set_lib_path(TEST_LAYER_PHYSDEV_ADD) + .set_api_version(VK_API_VERSION_1_1) + .set_disable_environment("TEST_DISABLE_ADD_PHYS_DEV")), + "test_layer_remove.json"); + + for (uint32_t icd = 0; icd < 2; ++icd) { + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2)); + auto& cur_icd = env.get_test_icd(icd); + cur_icd.icd_api_version = VK_API_VERSION_1_2; + VkPhysicalDeviceProperties properties{}; + properties.apiVersion = VK_API_VERSION_1_2; + properties.vendorID = 0x11000000 + (icd << 6); + char vendor_char = 'a' + icd; + for (uint32_t dev = 0; dev < 3; ++dev) { + properties.deviceID = properties.vendorID + dev; + properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + char dev_char = '0' + dev; + std::string dev_name = "physdev_"; + dev_name += vendor_char; + dev_name += "_"; + dev_name += dev_char; +#if defined(_WIN32) + strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1); +#else + strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE); +#endif + cur_icd.add_physical_device({}); + cur_icd.physical_devices.back().set_properties(properties); + } + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]); + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]); + cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]); + } + const uint32_t icd_groups = 4; + + InstWrapper inst{env.vulkan_functions}; + inst.create_info.set_api_version(VK_API_VERSION_1_1); + inst.CheckCreate(); + + uint32_t grp_count = 0; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &grp_count, nullptr)); + ASSERT_GT(grp_count, icd_groups); + + auto not_exp_phys_dev_groups = std::vector(icd_groups); + for (uint32_t group = 0; group < icd_groups; ++group) { + not_exp_phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES; + } + uint32_t returned_group_count = icd_groups; + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, not_exp_phys_dev_groups.data())); + ASSERT_EQ(icd_groups, returned_group_count); + + auto phys_dev_groups = std::vector(grp_count); + for (uint32_t group = 0; group < grp_count; ++group) { + phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES; + } + returned_group_count = grp_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, phys_dev_groups.data())); + ASSERT_EQ(grp_count, returned_group_count); + + uint32_t diff_count = grp_count - icd_groups; + uint32_t found_incomplete = 0; + uint32_t found_added_count = 0; + for (uint32_t grp = 0; grp < grp_count; ++grp) { + // Shortcut, only groups with 1 device could be added in the newly added count + if (1 == phys_dev_groups[grp].physicalDeviceCount) { + for (uint32_t dev = 0; dev < phys_dev_groups[grp].physicalDeviceCount; ++dev) { + VkPhysicalDeviceProperties props{}; + inst->vkGetPhysicalDeviceProperties(phys_dev_groups[grp].physicalDevices[dev], &props); + if (string_eq(props.deviceName, "physdev_added_xx")) { + found_added_count++; + } + } + } + for (uint32_t incomp = 0; incomp < icd_groups; ++incomp) { + if (GroupsAreTheSame(not_exp_phys_dev_groups[incomp], phys_dev_groups[grp])) { + found_incomplete++; + break; + } + } + } + + // We should have added the number of diff items, and the incomplete count should match the number of + // original physical devices. + ASSERT_EQ(found_added_count, diff_count); + ASSERT_EQ(found_incomplete, icd_groups); +} + +TEST_F(LayerPhysDeviceMod, RemovePhysicalDeviceGroups) { + FrameworkEnvironment env; + env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{} + .set_name("VkLayer_LunarG_remove_phys_dev") + .set_lib_path(TEST_LAYER_PHYSDEV_REMOVE) + .set_api_version(VK_API_VERSION_1_1) + .set_disable_environment("TEST_DISABLE_REMOVE_PHYS_DEV")), + "test_layer_remove.json"); + + for (uint32_t icd = 0; icd < 2; ++icd) { + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2)); + auto& cur_icd = env.get_test_icd(icd); + cur_icd.icd_api_version = VK_API_VERSION_1_2; + VkPhysicalDeviceProperties properties{}; + properties.apiVersion = VK_API_VERSION_1_2; + properties.vendorID = 0x11000000 + (icd << 6); + char vendor_char = 'a' + icd; + for (uint32_t dev = 0; dev < 3; ++dev) { + properties.deviceID = properties.vendorID + dev; + properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + char dev_char = '0' + dev; + std::string dev_name = "physdev_"; + dev_name += vendor_char; + dev_name += "_"; + dev_name += dev_char; + strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE); + cur_icd.add_physical_device({}); + cur_icd.physical_devices.back().set_properties(properties); + } + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]); + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]); + cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]); + } + const uint32_t icd_groups = 4; + + InstWrapper inst{env.vulkan_functions}; + inst.create_info.set_api_version(VK_API_VERSION_1_1); + inst.CheckCreate(); + + uint32_t grp_count = 0; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &grp_count, nullptr)); + ASSERT_LT(grp_count, icd_groups); + + auto phys_dev_groups = std::vector(grp_count); + for (uint32_t group = 0; group < grp_count; ++group) { + phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES; + } + uint32_t returned_group_count = grp_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, phys_dev_groups.data())); + ASSERT_EQ(grp_count, returned_group_count); +} + +TEST_F(LayerPhysDeviceMod, ReorderPhysicalDeviceGroups) { + FrameworkEnvironment env; + env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{} + .set_name("VkLayer_LunarG_reorder_phys_dev") + .set_lib_path(TEST_LAYER_PHYSDEV_REORDER) + .set_api_version(VK_API_VERSION_1_1) + .set_disable_environment("TEST_DISABLE_REORDER_PHYS_DEV")), + "test_layer_reorder.json"); + + for (uint32_t icd = 0; icd < 2; ++icd) { + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2)); + auto& cur_icd = env.get_test_icd(icd); + cur_icd.icd_api_version = VK_API_VERSION_1_2; + VkPhysicalDeviceProperties properties{}; + properties.apiVersion = VK_API_VERSION_1_2; + properties.vendorID = 0x11000000 + (icd << 6); + char vendor_char = 'a' + icd; + for (uint32_t dev = 0; dev < 3; ++dev) { + properties.deviceID = properties.vendorID + dev; + properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + char dev_char = '0' + dev; + std::string dev_name = "physdev_"; + dev_name += vendor_char; + dev_name += "_"; + dev_name += dev_char; +#if defined(_WIN32) + strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1); +#else + strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE); +#endif + cur_icd.add_physical_device({}); + cur_icd.physical_devices.back().set_properties(properties); + } + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]); + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]); + cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]); + } + const uint32_t icd_groups = 4; + + InstWrapper inst{env.vulkan_functions}; + inst.create_info.set_api_version(VK_API_VERSION_1_1); + inst.CheckCreate(); + + uint32_t grp_count = 0; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &grp_count, nullptr)); + ASSERT_EQ(grp_count, icd_groups); + + auto phys_dev_groups = std::vector(grp_count); + for (uint32_t group = 0; group < grp_count; ++group) { + phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES; + } + uint32_t returned_group_count = grp_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, phys_dev_groups.data())); + ASSERT_EQ(grp_count, returned_group_count); +} + +TEST_F(LayerPhysDeviceMod, AddRemoveAndReorderPhysicalDeviceGroups) { + FrameworkEnvironment env; + env.add_implicit_layer(ManifestLayer{}.add_layer(ManifestLayer::LayerDescription{} + .set_name("VkLayer_LunarG_all_phys_dev") + .set_lib_path(TEST_LAYER_PHYSDEV_ALL) + .set_api_version(VK_API_VERSION_1_1) + .set_disable_environment("TEST_DISABLE_ALL_PHYS_DEV")), + "test_layer_all.json"); + + for (uint32_t icd = 0; icd < 2; ++icd) { + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2)); + auto& cur_icd = env.get_test_icd(icd); + cur_icd.icd_api_version = VK_API_VERSION_1_2; + VkPhysicalDeviceProperties properties{}; + properties.apiVersion = VK_API_VERSION_1_2; + properties.vendorID = 0x11000000 + (icd << 6); + char vendor_char = 'a' + icd; + for (uint32_t dev = 0; dev < 3; ++dev) { + properties.deviceID = properties.vendorID + dev; + properties.deviceType = VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU; + char dev_char = '0' + dev; + std::string dev_name = "physdev_"; + dev_name += vendor_char; + dev_name += "_"; + dev_name += dev_char; +#if defined(_WIN32) + strncpy_s(properties.deviceName, VK_MAX_PHYSICAL_DEVICE_NAME_SIZE, dev_name.c_str(), dev_name.length() + 1); +#else + strncpy(properties.deviceName, dev_name.c_str(), VK_MAX_PHYSICAL_DEVICE_NAME_SIZE); +#endif + cur_icd.add_physical_device({}); + cur_icd.physical_devices.back().set_properties(properties); + } + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[0]); + cur_icd.physical_device_groups.emplace_back(cur_icd.physical_devices[1]); + cur_icd.physical_device_groups.back().use_physical_device(cur_icd.physical_devices[2]); + } + const uint32_t icd_groups = 4; + + InstWrapper inst{env.vulkan_functions}; + inst.create_info.set_api_version(VK_API_VERSION_1_1); + inst.CheckCreate(); + + uint32_t grp_count = 0; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &grp_count, nullptr)); + ASSERT_GT(grp_count, icd_groups); + + auto phys_dev_groups = std::vector(grp_count); + for (uint32_t group = 0; group < grp_count; ++group) { + phys_dev_groups[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES; + } + uint32_t returned_group_count = grp_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, phys_dev_groups.data())); + ASSERT_EQ(grp_count, returned_group_count); + + uint32_t diff_count = grp_count - icd_groups; + uint32_t found_added_count = 0; + for (uint32_t grp = 0; grp < grp_count; ++grp) { + // Shortcut, only groups with 1 device could be added in the newly added count + if (1 == phys_dev_groups[grp].physicalDeviceCount) { + for (uint32_t dev = 0; dev < phys_dev_groups[grp].physicalDeviceCount; ++dev) { + VkPhysicalDeviceProperties props{}; + inst->vkGetPhysicalDeviceProperties(phys_dev_groups[grp].physicalDevices[dev], &props); + if (string_eq(props.deviceName, "physdev_added_xx")) { + found_added_count++; + } + } + } + } + + // Should see 2 devices removed which should result in 1 group removed and since 3 + // devices were added we should have 3 new groups. So we should have a diff of 2 + // groups and 3 new groups + ASSERT_EQ(2, diff_count); + ASSERT_EQ(found_added_count, 3); +} diff --git a/tests/loader_regression_tests.cpp b/tests/loader_regression_tests.cpp index 4ce8263c..142a9f0e 100644 --- a/tests/loader_regression_tests.cpp +++ b/tests/loader_regression_tests.cpp @@ -385,8 +385,7 @@ TEST_F(EnumeratePhysicalDevices, OneCall) { TEST_F(EnumeratePhysicalDevices, TwoCall) { auto& driver = env->get_test_icd().set_min_icd_interface_version(5); - Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}; // known instance extensions - env->reset_icd().add_instance_extension(first_ext); + driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}); const uint32_t real_device_count = 2; for (uint32_t i = 0; i < real_device_count; i++) { @@ -411,8 +410,7 @@ TEST_F(EnumeratePhysicalDevices, TwoCall) { TEST_F(EnumeratePhysicalDevices, MatchOneAndTwoCallNumbers) { auto& driver = env->get_test_icd(); driver.set_min_icd_interface_version(5); - Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}; // known instance extensions - env->reset_icd().add_instance_extension(first_ext); + driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}); const uint32_t real_device_count = 3; for (uint32_t i = 0; i < real_device_count; i++) { @@ -446,8 +444,7 @@ TEST_F(EnumeratePhysicalDevices, MatchOneAndTwoCallNumbers) { TEST_F(EnumeratePhysicalDevices, TwoCallIncomplete) { auto& driver = env->get_test_icd().set_min_icd_interface_version(5); - Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}; // known instance extensions - env->reset_icd().add_instance_extension(first_ext); + driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}); const uint32_t real_device_count = 2; for (uint32_t i = 0; i < real_device_count; i++) { @@ -469,6 +466,20 @@ TEST_F(EnumeratePhysicalDevices, TwoCallIncomplete) { ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &physical_count, physical.data())); ASSERT_EQ(physical_count, 1); + + physical_count = 2; + std::array physical_2; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &physical_count, physical_2.data())); + + // Verify that the first physical device shows up in the list of the second ones + bool found = false; + for (uint32_t dev = 0; dev < physical_count; ++dev) { + if (physical_2[dev] == physical[0]) { + found = true; + break; + } + } + ASSERT_EQ(true, found); } TEST_F(EnumeratePhysicalDevices, ZeroPhysicalDevices) { @@ -502,10 +513,325 @@ TEST_F(EnumeratePhysicalDevices, ZeroPhysicalDevicesAfterCreateInstance) { inst->vkEnumeratePhysicalDeviceGroups(inst, &physical_device_group_count, &physical_device_group_properties)); } -// LX535 / MI-76: Device layers are deprecated. -// Ensure that no errors occur if a bogus device layer list is passed to vkCreateDevice. -// https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/vkspec.html#extendingvulkan-layers-devicelayerdeprecation -TEST_F(CreateDevice, LayersNotPresent) { +TEST_F(EnumeratePhysicalDevices, CallTwiceNormal) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5); + + for (size_t i = 0; i < 4; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + + InstWrapper inst{env->vulkan_functions}; + inst.CheckCreate(); + + // Call twice in a row and make sure nothing bad happened + uint32_t physical_count = static_cast(driver.physical_devices.size()); + uint32_t returned_physical_count = static_cast(driver.physical_devices.size()); + std::vector physical_device_handles_1 = std::vector(physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_1.data())); + ASSERT_EQ(physical_count, returned_physical_count); + std::vector physical_device_handles_2 = std::vector(physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_2.data())); + ASSERT_EQ(physical_count, returned_physical_count); + // Make sure devices are same between the two + for (uint32_t count = 0; count < driver.physical_devices.size(); ++count) { + ASSERT_EQ(physical_device_handles_1[count], physical_device_handles_2[count]); + } +} + +TEST_F(EnumeratePhysicalDevices, CallTwiceIncompleteOnceNormal) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5); + + for (size_t i = 0; i < 8; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + + InstWrapper inst{env->vulkan_functions}; + inst.CheckCreate(); + + // Query 3, then 5, then all + uint32_t physical_count = static_cast(driver.physical_devices.size()); + uint32_t returned_physical_count = 3; + std::vector physical_device_handles_1 = std::vector(returned_physical_count); + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_1.data())); + ASSERT_EQ(3, returned_physical_count); + returned_physical_count = 5; + std::vector physical_device_handles_2 = std::vector(returned_physical_count); + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_2.data())); + ASSERT_EQ(5, returned_physical_count); + returned_physical_count = physical_count; + std::vector physical_device_handles_3 = std::vector(returned_physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_3.data())); + ASSERT_EQ(physical_count, returned_physical_count); + // Make sure devices are same between the three + for (uint32_t count = 0; count < driver.physical_devices.size(); ++count) { + if (count < physical_device_handles_1.size()) { + ASSERT_EQ(physical_device_handles_1[count], physical_device_handles_3[count]); + } + if (count < physical_device_handles_2.size()) { + ASSERT_EQ(physical_device_handles_2[count], physical_device_handles_3[count]); + } + } +} + +TEST_F(EnumeratePhysicalDevices, CallThriceSuccessReduce) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5); + + for (size_t i = 0; i < 8; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + + InstWrapper inst{env->vulkan_functions}; + inst.CheckCreate(); + + // Query all at first, then 5, then 3 + uint32_t physical_count = static_cast(driver.physical_devices.size()); + uint32_t returned_physical_count = physical_count; + std::vector physical_device_handles_1 = std::vector(physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_1.data())); + ASSERT_EQ(physical_count, returned_physical_count); + returned_physical_count = 5; + std::vector physical_device_handles_2 = std::vector(returned_physical_count); + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_2.data())); + ASSERT_EQ(5, returned_physical_count); + returned_physical_count = 3; + std::vector physical_device_handles_3 = std::vector(returned_physical_count); + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_3.data())); + ASSERT_EQ(3, returned_physical_count); + // Make sure devices are same between the three + for (uint32_t count = 0; count < driver.physical_devices.size(); ++count) { + if (count < physical_device_handles_2.size()) { + ASSERT_EQ(physical_device_handles_2[count], physical_device_handles_1[count]); + } + if (count < physical_device_handles_3.size()) { + ASSERT_EQ(physical_device_handles_3[count], physical_device_handles_1[count]); + } + } +} + +TEST_F(EnumeratePhysicalDevices, CallThriceAddInBetween) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5); + + driver.physical_devices.emplace_back("physical_device_0"); + driver.physical_devices.emplace_back("physical_device_1"); + + InstWrapper inst{env->vulkan_functions}; + inst.CheckCreate(); + + uint32_t physical_count = static_cast(driver.physical_devices.size()); + uint32_t returned_physical_count = physical_count; + std::vector physical_device_handles_1 = std::vector(physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_1.data())); + ASSERT_EQ(physical_count, returned_physical_count); + + driver.physical_devices.emplace_back("physical_device_2"); + driver.physical_devices.emplace_back("physical_device_3"); + + std::vector physical_device_handles_2 = std::vector(returned_physical_count); + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_2.data())); + ASSERT_EQ(physical_count, returned_physical_count); + + physical_count = static_cast(driver.physical_devices.size()); + returned_physical_count = physical_count; + std::vector physical_device_handles_3 = std::vector(returned_physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_3.data())); + ASSERT_EQ(physical_count, returned_physical_count); + // Make sure devices are same between the three + for (uint32_t count = 0; count < physical_device_handles_3.size(); ++count) { + if (count < physical_device_handles_1.size()) { + ASSERT_EQ(physical_device_handles_1[count], physical_device_handles_3[count]); + } + if (count < physical_device_handles_2.size()) { + ASSERT_EQ(physical_device_handles_2[count], physical_device_handles_3[count]); + } + } +} + +TEST_F(EnumeratePhysicalDevices, CallThriceRemoveInBetween) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5); + + for (size_t i = 0; i < 4; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + + InstWrapper inst{env->vulkan_functions}; + inst.CheckCreate(); + + uint32_t physical_count = static_cast(driver.physical_devices.size()); + uint32_t returned_physical_count = physical_count; + std::vector physical_device_handles_1 = std::vector(physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_1.data())); + ASSERT_EQ(physical_count, returned_physical_count); + + // Delete the 2nd physical device + driver.physical_devices.erase(std::next(driver.physical_devices.begin())); + + physical_count = static_cast(driver.physical_devices.size()); + std::vector physical_device_handles_2 = std::vector(returned_physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_2.data())); + ASSERT_EQ(physical_count, returned_physical_count); + physical_device_handles_2.resize(returned_physical_count); + + returned_physical_count = physical_count; + std::vector physical_device_handles_3 = std::vector(returned_physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_device_handles_3.data())); + ASSERT_EQ(physical_count, returned_physical_count); + + // Make sure one has 1 more device that two or three + ASSERT_EQ(physical_device_handles_1.size(), physical_device_handles_2.size() + 1); + ASSERT_EQ(physical_device_handles_1.size(), physical_device_handles_3.size() + 1); + + // Make sure the devices in two and three are all found in one + uint32_t two_found = 0; + uint32_t three_found = 0; + for (uint32_t count = 0; count < physical_device_handles_1.size(); ++count) { + for (uint32_t int_count = 0; int_count < physical_device_handles_2.size(); ++int_count) { + if (physical_device_handles_2[int_count] == physical_device_handles_1[count]) { + two_found++; + break; + } + } + for (uint32_t int_count = 0; int_count < physical_device_handles_3.size(); ++int_count) { + if (physical_device_handles_3[int_count] == physical_device_handles_1[count]) { + three_found++; + break; + } + } + } + ASSERT_EQ(two_found, returned_physical_count); + ASSERT_EQ(three_found, returned_physical_count); +} + +TEST_F(EnumeratePhysicalDevices, MultipleAddRemoves) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5); + + for (size_t i = 0; i < 4; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + std::array, 8> physical_dev_handles; + + InstWrapper inst{env->vulkan_functions}; + inst.CheckCreate(); + + uint32_t physical_count = static_cast(driver.physical_devices.size()); + uint32_t returned_physical_count = physical_count; + physical_dev_handles[0].resize(physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[0].data())); + ASSERT_EQ(physical_count, returned_physical_count); + + // Delete the 2nd physical device (0, 2, 3) + driver.physical_devices.erase(std::next(driver.physical_devices.begin())); + + // Query using old number from last call (4), but it should only return 3 + physical_count = static_cast(driver.physical_devices.size()); + physical_dev_handles[1].resize(returned_physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[1].data())); + ASSERT_EQ(physical_count, returned_physical_count); + physical_dev_handles[1].resize(returned_physical_count); + + // Add two new physical devices to the front (A, B, 0, 2, 3) + driver.physical_devices.emplace(driver.physical_devices.begin(), "physical_device_B"); + driver.physical_devices.emplace(driver.physical_devices.begin(), "physical_device_A"); + + // Query using old number from last call (3), but it should be 5 + physical_count = static_cast(driver.physical_devices.size()); + physical_dev_handles[2].resize(returned_physical_count); + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[2].data())); + ASSERT_EQ(physical_count - 2, returned_physical_count); + physical_dev_handles[2].resize(returned_physical_count); + + // Query again to get all 5 + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, nullptr)); + physical_dev_handles[3].resize(returned_physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[3].data())); + ASSERT_EQ(physical_count, returned_physical_count); + + // Delete last two physical devices (A, B, 0, 2) + driver.physical_devices.pop_back(); + + // Query using old number from last call (5), but it should be 4 + physical_count = static_cast(driver.physical_devices.size()); + physical_dev_handles[4].resize(returned_physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[4].data())); + ASSERT_EQ(physical_count, returned_physical_count); + physical_dev_handles[4].resize(returned_physical_count); + // Adjust size and query again, should be the same + physical_dev_handles[5].resize(returned_physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[5].data())); + + // Insert a new physical device (A, B, C, 0, 2) + driver.physical_devices.insert(driver.physical_devices.begin() + 2, "physical_device_C"); + + // Query using old number from last call (4), but it should be 5 + physical_count = static_cast(driver.physical_devices.size()); + physical_dev_handles[6].resize(returned_physical_count); + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[6].data())); + ASSERT_EQ(physical_count - 1, returned_physical_count); + // Query again to get all 5 + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, nullptr)); + physical_dev_handles[7].resize(returned_physical_count); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_physical_count, physical_dev_handles[7].data())); + + // Check final results + // One [4] - 0, 1, 2, 3 + // Two [3] - 0, 2, 3 + // Three [3] - A, B, 0 + // Four [5] - A, B, 0, 2, 3 + // Five [4] - A, B, 0, 2 + // Six [4] - A, B, 0, 2 + // Seven [4] - A, B, C, 0 + // Eight [5] - A, B, C, 0, 2 + ASSERT_EQ(4, physical_dev_handles[0].size()); + ASSERT_EQ(3, physical_dev_handles[1].size()); + ASSERT_EQ(3, physical_dev_handles[2].size()); + ASSERT_EQ(5, physical_dev_handles[3].size()); + ASSERT_EQ(4, physical_dev_handles[4].size()); + ASSERT_EQ(4, physical_dev_handles[5].size()); + ASSERT_EQ(4, physical_dev_handles[6].size()); + ASSERT_EQ(5, physical_dev_handles[7].size()); + + // Make sure the devices in two and three are all found in one + uint32_t found_items[8]{}; + for (uint32_t handle = 1; handle < 8; ++handle) { + for (uint32_t count = 0; count < physical_dev_handles[0].size(); ++count) { + for (uint32_t int_count = 0; int_count < physical_dev_handles[handle].size(); ++int_count) { + if (physical_dev_handles[handle][int_count] == physical_dev_handles[0][count]) { + found_items[handle]++; + break; + } + } + } + } + // Items matching from first call (must be >= since handle re-use does occur) + ASSERT_EQ(found_items[1], 3); + ASSERT_GE(found_items[2], 1); + ASSERT_GE(found_items[3], 3); + ASSERT_GE(found_items[4], 2); + ASSERT_GE(found_items[5], 2); + ASSERT_GE(found_items[6], 1); + ASSERT_GE(found_items[7], 2); + + memset(found_items, 0, 8 * sizeof(uint32_t)); + for (uint32_t handle = 0; handle < 7; ++handle) { + for (uint32_t count = 0; count < physical_dev_handles[7].size(); ++count) { + for (uint32_t int_count = 0; int_count < physical_dev_handles[handle].size(); ++int_count) { + if (physical_dev_handles[handle][int_count] == physical_dev_handles[7][count]) { + found_items[handle]++; + break; + } + } + } + } + // Items matching from last call (must be >= since handle re-use does occur) + ASSERT_GE(found_items[0], 2); + ASSERT_GE(found_items[1], 2); + ASSERT_GE(found_items[2], 3); + ASSERT_GE(found_items[3], 4); + ASSERT_GE(found_items[4], 4); + ASSERT_GE(found_items[5], 4); + ASSERT_GE(found_items[6], 4); +} + +TEST_F(CreateDevice, ExtensionNotPresent) { auto& driver = env->get_test_icd(); MockQueueFamilyProperties family_props{{VK_QUEUE_GRAPHICS_BIT, 1, 0, {1, 1, 1}}, true}; @@ -528,13 +854,15 @@ TEST_F(CreateDevice, LayersNotPresent) { ASSERT_EQ(families, family_props.properties); DeviceWrapper dev{inst}; - DeviceCreateInfo dev_create_info; - dev.create_info.add_layer("NotPresent").add_device_queue(DeviceQueueCreateInfo{}.add_priority(0.0f)); + dev.create_info.add_extension("NotPresent").add_device_queue(DeviceQueueCreateInfo{}.add_priority(0.0f)); - dev.CheckCreate(phys_dev); + dev.CheckCreate(phys_dev, VK_ERROR_EXTENSION_NOT_PRESENT); } -TEST_F(CreateDevice, ExtensionNotPresent) { +// LX535 / MI-76: Device layers are deprecated. +// Ensure that no errors occur if a bogus device layer list is passed to vkCreateDevice. +// https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/vkspec.html#extendingvulkan-layers-devicelayerdeprecation +TEST_F(CreateDevice, LayersNotPresent) { auto& driver = env->get_test_icd(); MockQueueFamilyProperties family_props{{VK_QUEUE_GRAPHICS_BIT, 1, 0, {1, 1, 1}}, true}; @@ -557,9 +885,10 @@ TEST_F(CreateDevice, ExtensionNotPresent) { ASSERT_EQ(families, family_props.properties); DeviceWrapper dev{inst}; - dev.create_info.add_extension("NotPresent").add_device_queue(DeviceQueueCreateInfo{}.add_priority(0.0f)); + DeviceCreateInfo dev_create_info; + dev.create_info.add_layer("NotPresent").add_device_queue(DeviceQueueCreateInfo{}.add_priority(0.0f)); - dev.CheckCreate(phys_dev, VK_ERROR_EXTENSION_NOT_PRESENT); + dev.CheckCreate(phys_dev); } TEST(TryLoadWrongBinaries, WrongICD) { @@ -853,21 +1182,18 @@ TEST(TryLoadWrongBinaries, BadExplicitAndImplicit) { } TEST_F(EnumeratePhysicalDeviceGroups, OneCall) { - auto& driver = env->get_test_icd().set_min_icd_interface_version(5); - Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}; // known instance extensions - env->reset_icd().add_instance_extension(first_ext); - - // ICD contains 2 devices - driver.physical_devices.emplace_back("PhysicalDevice0", 12); - driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); - driver.physical_devices.emplace_back("PhysicalDevice1", 24); - driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); - // ICD contains 1 group, which contains both devices - driver.physical_device_groups.push_back({}); - driver.physical_device_groups.back() - .use_physical_device(driver.physical_devices[0]) - .use_physical_device(driver.physical_devices[1]); - uint32_t physical_device_count = 2; + auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1); + driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}); + + // ICD contains 3 devices in two groups + for (size_t i = 0; i < 3; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i), rand() % 50 + 3); + driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + } + driver.physical_device_groups.emplace_back(driver.physical_devices[0]); + driver.physical_device_groups.back().use_physical_device(driver.physical_devices[1]); + driver.physical_device_groups.emplace_back(driver.physical_devices[2]); + const uint32_t max_physical_device_count = 3; // Core function { @@ -875,58 +1201,89 @@ TEST_F(EnumeratePhysicalDeviceGroups, OneCall) { inst.create_info.set_api_version(VK_API_VERSION_1_1); inst.CheckCreate(); - auto physical_devices = std::vector(physical_device_count); - uint32_t returned_phys_dev_count = physical_device_count; + auto physical_devices = std::vector(max_physical_device_count); + uint32_t returned_phys_dev_count = max_physical_device_count; ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data())); handle_assert_has_values(physical_devices); uint32_t group_count = static_cast(driver.physical_device_groups.size()); uint32_t returned_group_count = group_count; - VkPhysicalDeviceGroupProperties group_props{}; - group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES; - ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, &group_props)); + std::vector group_props{}; + group_props.resize(group_count, VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props.data())); ASSERT_EQ(group_count, returned_group_count); + + // Make sure each physical device shows up in a group, but only once + std::array found{false}; + for (uint32_t group = 0; group < group_count; ++group) { + for (uint32_t g_dev = 0; g_dev < group_props[group].physicalDeviceCount; ++g_dev) { + for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) { + if (physical_devices[dev] == group_props[group].physicalDevices[g_dev]) { + ASSERT_EQ(false, found[dev]); + found[dev] = true; + break; + } + } + } + } + for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) { + ASSERT_EQ(true, found[dev]); + } } - driver.add_instance_extension({"VK_KHR_device_group_creation"}); + driver.add_instance_extension({VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME}); // Extension { InstWrapper inst{env->vulkan_functions}; - inst.create_info.add_extension("VK_KHR_device_group_creation"); + inst.create_info.add_extension(VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME); inst.CheckCreate(); auto vkEnumeratePhysicalDeviceGroupsKHR = reinterpret_cast( env->vulkan_functions.vkGetInstanceProcAddr(inst.inst, "vkEnumeratePhysicalDeviceGroupsKHR")); - auto physical_devices = std::vector(physical_device_count); - uint32_t returned_phys_dev_count = physical_device_count; + auto physical_devices = std::vector(max_physical_device_count); + uint32_t returned_phys_dev_count = max_physical_device_count; ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data())); handle_assert_has_values(physical_devices); uint32_t group_count = static_cast(driver.physical_device_groups.size()); uint32_t returned_group_count = group_count; - VkPhysicalDeviceGroupPropertiesKHR group_props{}; - group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR; - ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, &group_props)); + std::vector group_props{}; + group_props.resize(group_count, VkPhysicalDeviceGroupPropertiesKHR{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR}); + ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, group_props.data())); ASSERT_EQ(group_count, returned_group_count); + + // Make sure each physical device shows up in a group, but only once + std::array found{false}; + for (uint32_t group = 0; group < group_count; ++group) { + for (uint32_t g_dev = 0; g_dev < group_props[group].physicalDeviceCount; ++g_dev) { + for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) { + if (physical_devices[dev] == group_props[group].physicalDevices[g_dev]) { + ASSERT_EQ(false, found[dev]); + found[dev] = true; + break; + } + } + } + } + for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) { + ASSERT_EQ(true, found[dev]); + } } } TEST_F(EnumeratePhysicalDeviceGroups, TwoCall) { - auto& driver = env->get_test_icd().set_min_icd_interface_version(5); - Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}; // known instance extensions - env->reset_icd().add_instance_extension(first_ext); - - // ICD contains 2 devices - driver.physical_devices.emplace_back("PhysicalDevice0", 12); - driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); - driver.physical_devices.emplace_back("PhysicalDevice1", 24); - driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); - // ICD contains 1 group, which contains both devices - driver.physical_device_groups.push_back({}); - driver.physical_device_groups.back() - .use_physical_device(driver.physical_devices[0]) - .use_physical_device(driver.physical_devices[1]); - uint32_t physical_device_count = 2; + auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1); + driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}); + + // ICD contains 3 devices in two groups + for (size_t i = 0; i < 3; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i), rand() % 50 + 3); + driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + } + driver.physical_device_groups.emplace_back(driver.physical_devices[0]); + driver.physical_device_groups.back().use_physical_device(driver.physical_devices[1]); + driver.physical_device_groups.emplace_back(driver.physical_devices[2]); + const uint32_t max_physical_device_count = 3; // Core function { @@ -934,8 +1291,8 @@ TEST_F(EnumeratePhysicalDeviceGroups, TwoCall) { inst.create_info.set_api_version(VK_API_VERSION_1_1); inst.CheckCreate(); - auto physical_devices = std::vector(physical_device_count); - uint32_t returned_phys_dev_count = physical_device_count; + auto physical_devices = std::vector(max_physical_device_count); + uint32_t returned_phys_dev_count = max_physical_device_count; ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data())); handle_assert_has_values(physical_devices); @@ -944,20 +1301,37 @@ TEST_F(EnumeratePhysicalDeviceGroups, TwoCall) { ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, nullptr)); ASSERT_EQ(group_count, returned_group_count); - VkPhysicalDeviceGroupProperties group_props{}; - group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES; - ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, &group_props)); + std::vector group_props{}; + group_props.resize(group_count, VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props.data())); ASSERT_EQ(group_count, returned_group_count); + + // Make sure each physical device shows up in a group, but only once + std::array found{false}; + for (uint32_t group = 0; group < group_count; ++group) { + for (uint32_t g_dev = 0; g_dev < group_props[group].physicalDeviceCount; ++g_dev) { + for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) { + if (physical_devices[dev] == group_props[group].physicalDevices[g_dev]) { + ASSERT_EQ(false, found[dev]); + found[dev] = true; + break; + } + } + } + } + for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) { + ASSERT_EQ(true, found[dev]); + } } - driver.add_instance_extension({"VK_KHR_device_group_creation"}); + driver.add_instance_extension({VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME}); // Extension { InstWrapper inst{env->vulkan_functions}; inst.create_info.add_extension("VK_KHR_device_group_creation"); inst.CheckCreate(); - auto physical_devices = std::vector(physical_device_count); - uint32_t returned_phys_dev_count = physical_device_count; + auto physical_devices = std::vector(max_physical_device_count); + uint32_t returned_phys_dev_count = max_physical_device_count; ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &returned_phys_dev_count, physical_devices.data())); handle_assert_has_values(physical_devices); @@ -969,28 +1343,42 @@ TEST_F(EnumeratePhysicalDeviceGroups, TwoCall) { ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, nullptr)); ASSERT_EQ(group_count, returned_group_count); - VkPhysicalDeviceGroupPropertiesKHR group_props{}; - group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR; - ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, &group_props)); + std::vector group_props{}; + group_props.resize(group_count, VkPhysicalDeviceGroupPropertiesKHR{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR}); + ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, group_props.data())); ASSERT_EQ(group_count, returned_group_count); + + // Make sure each physical device shows up in a group, but only once + std::array found{false}; + for (uint32_t group = 0; group < group_count; ++group) { + for (uint32_t g_dev = 0; g_dev < group_props[group].physicalDeviceCount; ++g_dev) { + for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) { + if (physical_devices[dev] == group_props[group].physicalDevices[g_dev]) { + ASSERT_EQ(false, found[dev]); + found[dev] = true; + break; + } + } + } + } + for (uint32_t dev = 0; dev < max_physical_device_count; ++dev) { + ASSERT_EQ(true, found[dev]); + } } } TEST_F(EnumeratePhysicalDeviceGroups, TwoCallIncomplete) { - auto& driver = env->get_test_icd().set_min_icd_interface_version(5); - Extension first_ext{VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}; // known instance extensions - env->reset_icd().add_instance_extension(first_ext); - - // ICD contains 2 devices - driver.physical_devices.emplace_back("PhysicalDevice0", 12); - driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); - driver.physical_devices.emplace_back("PhysicalDevice1", 24); - driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); - // ICD contains 1 group, which contains both devices - driver.physical_device_groups.push_back({}); - driver.physical_device_groups.back() - .use_physical_device(driver.physical_devices[0]) - .use_physical_device(driver.physical_devices[1]); + auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1); + driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}); + + // ICD contains 3 devices in two groups + for (size_t i = 0; i < 3; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i), rand() % 50 + 3); + driver.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + } + driver.physical_device_groups.emplace_back(driver.physical_devices[0]); + driver.physical_device_groups.back().use_physical_device(driver.physical_devices[1]); + driver.physical_device_groups.emplace_back(driver.physical_devices[2]); // Core function { @@ -1002,15 +1390,32 @@ TEST_F(EnumeratePhysicalDeviceGroups, TwoCallIncomplete) { uint32_t returned_group_count = 0; ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, nullptr)); ASSERT_EQ(group_count, returned_group_count); - returned_group_count = 0; - VkPhysicalDeviceGroupProperties group_props{}; - group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES; - ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, &group_props)); - ASSERT_EQ(0, returned_group_count); - handle_assert_no_values(returned_group_count, group_props.physicalDevices); + returned_group_count = 1; + std::array group_props{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}; + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props.data())); + ASSERT_EQ(1, returned_group_count); + + returned_group_count = 2; + std::array group_props_2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_2.data())); + ASSERT_EQ(2, returned_group_count); + + // Make sure the incomplete group items appear in the complete group + for (uint32_t inc_group = 0; inc_group < 1; ++inc_group) { + bool found = false; + for (uint32_t full_group = 0; full_group < 2; ++full_group) { + if (group_props[inc_group].physicalDeviceCount == group_props_2[full_group].physicalDeviceCount && + group_props[inc_group].physicalDevices[0] == group_props_2[full_group].physicalDevices[0] && + group_props[inc_group].physicalDevices[1] == group_props_2[full_group].physicalDevices[1]) { + found = true; + break; + } + } + ASSERT_EQ(true, found); + } } - driver.add_instance_extension({"VK_KHR_device_group_creation"}); + driver.add_instance_extension({VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME}); // Extension { InstWrapper inst{env->vulkan_functions}; @@ -1024,13 +1429,633 @@ TEST_F(EnumeratePhysicalDeviceGroups, TwoCallIncomplete) { uint32_t returned_group_count = 0; ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, nullptr)); ASSERT_EQ(group_count, returned_group_count); - returned_group_count = 0; - VkPhysicalDeviceGroupPropertiesKHR group_props{}; - group_props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR; - ASSERT_EQ(VK_INCOMPLETE, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, &group_props)); - ASSERT_EQ(0, returned_group_count); - handle_assert_no_values(returned_group_count, group_props.physicalDevices); + returned_group_count = 1; + std::array group_props{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR}; + ASSERT_EQ(VK_INCOMPLETE, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, group_props.data())); + ASSERT_EQ(1, returned_group_count); + + returned_group_count = 2; + std::array group_props_2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR}; + ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, group_props_2.data())); + ASSERT_EQ(2, returned_group_count); + + // Make sure the incomplete group items appear in the complete group + for (uint32_t inc_group = 0; inc_group < 1; ++inc_group) { + bool found = false; + for (uint32_t full_group = 0; full_group < 2; ++full_group) { + if (group_props[inc_group].physicalDeviceCount == group_props_2[full_group].physicalDeviceCount && + group_props[inc_group].physicalDevices[0] == group_props_2[full_group].physicalDevices[0] && + group_props[inc_group].physicalDevices[1] == group_props_2[full_group].physicalDevices[1]) { + found = true; + break; + } + } + ASSERT_EQ(true, found); + } + } +} + +// Call the core vkEnumeratePhysicalDeviceGroups and the extension +// vkEnumeratePhysicalDeviceGroupsKHR, and make sure they return the same info. +TEST_F(EnumeratePhysicalDeviceGroups, TestCoreVersusExtensionSameReturns) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1); + driver.add_instance_extension({VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME}); + driver.add_instance_extension({VK_KHR_DEVICE_GROUP_CREATION_EXTENSION_NAME}); + + // Generate the devices + for (size_t i = 0; i < 6; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + + // Generate the starting groups + driver.physical_device_groups.emplace_back(driver.physical_devices[0]); + driver.physical_device_groups.emplace_back(driver.physical_devices[1]); + driver.physical_device_groups.back() + .use_physical_device(driver.physical_devices[2]) + .use_physical_device(driver.physical_devices[3]); + driver.physical_device_groups.emplace_back(driver.physical_devices[4]); + driver.physical_device_groups.back().use_physical_device(driver.physical_devices[5]); + + uint32_t expected_counts[3] = {1, 3, 2}; + uint32_t core_group_count = 0; + std::vector core_group_props{}; + uint32_t ext_group_count = 0; + std::vector ext_group_props{}; + + InstWrapper inst{env->vulkan_functions}; + inst.create_info.set_api_version(1, 1, 0); + inst.create_info.add_extension("VK_KHR_device_group_creation"); + inst.CheckCreate(); + + // Core function + core_group_count = static_cast(driver.physical_device_groups.size()); + uint32_t returned_group_count = 0; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, nullptr)); + ASSERT_EQ(core_group_count, returned_group_count); + + core_group_props.resize(returned_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, core_group_props.data())); + ASSERT_EQ(core_group_count, returned_group_count); + + auto vkEnumeratePhysicalDeviceGroupsKHR = reinterpret_cast( + env->vulkan_functions.vkGetInstanceProcAddr(inst.inst, "vkEnumeratePhysicalDeviceGroupsKHR")); + + ext_group_count = static_cast(driver.physical_device_groups.size()); + returned_group_count = 0; + ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, nullptr)); + ASSERT_EQ(ext_group_count, returned_group_count); + + ext_group_props.resize(returned_group_count, + VkPhysicalDeviceGroupPropertiesKHR{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES_KHR}); + ASSERT_EQ(VK_SUCCESS, vkEnumeratePhysicalDeviceGroupsKHR(inst, &returned_group_count, ext_group_props.data())); + ASSERT_EQ(ext_group_count, returned_group_count); + + // Make sure data from each matches + ASSERT_EQ(core_group_count, 3); + ASSERT_EQ(ext_group_count, 3); + for (uint32_t group = 0; group < core_group_count; ++group) { + ASSERT_EQ(core_group_props[group].physicalDeviceCount, expected_counts[group]); + ASSERT_EQ(ext_group_props[group].physicalDeviceCount, expected_counts[group]); + for (uint32_t dev = 0; dev < core_group_props[group].physicalDeviceCount; ++dev) { + ASSERT_EQ(core_group_props[group].physicalDevices[dev], ext_group_props[group].physicalDevices[dev]); + } + } + // Make sure no physical device appears in more than one group + for (uint32_t group1 = 0; group1 < core_group_count; ++group1) { + for (uint32_t group2 = group1 + 1; group2 < core_group_count; ++group2) { + for (uint32_t dev1 = 0; dev1 < core_group_props[group1].physicalDeviceCount; ++dev1) { + for (uint32_t dev2 = 0; dev2 < core_group_props[group1].physicalDeviceCount; ++dev2) { + ASSERT_NE(core_group_props[group1].physicalDevices[dev1], core_group_props[group2].physicalDevices[dev2]); + } + } + } + } +} + +// Start with 6 devices in 3 different groups, and then add a group, +// querying vkEnumeratePhysicalDeviceGroups before and after the add. +TEST_F(EnumeratePhysicalDeviceGroups, CallThriceAddGroupInBetween) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1); + + // Generate the devices + for (size_t i = 0; i < 7; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + + // Generate the starting groups + driver.physical_device_groups.emplace_back(driver.physical_devices[0]); + driver.physical_device_groups.emplace_back(driver.physical_devices[1]); + driver.physical_device_groups.back() + .use_physical_device(driver.physical_devices[2]) + .use_physical_device(driver.physical_devices[3]); + driver.physical_device_groups.emplace_back(driver.physical_devices[4]); + driver.physical_device_groups.back().use_physical_device(driver.physical_devices[5]); + + uint32_t before_expected_counts[3] = {1, 3, 2}; + uint32_t after_expected_counts[4] = {1, 3, 1, 2}; + uint32_t before_group_count = 3; + uint32_t after_group_count = 4; + uint32_t returned_group_count = 0; + + InstWrapper inst{env->vulkan_functions}; + inst.create_info.set_api_version(1, 1, 0); + inst.CheckCreate(); + + std::vector group_props_before{}; + group_props_before.resize(before_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + returned_group_count = before_group_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_before.data())); + ASSERT_EQ(before_group_count, returned_group_count); + for (uint32_t group = 0; group < before_group_count; ++group) { + ASSERT_EQ(group_props_before[group].physicalDeviceCount, before_expected_counts[group]); + } + + // Insert new group after first two + driver.physical_device_groups.insert(driver.physical_device_groups.begin() + 2, driver.physical_devices[6]); + + std::vector group_props_after{}; + group_props_after.resize(before_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + ASSERT_EQ(VK_INCOMPLETE, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after.data())); + ASSERT_EQ(before_group_count, returned_group_count); + for (uint32_t group = 0; group < before_group_count; ++group) { + ASSERT_EQ(group_props_after[group].physicalDeviceCount, after_expected_counts[group]); + } + + group_props_after.resize(after_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + returned_group_count = after_group_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after.data())); + ASSERT_EQ(after_group_count, returned_group_count); + for (uint32_t group = 0; group < after_group_count; ++group) { + ASSERT_EQ(group_props_after[group].physicalDeviceCount, after_expected_counts[group]); + } + + // Make sure all devices in the old group info are still found in the new group info + for (uint32_t group1 = 0; group1 < group_props_before.size(); ++group1) { + for (uint32_t group2 = 0; group2 < group_props_after.size(); ++group2) { + if (group_props_before[group1].physicalDeviceCount == group_props_after[group2].physicalDeviceCount) { + uint32_t found_count = 0; + bool found; + for (uint32_t dev1 = 0; dev1 < group_props_before[group1].physicalDeviceCount; ++dev1) { + found = false; + for (uint32_t dev2 = 0; dev2 < group_props_after[group2].physicalDeviceCount; ++dev2) { + if (group_props_before[group1].physicalDevices[dev1] == group_props_after[group2].physicalDevices[dev2]) { + found_count++; + found = true; + break; + } + } + } + ASSERT_EQ(found, found_count == group_props_before[group1].physicalDeviceCount); + } + } + } +} + +// Start with 7 devices in 4 different groups, and then remove a group, +// querying vkEnumeratePhysicalDeviceGroups before and after the remove. +TEST_F(EnumeratePhysicalDeviceGroups, CallTwiceRemoveGroupInBetween) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1); + + // Generate the devices + for (size_t i = 0; i < 7; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + + // Generate the starting groups + driver.physical_device_groups.emplace_back(driver.physical_devices[0]); + driver.physical_device_groups.emplace_back(driver.physical_devices[1]); + driver.physical_device_groups.back() + .use_physical_device(driver.physical_devices[2]) + .use_physical_device(driver.physical_devices[3]); + driver.physical_device_groups.emplace_back(driver.physical_devices[4]); + driver.physical_device_groups.emplace_back(driver.physical_devices[5]); + driver.physical_device_groups.back().use_physical_device(driver.physical_devices[6]); + + uint32_t before_expected_counts[4] = {1, 3, 1, 2}; + uint32_t after_expected_counts[3] = {1, 3, 2}; + uint32_t before_group_count = 4; + uint32_t after_group_count = 3; + uint32_t returned_group_count = 0; + + InstWrapper inst{env->vulkan_functions}; + inst.create_info.set_api_version(1, 1, 0); + inst.CheckCreate(); + + std::vector group_props_before{}; + group_props_before.resize(before_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + returned_group_count = before_group_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_before.data())); + ASSERT_EQ(before_group_count, returned_group_count); + for (uint32_t group = 0; group < before_group_count; ++group) { + ASSERT_EQ(group_props_before[group].physicalDeviceCount, before_expected_counts[group]); + } + + // Insert new group after first two + driver.physical_device_groups.erase(driver.physical_device_groups.begin() + 2); + + std::vector group_props_after{}; + group_props_after.resize(after_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after.data())); + ASSERT_EQ(after_group_count, returned_group_count); + for (uint32_t group = 0; group < after_group_count; ++group) { + ASSERT_EQ(group_props_after[group].physicalDeviceCount, after_expected_counts[group]); + } + + // Make sure all devices in the new group info are found in the old group info + for (uint32_t group1 = 0; group1 < group_props_after.size(); ++group1) { + for (uint32_t group2 = 0; group2 < group_props_before.size(); ++group2) { + if (group_props_after[group1].physicalDeviceCount == group_props_before[group2].physicalDeviceCount) { + uint32_t found_count = 0; + bool found; + for (uint32_t dev1 = 0; dev1 < group_props_after[group1].physicalDeviceCount; ++dev1) { + found = false; + for (uint32_t dev2 = 0; dev2 < group_props_before[group2].physicalDeviceCount; ++dev2) { + if (group_props_after[group1].physicalDevices[dev1] == group_props_before[group2].physicalDevices[dev2]) { + found_count++; + found = true; + break; + } + } + } + ASSERT_EQ(found, found_count == group_props_after[group1].physicalDeviceCount); + } + } + } +} + +// Start with 6 devices in 3 different groups, and then add a device to the middle group, +// querying vkEnumeratePhysicalDeviceGroups before and after the add. +TEST_F(EnumeratePhysicalDeviceGroups, CallTwiceAddDeviceInBetween) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1); + + // Generate the devices + for (size_t i = 0; i < 7; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + + // Generate the starting groups + driver.physical_device_groups.emplace_back(driver.physical_devices[0]); + driver.physical_device_groups.emplace_back(driver.physical_devices[1]); + driver.physical_device_groups.back() + .use_physical_device(driver.physical_devices[2]) + .use_physical_device(driver.physical_devices[3]); + driver.physical_device_groups.emplace_back(driver.physical_devices[4]); + driver.physical_device_groups.back().use_physical_device(driver.physical_devices[5]); + + uint32_t expected_group_count = 3; + uint32_t before_expected_counts[3] = {1, 3, 2}; + uint32_t after_expected_counts[3] = {1, 4, 2}; + uint32_t returned_group_count = 0; + + InstWrapper inst{env->vulkan_functions}; + inst.create_info.set_api_version(1, 1, 0); + inst.CheckCreate(); + + std::vector group_props_before{}; + group_props_before.resize(expected_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + returned_group_count = expected_group_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_before.data())); + ASSERT_EQ(expected_group_count, returned_group_count); + for (uint32_t group = 0; group < expected_group_count; ++group) { + ASSERT_EQ(group_props_before[group].physicalDeviceCount, before_expected_counts[group]); + } + + // Insert new device to 2nd group + driver.physical_device_groups[1].use_physical_device(driver.physical_devices[6]); + + std::vector group_props_after{}; + group_props_after.resize(expected_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after.data())); + ASSERT_EQ(expected_group_count, returned_group_count); + for (uint32_t group = 0; group < expected_group_count; ++group) { + ASSERT_EQ(group_props_after[group].physicalDeviceCount, after_expected_counts[group]); + } + + // Make sure all devices in the old group info are still found in the new group info + for (uint32_t group1 = 0; group1 < group_props_before.size(); ++group1) { + for (uint32_t group2 = 0; group2 < group_props_after.size(); ++group2) { + uint32_t found_count = 0; + bool found; + for (uint32_t dev1 = 0; dev1 < group_props_before[group1].physicalDeviceCount; ++dev1) { + found = false; + for (uint32_t dev2 = 0; dev2 < group_props_after[group2].physicalDeviceCount; ++dev2) { + if (group_props_before[group1].physicalDevices[dev1] == group_props_after[group2].physicalDevices[dev2]) { + found_count++; + found = true; + break; + } + } + } + ASSERT_EQ(found, found_count != 0 && found_count == before_expected_counts[group1]); + if (found) { + break; + } + } + } +} + +// Start with 6 devices in 3 different groups, and then remove a device to the middle group, +// querying vkEnumeratePhysicalDeviceGroups before and after the remove. +TEST_F(EnumeratePhysicalDeviceGroups, CallTwiceRemoveDeviceInBetween) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1); + + // Generate the devices + for (size_t i = 0; i < 6; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + + // Generate the starting groups + driver.physical_device_groups.emplace_back(driver.physical_devices[0]); + driver.physical_device_groups.emplace_back(driver.physical_devices[1]); + driver.physical_device_groups.back() + .use_physical_device(driver.physical_devices[2]) + .use_physical_device(driver.physical_devices[3]); + driver.physical_device_groups.emplace_back(driver.physical_devices[4]); + driver.physical_device_groups.back().use_physical_device(driver.physical_devices[5]); + + uint32_t before_expected_counts[3] = {1, 3, 2}; + uint32_t after_expected_counts[3] = {1, 2, 2}; + uint32_t expected_group_count = 3; + uint32_t returned_group_count = 0; + + InstWrapper inst{env->vulkan_functions}; + inst.create_info.set_api_version(1, 1, 0); + inst.CheckCreate(); + + std::vector group_props_before{}; + group_props_before.resize(expected_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + returned_group_count = expected_group_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_before.data())); + ASSERT_EQ(expected_group_count, returned_group_count); + printf("Before:\n"); + for (uint32_t group = 0; group < expected_group_count; ++group) { + printf(" Group %u:\n", group); + ASSERT_EQ(group_props_before[group].physicalDeviceCount, before_expected_counts[group]); + for (uint32_t dev = 0; dev < group_props_before[group].physicalDeviceCount; ++dev) { + printf(" Dev %u: %p\n", dev, group_props_before[group].physicalDevices[dev]); + } + } + + // Remove middle device in middle group + driver.physical_device_groups[1].physical_device_handles.erase( + driver.physical_device_groups[1].physical_device_handles.begin() + 1); + + std::vector group_props_after{}; + group_props_after.resize(expected_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after.data())); + ASSERT_EQ(expected_group_count, returned_group_count); + printf("After:\n"); + for (uint32_t group = 0; group < expected_group_count; ++group) { + printf(" Group %u:\n", group); + ASSERT_EQ(group_props_after[group].physicalDeviceCount, after_expected_counts[group]); + for (uint32_t dev = 0; dev < group_props_after[group].physicalDeviceCount; ++dev) { + printf(" Dev %u: %p\n", dev, group_props_after[group].physicalDevices[dev]); + } + } + + // Make sure all devices in the new group info are found in the old group info + for (uint32_t group1 = 0; group1 < group_props_after.size(); ++group1) { + for (uint32_t group2 = 0; group2 < group_props_before.size(); ++group2) { + uint32_t found_count = 0; + bool found; + for (uint32_t dev1 = 0; dev1 < group_props_after[group1].physicalDeviceCount; ++dev1) { + found = false; + for (uint32_t dev2 = 0; dev2 < group_props_before[group2].physicalDeviceCount; ++dev2) { + if (group_props_after[group1].physicalDevices[dev1] == group_props_before[group2].physicalDevices[dev2]) { + found_count++; + found = true; + break; + } + } + } + ASSERT_EQ(found, found_count != 0 && found_count == after_expected_counts[group1]); + if (found) { + break; + } + } + } +} + +// Start with 9 devices but only some in 3 different groups, add and remove +// various devices and groups while querying in between. +TEST_F(EnumeratePhysicalDeviceGroups, MultipleAddRemoves) { + auto& driver = env->get_test_icd().set_min_icd_interface_version(5).set_icd_api_version(VK_API_VERSION_1_1); + + // Generate the devices + for (size_t i = 0; i < 9; i++) { + driver.physical_devices.emplace_back(std::string("physical_device_") + std::to_string(i)); + } + + // Generate the starting groups + driver.physical_device_groups.emplace_back(driver.physical_devices[0]); + driver.physical_device_groups.emplace_back(driver.physical_devices[1]); + driver.physical_device_groups.back() + .use_physical_device(driver.physical_devices[2]) + .use_physical_device(driver.physical_devices[3]); + driver.physical_device_groups.emplace_back(driver.physical_devices[4]); + driver.physical_device_groups.back().use_physical_device(driver.physical_devices[5]); + + uint32_t before_expected_counts[3] = {1, 3, 2}; + uint32_t after_add_group_expected_counts[4] = {1, 3, 1, 2}; + uint32_t after_remove_dev_expected_counts[4] = {1, 2, 1, 2}; + uint32_t after_remove_group_expected_counts[3] = {2, 1, 2}; + uint32_t after_add_dev_expected_counts[3] = {2, 1, 4}; + uint32_t before_group_count = 3; + uint32_t after_group_count = 4; + uint32_t returned_group_count = 0; + + InstWrapper inst{env->vulkan_functions}; + inst.create_info.set_api_version(1, 1, 0); + inst.CheckCreate(); + + // Should be: 3 Groups { { 0 }, { 1, 2, 3 }, { 4, 5 } } + std::vector group_props_before{}; + group_props_before.resize(before_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + returned_group_count = before_group_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_before.data())); + ASSERT_EQ(before_group_count, returned_group_count); + for (uint32_t group = 0; group < before_group_count; ++group) { + ASSERT_EQ(group_props_before[group].physicalDeviceCount, before_expected_counts[group]); + } + + // Insert new group after first two + driver.physical_device_groups.insert(driver.physical_device_groups.begin() + 2, driver.physical_devices[6]); + + // Should be: 4 Groups { { 0 }, { 1, 2, 3 }, { 6 }, { 4, 5 } } + std::vector group_props_after_add_group{}; + group_props_after_add_group.resize(after_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + returned_group_count = after_group_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after_add_group.data())); + ASSERT_EQ(after_group_count, returned_group_count); + for (uint32_t group = 0; group < before_group_count; ++group) { + ASSERT_EQ(group_props_after_add_group[group].physicalDeviceCount, after_add_group_expected_counts[group]); + } + + // Remove first device in 2nd group + driver.physical_device_groups[1].physical_device_handles.erase( + driver.physical_device_groups[1].physical_device_handles.begin()); + + // Should be: 4 Groups { { 0 }, { 2, 3 }, { 6 }, { 4, 5 } } + std::vector group_props_after_remove_device{}; + group_props_after_remove_device.resize(after_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + returned_group_count = after_group_count; + ASSERT_EQ(VK_SUCCESS, + inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after_remove_device.data())); + ASSERT_EQ(after_group_count, returned_group_count); + for (uint32_t group = 0; group < before_group_count; ++group) { + ASSERT_EQ(group_props_after_remove_device[group].physicalDeviceCount, after_remove_dev_expected_counts[group]); + } + + // Remove first group + driver.physical_device_groups.erase(driver.physical_device_groups.begin()); + + // Should be: 3 Groups { { 2, 3 }, { 6 }, { 4, 5 } } + std::vector group_props_after_remove_group{}; + group_props_after_remove_group.resize(before_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + returned_group_count = before_group_count; + ASSERT_EQ(VK_SUCCESS, + inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after_remove_group.data())); + ASSERT_EQ(before_group_count, returned_group_count); + for (uint32_t group = 0; group < before_group_count; ++group) { + ASSERT_EQ(group_props_after_remove_group[group].physicalDeviceCount, after_remove_group_expected_counts[group]); + } + + // Add two devices to last group + driver.physical_device_groups.back() + .use_physical_device(driver.physical_devices[7]) + .use_physical_device(driver.physical_devices[8]); + + // Should be: 3 Groups { { 2, 3 }, { 6 }, { 4, 5, 7, 8 } } + std::vector group_props_after_add_device{}; + group_props_after_add_device.resize(before_group_count, + VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + returned_group_count = before_group_count; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props_after_add_device.data())); + ASSERT_EQ(before_group_count, returned_group_count); + for (uint32_t group = 0; group < before_group_count; ++group) { + ASSERT_EQ(group_props_after_add_device[group].physicalDeviceCount, after_add_dev_expected_counts[group]); + } +} + +// Fill in random but valid data into the device properties struct for the current physical device +static void FillInRandomDeviceProps(VkPhysicalDeviceProperties& props, VkPhysicalDeviceType dev_type, uint32_t api_vers, + uint32_t vendor, uint32_t device) { + props.apiVersion = api_vers; + props.vendorID = vendor; + props.deviceID = device; + props.deviceType = dev_type; + for (uint8_t idx = 0; idx < VK_UUID_SIZE; ++idx) { + props.pipelineCacheUUID[idx] = static_cast(rand() % 255); + } +} + +// Pass in a PNext that the fake ICD will fill in some data for. +TEST_F(EnumeratePhysicalDeviceGroups, FakePNext) { + FrameworkEnvironment env{}; + + // ICD 0: Vulkan 1.1 + // PhysDev 0: pd0, Discrete, Vulkan 1.1, Bus 7 + // PhysDev 1: pd1, Integrated, Vulkan 1.1, Bus 3 + // PhysDev 2: pd2, Discrete, Vulkan 1.1, Bus 6 + // Group 0: PhysDev 0, PhysDev 2 + // Group 1: PhysDev 1 + // ICD 1: Vulkan 1.1 + // PhysDev 4: pd4, Discrete, Vulkan 1.1, Bus 1 + // PhysDev 5: pd5, Discrete, Vulkan 1.1, Bus 4 + // PhysDev 6: pd6, Discrete, Vulkan 1.1, Bus 2 + // Group 0: PhysDev 5, PhysDev 6 + // Group 1: PhysDev 4 + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_6)); + auto& cur_icd_0 = env.get_test_icd(0); + cur_icd_0.set_icd_api_version(VK_API_VERSION_1_1); + cur_icd_0.physical_devices.push_back({"pd0", 7}); + cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + 888, 0xAAA001); + cur_icd_0.physical_devices.push_back({"pd1", 3}); + cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU, + VK_API_VERSION_1_1, 888, 0xAAA002); + cur_icd_0.physical_devices.push_back({"pd2", 6}); + cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + 888, 0xAAA003); + cur_icd_0.physical_device_groups.push_back({}); + cur_icd_0.physical_device_groups.back() + .use_physical_device(cur_icd_0.physical_devices[0]) + .use_physical_device(cur_icd_0.physical_devices[2]); + cur_icd_0.physical_device_groups.push_back({cur_icd_0.physical_devices[1]}); + + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_6)); + auto& cur_icd_1 = env.get_test_icd(1); + cur_icd_1.set_icd_api_version(VK_API_VERSION_1_1); + cur_icd_1.physical_devices.push_back({"pd4", 1}); + cur_icd_1.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_1.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + 75, 0xCCCC001); + cur_icd_1.physical_devices.push_back({"pd5", 4}); + cur_icd_1.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_1.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + 75, 0xCCCC002); + cur_icd_1.physical_devices.push_back({"pd6", 2}); + cur_icd_1.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_1.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + 75, 0xCCCC003); + cur_icd_1.physical_device_groups.push_back({}); + cur_icd_1.physical_device_groups.back() + .use_physical_device(cur_icd_1.physical_devices[1]) + .use_physical_device(cur_icd_1.physical_devices[2]); + cur_icd_1.physical_device_groups.push_back({cur_icd_1.physical_devices[0]}); + + InstWrapper inst(env.vulkan_functions); + inst.create_info.set_api_version(VK_API_VERSION_1_1); + inst.CheckCreate(); + + auto GetPhysDevProps2 = reinterpret_cast( + inst.functions->vkGetInstanceProcAddr(inst, "vkGetPhysicalDeviceProperties2")); + ASSERT_NE(GetPhysDevProps2, nullptr); + + // NOTE: This is a fake struct to make sure the pNext chain is properly passed down to the ICD + // vkEnumeratePhysicalDeviceGroups. + // The two versions must match: + // "FakePNext" test in loader_regresion_tests.cpp + // "test_vkEnumeratePhysicalDeviceGroups" in test_icd.cpp + struct FakePnextSharedWithICD { + VkStructureType sType; + void* pNext; + uint32_t value; + }; + + const uint32_t max_phys_dev_groups = 4; + uint32_t group_count = max_phys_dev_groups; + std::array fake_structs; + std::array physical_device_groups{ + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}; + for (uint32_t group = 0; group < max_phys_dev_groups; ++group) { + fake_structs[group].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MULTI_DRAW_PROPERTIES_EXT; + physical_device_groups[group].pNext = &fake_structs[group]; + } + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &group_count, physical_device_groups.data())); + ASSERT_EQ(group_count, max_phys_dev_groups); + + // Value should get written to 0xDECAFBADD by the fake ICD + for (uint32_t group = 0; group < max_phys_dev_groups; ++group) { + ASSERT_EQ(fake_structs[group].value, 0xDECAFBAD); } } @@ -1212,18 +2237,6 @@ TEST_F(CreateInstance, InstanceNullExtensionPtr) { ASSERT_EQ(env->vulkan_functions.vkCreateInstance(&info, VK_NULL_HANDLE, &inst), VK_ERROR_EXTENSION_NOT_PRESENT); } -// Fill in random but valid data into the device properties struct for the current physical device -static void FillInRandomDeviceProps(VkPhysicalDeviceProperties& props, VkPhysicalDeviceType dev_type, uint32_t api_vers, - uint32_t vendor, uint32_t device) { - props.apiVersion = api_vers; - props.vendorID = vendor; - props.deviceID = device; - props.deviceType = dev_type; - for (uint8_t idx = 0; idx < VK_UUID_SIZE; ++idx) { - props.pipelineCacheUUID[idx] = static_cast(rand() % 255); - } -} - #if defined(__linux__) || defined(__FreeBSD__) // NOTE: Sort order only affects Linux TEST(SortedPhysicalDevices, DevicesSortEnabled) { @@ -1436,6 +2449,9 @@ TEST(SortedPhysicalDevices, DevicesSortedDisabled) { default: ASSERT_EQ(false, true); } + if (!sorted) { + break; + } } ASSERT_EQ(false, sorted); @@ -1450,61 +2466,81 @@ TEST(SortedPhysicalDevices, DevicesSortedDisabled) { remove_env_var("VK_LOADER_DISABLE_SELECT"); } -#if 0 // TODO: Enable check on physical device group sorting to make sure proper order returned. - // This is disabled because the test framework needs a little more work for this. TEST(SortedPhysicalDevices, DeviceGroupsSortedEnabled) { FrameworkEnvironment env{}; + // ICD 0: Vulkan 1.1 + // PhysDev 0: pd0, Discrete, Vulkan 1.1, Bus 7 + // PhysDev 1: pd1, Integrated, Vulkan 1.1, Bus 3 + // PhysDev 2: pd2, Discrete, Vulkan 1.1, Bus 6 + // Group 0: PhysDev 0, PhysDev 2 + // Group 1: PhysDev 1 + // ICD 1: Vulkan 1.1 + // PhysDev 3: pd3, CPU, Vulkan 1.1, Bus 0 + // ICD 2: Vulkan 1.1 + // PhysDev 4: pd4, Discrete, Vulkan 1.1, Bus 1 + // PhysDev 5: pd5, Discrete, Vulkan 1.1, Bus 4 + // PhysDev 6: pd6, Discrete, Vulkan 1.1, Bus 2 + // Group 0: PhysDev 5, PhysDev 6 + // Group 1: PhysDev 4 + // ICD 3: Vulkan 1.1 + // PhysDev 7: pd7, Virtual, Vulkan 1.1, Bus 0 env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1)); auto& cur_icd_0 = env.get_test_icd(0); cur_icd_0.set_icd_api_version(VK_API_VERSION_1_1); cur_icd_0.physical_devices.push_back({"pd0", 7}); + cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, 888, 0xAAA001); cur_icd_0.physical_devices.push_back({"pd1", 3}); + cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU, VK_API_VERSION_1_1, 888, 0xAAA002); cur_icd_0.physical_devices.push_back({"pd2", 6}); + cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, 888, 0xAAA003); cur_icd_0.physical_device_groups.push_back({}); cur_icd_0.physical_device_groups.back() .use_physical_device(cur_icd_0.physical_devices[0]) .use_physical_device(cur_icd_0.physical_devices[2]); - cur_icd_0.physical_device_groups.push_back({}); - cur_icd_0.physical_device_groups.back().use_physical_device(cur_icd_0.physical_devices[1]); + cur_icd_0.physical_device_groups.push_back({cur_icd_0.physical_devices[1]}); env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1)); auto& cur_icd_1 = env.get_test_icd(1); cur_icd_1.set_icd_api_version(VK_API_VERSION_1_1); cur_icd_1.physical_devices.push_back({"pd3", 0}); - FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_CPU, VK_API_VERSION_1_1, 1, + cur_icd_1.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_1.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_CPU, VK_API_VERSION_1_1, 1, 0xBBBB001); env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1)); auto& cur_icd_2 = env.get_test_icd(2); cur_icd_2.set_icd_api_version(VK_API_VERSION_1_1); cur_icd_2.physical_devices.push_back({"pd4", 1}); - FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, 75, 0xCCCC001); cur_icd_2.physical_devices.push_back({"pd5", 4}); - FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, 75, 0xCCCC002); cur_icd_2.physical_devices.push_back({"pd6", 2}); - FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, 75, 0xCCCC003); cur_icd_2.physical_device_groups.push_back({}); cur_icd_2.physical_device_groups.back() .use_physical_device(cur_icd_2.physical_devices[1]) .use_physical_device(cur_icd_2.physical_devices[2]); - cur_icd_2.physical_device_groups.push_back({}); - cur_icd_2.physical_device_groups.back().use_physical_device(cur_icd_2.physical_devices[0]); + cur_icd_2.physical_device_groups.push_back({cur_icd_2.physical_devices[0]}); env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1)); auto& cur_icd_3 = env.get_test_icd(3); cur_icd_3.set_icd_api_version(VK_API_VERSION_1_1); cur_icd_3.physical_devices.push_back({"pd7", 0}); - FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU, VK_API_VERSION_1_1, + cur_icd_3.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_3.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU, VK_API_VERSION_1_1, 6940, 0xDDDD001); InstWrapper inst(env.vulkan_functions); @@ -1521,7 +2557,7 @@ TEST(SortedPhysicalDevices, DeviceGroupsSortedEnabled) { ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &device_count, physical_devices.data())); ASSERT_EQ(device_count, max_phys_devs); - const uint32_t max_phys_dev_groups = 8; + const uint32_t max_phys_dev_groups = 6; uint32_t group_count = max_phys_dev_groups; std::array physical_device_groups{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}; @@ -1532,71 +2568,259 @@ TEST(SortedPhysicalDevices, DeviceGroupsSortedEnabled) { for (uint32_t group = 0; group < max_phys_dev_groups; ++group) { for (uint32_t dev = 0; dev < physical_device_groups[group].physicalDeviceCount; ++dev) { VkPhysicalDeviceProperties props{}; - inst->vkGetPhysicalDeviceProperties(physical_devices[dev], &props); + inst->vkGetPhysicalDeviceProperties(physical_device_groups[group].physicalDevices[dev], &props); VkPhysicalDeviceProperties2 props2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2}; VkPhysicalDevicePCIBusInfoPropertiesEXT pci_bus_info{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT}; props2.pNext = &pci_bus_info; - GetPhysDevProps2(physical_devices[dev], &props2); - /* - switch (++cur_dev) { - case 0: - ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU); - ASSERT_EQ(true, !strcmp("pd3", props.deviceName)); - ASSERT_EQ(props.vendorID, 75); - ASSERT_EQ(props.deviceID, 0xCCCC001); - ASSERT_EQ(pci_bus_info.pciBus, 1); - break; - case 1: - ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU); - ASSERT_EQ(true, !strcmp("pd4", props.deviceName)); - ASSERT_EQ(props.vendorID, 75); - ASSERT_EQ(props.deviceID, 0xCCCC002); - ASSERT_EQ(pci_bus_info.pciBus, 4); - break; - case 2: - ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU); - ASSERT_EQ(true, !strcmp("pd0", props.deviceName)); - ASSERT_EQ(props.vendorID, 888); - ASSERT_EQ(props.deviceID, 0xAAA001); - ASSERT_EQ(pci_bus_info.pciBus, 7); - break; - case 3: - ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU); - ASSERT_EQ(true, !strcmp("pd1", props.deviceName)); - ASSERT_EQ(props.vendorID, 888); - ASSERT_EQ(props.deviceID, 0xAAA002); - ASSERT_EQ(pci_bus_info.pciBus, 3); - break; - case 4: - ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU); - ASSERT_EQ(true, !strcmp("pd5", props.deviceName)); - ASSERT_EQ(props.vendorID, 6940); - ASSERT_EQ(props.deviceID, 0xDDDD001); - ASSERT_EQ(pci_bus_info.pciBus, 0); - break; - case 5: - ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_CPU); - ASSERT_EQ(true, !strcmp("pd2", props.deviceName)); - ASSERT_EQ(props.vendorID, 1); - ASSERT_EQ(props.deviceID, 0xBBBB001); - ASSERT_EQ(pci_bus_info.pciBus, 0); - break; - default: - ASSERT_EQ(false, true); - } - */ + GetPhysDevProps2(physical_device_groups[group].physicalDevices[dev], &props2); + switch (cur_dev++) { + case 0: + ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU); + ASSERT_EQ(true, !strcmp("pd4", props.deviceName)); + ASSERT_EQ(props.vendorID, 75); + ASSERT_EQ(props.deviceID, 0xCCCC001); + ASSERT_EQ(pci_bus_info.pciBus, 1); + break; + case 1: + ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU); + ASSERT_EQ(true, !strcmp("pd6", props.deviceName)); + ASSERT_EQ(props.vendorID, 75); + ASSERT_EQ(props.deviceID, 0xCCCC003); + ASSERT_EQ(pci_bus_info.pciBus, 2); + break; + case 2: + ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU); + ASSERT_EQ(true, !strcmp("pd5", props.deviceName)); + ASSERT_EQ(props.vendorID, 75); + ASSERT_EQ(props.deviceID, 0xCCCC002); + ASSERT_EQ(pci_bus_info.pciBus, 4); + break; + case 3: + ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU); + ASSERT_EQ(true, !strcmp("pd2", props.deviceName)); + ASSERT_EQ(props.vendorID, 888); + ASSERT_EQ(props.deviceID, 0xAAA003); + ASSERT_EQ(pci_bus_info.pciBus, 6); + break; + case 4: + ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU); + ASSERT_EQ(true, !strcmp("pd0", props.deviceName)); + ASSERT_EQ(props.vendorID, 888); + ASSERT_EQ(props.deviceID, 0xAAA001); + ASSERT_EQ(pci_bus_info.pciBus, 7); + break; + case 5: + ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU); + ASSERT_EQ(true, !strcmp("pd1", props.deviceName)); + ASSERT_EQ(props.vendorID, 888); + ASSERT_EQ(props.deviceID, 0xAAA002); + ASSERT_EQ(pci_bus_info.pciBus, 3); + break; + case 6: + ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU); + ASSERT_EQ(true, !strcmp("pd7", props.deviceName)); + ASSERT_EQ(props.vendorID, 6940); + ASSERT_EQ(props.deviceID, 0xDDDD001); + ASSERT_EQ(pci_bus_info.pciBus, 0); + break; + case 7: + ASSERT_EQ(props.deviceType, VK_PHYSICAL_DEVICE_TYPE_CPU); + ASSERT_EQ(true, !strcmp("pd3", props.deviceName)); + ASSERT_EQ(props.vendorID, 1); + ASSERT_EQ(props.deviceID, 0xBBBB001); + ASSERT_EQ(pci_bus_info.pciBus, 0); + break; + default: + ASSERT_EQ(false, true); + } } } + // Make sure if we call enumerate again, the information is the same std::array physical_device_groups_again{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}; ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &group_count, physical_device_groups_again.data())); ASSERT_EQ(group_count, max_phys_dev_groups); for (uint32_t group = 0; group < max_phys_dev_groups; ++group) { + ASSERT_EQ(physical_device_groups[group].physicalDeviceCount, physical_device_groups_again[group].physicalDeviceCount); for (uint32_t dev = 0; dev < physical_device_groups[group].physicalDeviceCount; ++dev) { ASSERT_EQ(physical_device_groups[group].physicalDevices[dev], physical_device_groups_again[group].physicalDevices[dev]); } } } -#endif -#endif // __linux__ || __FreeBSD__ \ No newline at end of file + +TEST(SortedPhysicalDevices, DeviceGroupsSortedDisabled) { + FrameworkEnvironment env{}; + + set_env_var("VK_LOADER_DISABLE_SELECT", "1"); + + // ICD 0: Vulkan 1.1 + // PhysDev 0: pd0, Discrete, Vulkan 1.1, Bus 7 + // PhysDev 1: pd1, Integrated, Vulkan 1.1, Bus 3 + // PhysDev 2: pd2, Discrete, Vulkan 1.1, Bus 6 + // Group 0: PhysDev 0, PhysDev 2 + // Group 1: PhysDev 1 + // ICD 1: Vulkan 1.1 + // PhysDev 3: pd3, CPU, Vulkan 1.1, Bus 0 + // ICD 2: Vulkan 1.1 + // PhysDev 4: pd4, Discrete, Vulkan 1.1, Bus 1 + // PhysDev 5: pd5, Discrete, Vulkan 1.1, Bus 4 + // PhysDev 6: pd6, Discrete, Vulkan 1.1, Bus 2 + // Group 0: PhysDev 5, PhysDev 6 + // Group 1: PhysDev 4 + // ICD 3: Vulkan 1.1 + // PhysDev 7: pd7, Virtual, Vulkan 1.1, Bus 0 + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1)); + auto& cur_icd_0 = env.get_test_icd(0); + cur_icd_0.set_icd_api_version(VK_API_VERSION_1_1); + cur_icd_0.physical_devices.push_back({"pd0", 7}); + cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + 888, 0xAAA001); + cur_icd_0.physical_devices.push_back({"pd1", 3}); + cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU, + VK_API_VERSION_1_1, 888, 0xAAA002); + cur_icd_0.physical_devices.push_back({"pd2", 6}); + cur_icd_0.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_0.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + 888, 0xAAA003); + cur_icd_0.physical_device_groups.push_back({}); + cur_icd_0.physical_device_groups.back() + .use_physical_device(cur_icd_0.physical_devices[0]) + .use_physical_device(cur_icd_0.physical_devices[2]); + cur_icd_0.physical_device_groups.push_back({cur_icd_0.physical_devices[1]}); + + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1)); + auto& cur_icd_1 = env.get_test_icd(1); + cur_icd_1.set_icd_api_version(VK_API_VERSION_1_1); + cur_icd_1.physical_devices.push_back({"pd3", 0}); + cur_icd_1.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_1.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_CPU, VK_API_VERSION_1_1, 1, + 0xBBBB001); + + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1)); + auto& cur_icd_2 = env.get_test_icd(2); + cur_icd_2.set_icd_api_version(VK_API_VERSION_1_1); + cur_icd_2.physical_devices.push_back({"pd4", 1}); + cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + 75, 0xCCCC001); + cur_icd_2.physical_devices.push_back({"pd5", 4}); + cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + 75, 0xCCCC002); + cur_icd_2.physical_devices.push_back({"pd6", 2}); + cur_icd_2.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_2.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU, VK_API_VERSION_1_1, + 75, 0xCCCC003); + cur_icd_2.physical_device_groups.push_back({}); + cur_icd_2.physical_device_groups.back() + .use_physical_device(cur_icd_2.physical_devices[1]) + .use_physical_device(cur_icd_2.physical_devices[2]); + cur_icd_2.physical_device_groups.push_back({cur_icd_2.physical_devices[0]}); + + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2, VK_API_VERSION_1_1)); + auto& cur_icd_3 = env.get_test_icd(3); + cur_icd_3.set_icd_api_version(VK_API_VERSION_1_1); + cur_icd_3.physical_devices.push_back({"pd7", 0}); + cur_icd_3.physical_devices.back().extensions.push_back({VK_EXT_PCI_BUS_INFO_EXTENSION_NAME, 0}); + FillInRandomDeviceProps(cur_icd_3.physical_devices.back().properties, VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU, VK_API_VERSION_1_1, + 6940, 0xDDDD001); + + InstWrapper inst(env.vulkan_functions); + inst.create_info.set_api_version(VK_API_VERSION_1_1); + inst.CheckCreate(); + + auto GetPhysDevProps2 = reinterpret_cast( + inst.functions->vkGetInstanceProcAddr(inst, "vkGetPhysicalDeviceProperties2")); + ASSERT_NE(GetPhysDevProps2, nullptr); + + const uint32_t max_phys_devs = 8; + uint32_t device_count = max_phys_devs; + std::array physical_devices; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDevices(inst, &device_count, physical_devices.data())); + ASSERT_EQ(device_count, max_phys_devs); + + const uint32_t max_phys_dev_groups = 6; + uint32_t group_count = max_phys_dev_groups; + std::array physical_device_groups{ + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &group_count, physical_device_groups.data())); + ASSERT_EQ(group_count, max_phys_dev_groups); + + // Make sure the devices are not in the sorted order. The order is really undefined, but the chances of + // it being exactly the expected sorted is very low. + bool sorted = true; + uint32_t cur_dev = 0; + for (uint32_t group = 0; group < max_phys_dev_groups; ++group) { + for (uint32_t dev = 0; dev < physical_device_groups[group].physicalDeviceCount; ++dev) { + VkPhysicalDeviceProperties props{}; + inst->vkGetPhysicalDeviceProperties(physical_device_groups[group].physicalDevices[dev], &props); + switch (cur_dev++) { + case 0: + if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || strcmp("pd4", props.deviceName)) { + sorted = false; + } + break; + case 1: + if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || strcmp("pd6", props.deviceName)) { + sorted = false; + } + break; + case 2: + if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || strcmp("pd5", props.deviceName)) { + sorted = false; + } + break; + case 3: + if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || strcmp("pd2", props.deviceName)) { + sorted = false; + } + break; + case 4: + if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU || strcmp("pd0", props.deviceName)) { + sorted = false; + } + break; + case 5: + if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU || strcmp("pd1", props.deviceName)) { + sorted = false; + } + break; + case 6: + if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU || strcmp("pd7", props.deviceName)) { + sorted = false; + } + break; + case 7: + if (props.deviceType != VK_PHYSICAL_DEVICE_TYPE_CPU || strcmp("pd3", props.deviceName)) { + sorted = false; + } + break; + default: + ASSERT_EQ(false, true); + } + } + if (!sorted) { + break; + } + } + ASSERT_EQ(false, sorted); + + // Make sure if we call enumerate again, the information is the same + std::array physical_device_groups_again{ + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &group_count, physical_device_groups_again.data())); + ASSERT_EQ(group_count, max_phys_dev_groups); + for (uint32_t group = 0; group < max_phys_dev_groups; ++group) { + ASSERT_EQ(physical_device_groups[group].physicalDeviceCount, physical_device_groups_again[group].physicalDeviceCount); + for (uint32_t dev = 0; dev < physical_device_groups[group].physicalDeviceCount; ++dev) { + ASSERT_EQ(physical_device_groups[group].physicalDevices[dev], physical_device_groups_again[group].physicalDevices[dev]); + } + } + + remove_env_var("VK_LOADER_DISABLE_SELECT"); +} + +#endif // __linux__ || __FreeBSD__ diff --git a/tests/loader_version_tests.cpp b/tests/loader_version_tests.cpp index d89a9078..a4fd5cf7 100644 --- a/tests/loader_version_tests.cpp +++ b/tests/loader_version_tests.cpp @@ -320,6 +320,94 @@ TEST(MultipleDriverConfig, DifferentICDInterfaceVersions) { ASSERT_EQ(env.vulkan_functions.vkEnumeratePhysicalDevices(inst, &phys_dev_count, phys_devs_array.data()), VK_SUCCESS); ASSERT_EQ(phys_dev_count, 2); } + +TEST(MultipleDriverConfig, DifferentICDsWithDevices) { + FrameworkEnvironment env{}; + env.add_icd(TestICDDetails(TEST_ICD_PATH_EXPORT_ICD_GIPA)); + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2)); + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2_EXPORT_ICD_GPDPA)); + + // Make sure the loader returns all devices from all active ICDs. Many of the other + // tests add multiple devices to a single ICD, this just makes sure the loader combines + // device info across multiple drivers properly. + TestICD& icd0 = env.get_test_icd(0); + icd0.physical_devices.emplace_back("physical_device_0"); + icd0.min_icd_interface_version = 5; + icd0.max_icd_interface_version = 5; + + TestICD& icd1 = env.get_test_icd(1); + icd1.physical_devices.emplace_back("physical_device_1"); + icd1.physical_devices.emplace_back("physical_device_2"); + icd1.min_icd_interface_version = 5; + icd1.max_icd_interface_version = 5; + + TestICD& icd2 = env.get_test_icd(2); + icd2.physical_devices.emplace_back("physical_device_3"); + icd2.min_icd_interface_version = 5; + icd2.max_icd_interface_version = 5; + + InstWrapper inst{env.vulkan_functions}; + inst.CheckCreate(); + + std::array phys_devs_array; + uint32_t phys_dev_count = 4; + ASSERT_EQ(env.vulkan_functions.vkEnumeratePhysicalDevices(inst, &phys_dev_count, phys_devs_array.data()), VK_SUCCESS); + ASSERT_EQ(phys_dev_count, 4); +} + +TEST(MultipleDriverConfig, DifferentICDsWithDevicesAndGroups) { + FrameworkEnvironment env{}; + env.add_icd(TestICDDetails(TEST_ICD_PATH_EXPORT_ICD_GIPA)); + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2)); + env.add_icd(TestICDDetails(TEST_ICD_PATH_VERSION_2_EXPORT_ICD_GPDPA)); + + // The loader has to be able to handle drivers that support device groups in combination + // with drivers that don't support device groups. When this is the case, the loader needs + // to take every driver that doesn't support device groups and put each of its devices in + // a separate group. Then it combines that information with the drivers that support + // device groups returned info. + + // ICD 0 : No 1.1 support (so 1 device will become 1 group in loader) + TestICD& icd0 = env.get_test_icd(0); + icd0.physical_devices.emplace_back("physical_device_0"); + icd0.min_icd_interface_version = 5; + icd0.max_icd_interface_version = 5; + icd0.set_icd_api_version(VK_API_VERSION_1_0); + + // ICD 1 : 1.1 support (with 1 group with 2 devices) + TestICD& icd1 = env.get_test_icd(1); + icd1.physical_devices.emplace_back("physical_device_1"); + icd1.physical_devices.emplace_back("physical_device_2"); + icd1.physical_device_groups.emplace_back(icd1.physical_devices[0]); + icd1.physical_device_groups.back().use_physical_device(icd1.physical_devices[1]); + icd1.min_icd_interface_version = 5; + icd1.max_icd_interface_version = 5; + icd1.set_icd_api_version(VK_API_VERSION_1_1); + + // ICD 2 : No 1.1 support (so 3 devices will become 3 groups in loader) + TestICD& icd2 = env.get_test_icd(2); + icd2.physical_devices.emplace_back("physical_device_3"); + icd2.physical_devices.emplace_back("physical_device_4"); + icd2.physical_devices.emplace_back("physical_device_5"); + icd2.min_icd_interface_version = 5; + icd2.max_icd_interface_version = 5; + icd2.set_icd_api_version(VK_API_VERSION_1_0); + + InstWrapper inst{env.vulkan_functions}; + inst.create_info.set_api_version(1, 1, 0); + inst.CheckCreate(); + + uint32_t group_count = static_cast(5); + uint32_t returned_group_count = 0; + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, nullptr)); + ASSERT_EQ(group_count, returned_group_count); + + std::vector group_props{}; + group_props.resize(group_count, VkPhysicalDeviceGroupProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GROUP_PROPERTIES}); + ASSERT_EQ(VK_SUCCESS, inst->vkEnumeratePhysicalDeviceGroups(inst, &returned_group_count, group_props.data())); + ASSERT_EQ(group_count, returned_group_count); +} + // shim function pointers for 1.3 // Should use autogen for this - it generates 'shim' functions for validation layers, maybe that could be used here. void test_vkCmdBeginRendering(VkCommandBuffer commandBuffer, const VkRenderPassBeginInfo* pRenderPassBegin, @@ -668,4 +756,4 @@ TEST(LayerManifest, ExplicitNonVulkanVariant) { inst.CheckCreate(VK_ERROR_LAYER_NOT_PRESENT); ASSERT_TRUE(log.find(std::string("Layer ") + explicit_layer_name + " has an \'api_version\' field which contains a non-zero variant value of 1. Skipping Layer.")); -} \ No newline at end of file +}