From 47b95416ae9a2ba3e310ab4c6f8a21d36a433cd8 Mon Sep 17 00:00:00 2001 From: Karol Herbst Date: Wed, 1 Jun 2022 01:04:42 +0200 Subject: [PATCH] nvk: add nvk_bo_sync Part-of: --- src/nouveau/vulkan/meson.build | 2 + src/nouveau/vulkan/nvk_bo_sync.c | 240 +++++++++++++++++++++++++++++++ src/nouveau/vulkan/nvk_bo_sync.h | 24 ++++ src/nouveau/vulkan/nvk_device.c | 45 ++++++ src/nouveau/vulkan/nvk_device.h | 3 + src/nouveau/vulkan/nvk_physical_device.c | 7 + src/nouveau/vulkan/nvk_physical_device.h | 2 + 7 files changed, 323 insertions(+) create mode 100644 src/nouveau/vulkan/nvk_bo_sync.c create mode 100644 src/nouveau/vulkan/nvk_bo_sync.h diff --git a/src/nouveau/vulkan/meson.build b/src/nouveau/vulkan/meson.build index 3d6dbe5..4d9ed19 100644 --- a/src/nouveau/vulkan/meson.build +++ b/src/nouveau/vulkan/meson.build @@ -1,4 +1,6 @@ nvk_files = files( + 'nvk_bo_sync.c', + 'nvk_bo_sync.h', 'nvk_buffer.c', 'nvk_buffer.h', 'nvk_cmd_buffer.c', diff --git a/src/nouveau/vulkan/nvk_bo_sync.c b/src/nouveau/vulkan/nvk_bo_sync.c new file mode 100644 index 0000000..d16da24 --- /dev/null +++ b/src/nouveau/vulkan/nvk_bo_sync.c @@ -0,0 +1,240 @@ +/* + * Copyright © 2021 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "nvk_bo_sync.h" + +#include "nouveau_bo.h" + +#include "nvk_device.h" +#include "nvk_device_memory.h" +#include "nvk_physical_device.h" + +#include "util/os_time.h" +#include "util/timespec.h" + +static struct nvk_bo_sync * +to_nvk_bo_sync(struct vk_sync *sync) +{ + assert(sync->type == &nvk_bo_sync_type); + return container_of(sync, struct nvk_bo_sync, sync); +} + +static VkResult +nvk_bo_sync_init(struct vk_device *vk_device, + struct vk_sync *vk_sync, + uint64_t initial_value) +{ + struct nvk_device *device = container_of(vk_device, struct nvk_device, vk); + struct nvk_bo_sync *sync = to_nvk_bo_sync(vk_sync); + + sync->state = initial_value ? NVK_BO_SYNC_STATE_SIGNALED : + NVK_BO_SYNC_STATE_RESET; + + sync->bo = nouveau_ws_bo_new(device->pdev->dev, 0x1000, 0, NOUVEAU_WS_BO_GART); + if (!sync->bo) + return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + return VK_SUCCESS; +} + +static void +nvk_bo_sync_finish(struct vk_device *vk_device, + struct vk_sync *vk_sync) +{ + struct nvk_bo_sync *sync = to_nvk_bo_sync(vk_sync); + + nouveau_ws_bo_destroy(sync->bo); +} + +static VkResult +nvk_bo_sync_reset(struct vk_device *vk_device, + struct vk_sync *vk_sync) +{ + struct nvk_bo_sync *sync = to_nvk_bo_sync(vk_sync); + + sync->state = NVK_BO_SYNC_STATE_RESET; + + return VK_SUCCESS; +} + +static int64_t +nvk_get_relative_timeout(uint64_t abs_timeout) +{ + uint64_t now = os_time_get_nano(); + + /* We don't want negative timeouts. + * + * DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is + * supposed to block indefinitely timeouts < 0. Unfortunately, + * this was broken for a couple of kernel releases. Since there's + * no way to know whether or not the kernel we're using is one of + * the broken ones, the best we can do is to clamp the timeout to + * INT64_MAX. This limits the maximum timeout from 584 years to + * 292 years - likely not a big deal. + */ + if (abs_timeout < now) + return 0; + + uint64_t rel_timeout = abs_timeout - now; + if (rel_timeout > (uint64_t) INT64_MAX) + rel_timeout = INT64_MAX; + + return rel_timeout; +} + +static VkResult +nvk_bo_sync_wait(struct vk_device *vk_device, + uint32_t wait_count, + const struct vk_sync_wait *waits, + enum vk_sync_wait_flags wait_flags, + uint64_t abs_timeout_ns) +{ + struct nvk_device *device = container_of(vk_device, struct nvk_device, vk); + + uint32_t pending = wait_count; + while (pending) { + pending = 0; + bool signaled = false; + for (uint32_t i = 0; i < wait_count; i++) { + struct nvk_bo_sync *sync = to_nvk_bo_sync(waits[i].sync); + switch (sync->state) { + case NVK_BO_SYNC_STATE_RESET: + /* This fence hasn't been submitted yet, we'll catch it the next + * time around. Yes, this may mean we dead-loop but, short of + * lots of locking and a condition variable, there's not much that + * we can do about that. + */ + assert(!(wait_flags & VK_SYNC_WAIT_PENDING)); + pending++; + continue; + + case NVK_BO_SYNC_STATE_SIGNALED: + /* This fence is not pending. If waitAll isn't set, we can return + * early. Otherwise, we have to keep going. + */ + if (wait_flags & VK_SYNC_WAIT_ANY) + return VK_SUCCESS; + continue; + + case NVK_BO_SYNC_STATE_SUBMITTED: + /* These are the fences we really care about. Go ahead and wait + * on it until we hit a timeout. + */ + if (!(wait_flags & VK_SYNC_WAIT_PENDING)) { + if (!nouveau_ws_bo_wait(sync->bo, NOUVEAU_WS_BO_RDWR)) { + return vk_error(device, VK_TIMEOUT); + } + + sync->state = NVK_BO_SYNC_STATE_SIGNALED; + signaled = true; + } + if (wait_flags & VK_SYNC_WAIT_ANY) + return VK_SUCCESS; + break; + + default: + unreachable("Invalid BO sync state"); + } + } + + if (pending && !signaled) { + /* If we've hit this then someone decided to vkWaitForFences before + * they've actually submitted any of them to a queue. This is a + * fairly pessimal case, so it's ok to lock here and use a standard + * pthreads condition variable. + */ + pthread_mutex_lock(&device->mutex); + + /* It's possible that some of the fences have changed state since the + * last time we checked. Now that we have the lock, check for + * pending fences again and don't wait if it's changed. + */ + uint32_t now_pending = 0; + for (uint32_t i = 0; i < wait_count; i++) { + struct nvk_bo_sync *sync = to_nvk_bo_sync(waits[i].sync); + if (sync->state == NVK_BO_SYNC_STATE_RESET) + now_pending++; + } + assert(now_pending <= pending); + + if (now_pending == pending) { + struct timespec abstime = { + .tv_sec = abs_timeout_ns / NSEC_PER_SEC, + .tv_nsec = abs_timeout_ns % NSEC_PER_SEC, + }; + + ASSERTED int ret; + ret = pthread_cond_timedwait(&device->queue_submit, + &device->mutex, &abstime); + assert(ret != EINVAL); + if (os_time_get_nano() >= abs_timeout_ns) { + pthread_mutex_unlock(&device->mutex); + return VK_TIMEOUT; + } + } + + pthread_mutex_unlock(&device->mutex); + } + } + + return VK_SUCCESS; +} + +const struct vk_sync_type nvk_bo_sync_type = { + .size = sizeof(struct nvk_bo_sync), + .features = VK_SYNC_FEATURE_BINARY | + VK_SYNC_FEATURE_GPU_WAIT | + VK_SYNC_FEATURE_GPU_MULTI_WAIT | + VK_SYNC_FEATURE_CPU_WAIT | + VK_SYNC_FEATURE_CPU_RESET | + VK_SYNC_FEATURE_WAIT_ANY | + VK_SYNC_FEATURE_WAIT_PENDING, + .init = nvk_bo_sync_init, + .finish = nvk_bo_sync_finish, + .reset = nvk_bo_sync_reset, + .wait_many = nvk_bo_sync_wait, +}; + +VKAPI_ATTR VkResult VKAPI_CALL +nvk_create_sync_for_memory(struct vk_device *device, + VkDeviceMemory memory, + bool signal_memory, + struct vk_sync **sync_out) +{ + VK_FROM_HANDLE(nvk_device_memory, mem, memory); + struct nvk_bo_sync *bo_sync; + + bo_sync = vk_zalloc(&device->alloc, sizeof(*bo_sync), 8, + VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); + if (bo_sync == NULL) + return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); + + bo_sync->sync.type = &nvk_bo_sync_type; + bo_sync->state = signal_memory ? NVK_BO_SYNC_STATE_RESET : + NVK_BO_SYNC_STATE_SUBMITTED; + bo_sync->bo = mem->bo; + nouveau_ws_bo_ref(mem->bo); + + *sync_out = &bo_sync->sync; + + return VK_SUCCESS; +} diff --git a/src/nouveau/vulkan/nvk_bo_sync.h b/src/nouveau/vulkan/nvk_bo_sync.h new file mode 100644 index 0000000..60b3f7c --- /dev/null +++ b/src/nouveau/vulkan/nvk_bo_sync.h @@ -0,0 +1,24 @@ +#include "nvk_private.h" + +#include "vulkan/runtime/vk_sync.h" + +extern const struct vk_sync_type nvk_bo_sync_type; + +enum nvk_bo_sync_state { + NVK_BO_SYNC_STATE_RESET, + NVK_BO_SYNC_STATE_SUBMITTED, + NVK_BO_SYNC_STATE_SIGNALED, +}; + +struct nvk_bo_sync { + struct vk_sync sync; + + enum nvk_bo_sync_state state; + struct nouveau_ws_bo *bo; +}; + +VKAPI_ATTR VkResult VKAPI_CALL +nvk_create_sync_for_memory(struct vk_device *device, + VkDeviceMemory memory, + bool signal_memory, + struct vk_sync **sync_out); diff --git a/src/nouveau/vulkan/nvk_device.c b/src/nouveau/vulkan/nvk_device.c index b1285e5..97ca583 100644 --- a/src/nouveau/vulkan/nvk_device.c +++ b/src/nouveau/vulkan/nvk_device.c @@ -1,5 +1,6 @@ #include "nvk_device.h" +#include "nvk_bo_sync.h" #include "nvk_cmd_buffer.h" #include "nvk_instance.h" #include "nvk_physical_device.h" @@ -14,11 +15,27 @@ nvk_queue_submit(struct vk_queue *queue, struct vk_queue_submit *submission) { struct nvk_device *device = container_of(queue->base.device, struct nvk_device, vk); + pthread_mutex_lock(&device->mutex); for (unsigned i = 0; i < submission->command_buffer_count; i++) { struct nvk_cmd_buffer *cmd = (struct nvk_cmd_buffer *)submission->command_buffers[i]; + + for (uint32_t i = 0; i < submission->signal_count; i++) { + struct nvk_bo_sync *bo_sync = container_of(submission->signals[i].sync, struct nvk_bo_sync, sync); + nouveau_ws_push_ref(cmd->push, bo_sync->bo, NOUVEAU_WS_BO_RDWR); + } + nouveau_ws_push_submit(cmd->push, device->pdev->dev, device->ctx); } + for (uint32_t i = 0; i < submission->signal_count; i++) { + struct nvk_bo_sync *bo_sync = container_of(submission->signals[i].sync, struct nvk_bo_sync, sync); + assert(bo_sync->state == NVK_BO_SYNC_STATE_RESET); + bo_sync->state = NVK_BO_SYNC_STATE_SUBMITTED; + } + + pthread_cond_broadcast(&device->queue_submit); + pthread_mutex_unlock(&device->mutex); + return VK_SUCCESS; } @@ -62,6 +79,28 @@ nvk_CreateDevice(VkPhysicalDevice physicalDevice, if (result != VK_SUCCESS) goto fail_ctx; + if (pthread_mutex_init(&device->mutex, NULL) != 0) { + result = vk_error(device, VK_ERROR_INITIALIZATION_FAILED); + goto fail_queue; + } + + pthread_condattr_t condattr; + if (pthread_condattr_init(&condattr) != 0) { + result = vk_error(device, VK_ERROR_INITIALIZATION_FAILED); + goto fail_mutex; + } + if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC) != 0) { + pthread_condattr_destroy(&condattr); + result = vk_error(device, VK_ERROR_INITIALIZATION_FAILED); + goto fail_mutex; + } + if (pthread_cond_init(&device->queue_submit, &condattr) != 0) { + pthread_condattr_destroy(&condattr); + result = vk_error(device, VK_ERROR_INITIALIZATION_FAILED); + goto fail_mutex; + } + pthread_condattr_destroy(&condattr); + device->queue.driver_submit = nvk_queue_submit; device->pdev = physical_device; @@ -70,6 +109,10 @@ nvk_CreateDevice(VkPhysicalDevice physicalDevice, return VK_SUCCESS; +fail_mutex: + pthread_mutex_destroy(&device->mutex); +fail_queue: + vk_queue_finish(&device->queue); fail_ctx: nouveau_ws_context_destroy(device->ctx); fail_init: @@ -87,6 +130,8 @@ nvk_DestroyDevice(VkDevice _device, const VkAllocationCallbacks *pAllocator) if (!device) return; + pthread_cond_destroy(&device->queue_submit); + pthread_mutex_destroy(&device->mutex); vk_queue_finish(&device->queue); vk_device_finish(&device->vk); nouveau_ws_context_destroy(device->ctx); diff --git a/src/nouveau/vulkan/nvk_device.h b/src/nouveau/vulkan/nvk_device.h index d54a32f..f771ff6 100644 --- a/src/nouveau/vulkan/nvk_device.h +++ b/src/nouveau/vulkan/nvk_device.h @@ -16,6 +16,9 @@ struct nvk_device { struct nouveau_ws_context *ctx; struct vk_queue queue; + + pthread_mutex_t mutex; + pthread_cond_t queue_submit; }; VK_DEFINE_HANDLE_CASTS(nvk_device, vk.base, VkDevice, VK_OBJECT_TYPE_DEVICE) diff --git a/src/nouveau/vulkan/nvk_physical_device.c b/src/nouveau/vulkan/nvk_physical_device.c index b2252ea..1d30073 100644 --- a/src/nouveau/vulkan/nvk_physical_device.c +++ b/src/nouveau/vulkan/nvk_physical_device.c @@ -1,5 +1,6 @@ #include "nvk_physical_device.h" +#include "nvk_bo_sync.h" #include "nvk_entrypoints.h" #include "nvk_instance.h" #include "nvk_wsi.h" @@ -212,6 +213,12 @@ nvk_physical_device_try_create(struct nvk_instance *instance, device->mem_types[0].propertyFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT; } + unsigned st_idx = 0; + device->sync_types[st_idx++] = &nvk_bo_sync_type; + device->sync_types[st_idx++] = NULL; + assert(st_idx <= ARRAY_SIZE(device->sync_types)); + device->vk.supported_sync_types = device->sync_types; + *device_out = device; close(fd); diff --git a/src/nouveau/vulkan/nvk_physical_device.h b/src/nouveau/vulkan/nvk_physical_device.h index fccea04..8d10460 100644 --- a/src/nouveau/vulkan/nvk_physical_device.h +++ b/src/nouveau/vulkan/nvk_physical_device.h @@ -26,6 +26,8 @@ struct nvk_physical_device { VkMemoryType mem_types[2]; uint8_t mem_heap_cnt; uint8_t mem_type_cnt; + + const struct vk_sync_type *sync_types[2]; }; VK_DEFINE_HANDLE_CASTS(nvk_physical_device, -- 2.7.4