vulkan/queue: Rework submit thread enabling

Now that we have a threading mode in the device, we can set that based
on the environment variable instead of delaying it to submit time.  This
allows us to avoid the static variable trickery we use to avoid reading
environment variables over and over again.  We also move the enabling of
the submit thread up a level or two and give it a bit more obvious
condition.

Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/15566>

diff --git a/src/vulkan/runtime/vk_device.c b/src/vulkan/runtime/vk_device.c
index 8ced6b4..517820f 100644 (file)
--- a/src/vulkan/runtime/vk_device.c
+++ b/src/vulkan/runtime/vk_device.c
@@ -150,7 +150,11 @@ vk_device_init(struct vk_device *device,
       break;
 
    case VK_DEVICE_TIMELINE_MODE_ASSISTED:
-      device->submit_mode = VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND;
+      if (env_var_as_boolean("MESA_VK_ENABLE_SUBMIT_THREAD", false)) {
+         device->submit_mode = VK_QUEUE_SUBMIT_MODE_THREADED;
+      } else {
+         device->submit_mode = VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND;
+      }
       break;
 
    default:

diff --git a/src/vulkan/runtime/vk_queue.c b/src/vulkan/runtime/vk_queue.c
index f650cfe..7f42269 100644 (file)
--- a/src/vulkan/runtime/vk_queue.c
+++ b/src/vulkan/runtime/vk_queue.c
@@ -43,6 +43,9 @@
 
 #include "vulkan/wsi/wsi_common.h"
 
+static VkResult
+vk_queue_enable_submit_thread(struct vk_queue *queue);
+
 VkResult
 vk_queue_init(struct vk_queue *queue, struct vk_device *device,
               const VkDeviceQueueCreateInfo *pCreateInfo,
@@ -86,11 +89,19 @@ vk_queue_init(struct vk_queue *queue, struct vk_device *device,
       goto fail_pop;
    }
 
+   if (queue->submit.mode == VK_QUEUE_SUBMIT_MODE_THREADED) {
+      result = vk_queue_enable_submit_thread(queue);
+      if (result != VK_SUCCESS)
+         goto fail_thread;
+   }
+
    util_dynarray_init(&queue->labels, NULL);
    queue->region_begin = true;
 
    return VK_SUCCESS;
 
+fail_thread:
+   cnd_destroy(&queue->submit.pop);
 fail_pop:
    cnd_destroy(&queue->submit.push);
 fail_push:
@@ -559,6 +570,7 @@ static VkResult
 vk_queue_submit(struct vk_queue *queue,
                 const struct vulkan_submit_info *info)
 {
+   struct vk_device *device = queue->base.device;
    VkResult result;
    uint32_t sparse_memory_bind_entry_count = 0;
    uint32_t sparse_memory_image_bind_entry_count = 0;
@@ -778,31 +790,28 @@ vk_queue_submit(struct vk_queue *queue,
 
    assert(signal_count == submit->signal_count);
 
-   switch (queue->base.device->timeline_mode) {
-   case VK_DEVICE_TIMELINE_MODE_ASSISTED:
-      if (queue->submit.mode != VK_QUEUE_SUBMIT_MODE_THREADED) {
-         static int force_submit_thread = -1;
-         if (unlikely(force_submit_thread < 0)) {
-            force_submit_thread =
-               env_var_as_boolean("MESA_VK_ENABLE_SUBMIT_THREAD", false);
-         }
+   /* If this device supports threaded submit, we can't rely on the client
+    * ordering requirements to ensure submits happen in the right order.  Even
+    * if this queue doesn't have a submit thread, another queue (possibly in a
+    * different process) may and that means we our dependencies may not have
+    * been submitted to the kernel yet.  Do a quick zero-timeout WAIT_PENDING
+    * on all the wait semaphores to see if we need to start up our own thread.
+    */
+   if (device->submit_mode == VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND &&
+       queue->submit.mode != VK_QUEUE_SUBMIT_MODE_THREADED) {
+      assert(queue->submit.mode == VK_QUEUE_SUBMIT_MODE_IMMEDIATE);
 
-         if (unlikely(force_submit_thread)) {
-            result = vk_queue_enable_submit_thread(queue);
-         } else {
-            /* Otherwise, only enable the submit thread if we need it in order
-             * to resolve timeline semaphore wait-before-signal issues.
-             */
-            result = vk_sync_wait_many(queue->base.device,
-                                       submit->wait_count, submit->waits,
-                                       VK_SYNC_WAIT_PENDING, 0);
-            if (result == VK_TIMEOUT)
-               result = vk_queue_enable_submit_thread(queue);
-         }
-         if (unlikely(result != VK_SUCCESS))
-            goto fail;
-      }
+      result = vk_sync_wait_many(queue->base.device,
+                                 submit->wait_count, submit->waits,
+                                 VK_SYNC_WAIT_PENDING, 0);
+      if (result == VK_TIMEOUT)
+         result = vk_queue_enable_submit_thread(queue);
+      if (unlikely(result != VK_SUCCESS))
+         goto fail;
+   }
 
+   switch (queue->base.device->timeline_mode) {
+   case VK_DEVICE_TIMELINE_MODE_ASSISTED:
       if (queue->submit.mode == VK_QUEUE_SUBMIT_MODE_THREADED) {
          if (has_binary_permanent_semaphore_wait) {
             for (uint32_t i = 0; i < info->wait_count; i++) {