zink: support multiple heaps per memory type

some allocations require a different memory heap even when using the
same memory bits, so allow iterating over heaps of the same memory type
to find the one that works

Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/19281>

diff --git a/src/gallium/drivers/zink/zink_bo.c b/src/gallium/drivers/zink/zink_bo.c
index 01b6cdf..a511f5b 100644 (file)
--- a/src/gallium/drivers/zink/zink_bo.c
+++ b/src/gallium/drivers/zink/zink_bo.c
@@ -381,7 +381,7 @@ sparse_backing_alloc(struct zink_screen *screen, struct zink_bo *bo,
       size = MAX2(size, ZINK_SPARSE_BUFFER_PAGE_SIZE);
 
       buf = zink_bo_create(screen, size, ZINK_SPARSE_BUFFER_PAGE_SIZE,
-                           ZINK_HEAP_DEVICE_LOCAL, 0, screen->heap_map[ZINK_HEAP_DEVICE_LOCAL], NULL);
+                           ZINK_HEAP_DEVICE_LOCAL, 0, screen->heap_map[ZINK_HEAP_DEVICE_LOCAL][0], NULL);
       if (!buf) {
          FREE(best_backing->chunks);
          FREE(best_backing);

diff --git a/src/gallium/drivers/zink/zink_bo.h b/src/gallium/drivers/zink/zink_bo.h
index fe61b64..4d3a89b 100644 (file)
--- a/src/gallium/drivers/zink/zink_bo.h
+++ b/src/gallium/drivers/zink/zink_bo.h
@@ -93,6 +93,17 @@ zink_heap_from_domain_flags(VkMemoryPropertyFlags domains, enum zink_alloc_flag
    return ZINK_HEAP_HOST_VISIBLE_COHERENT;
 }
 
+static inline unsigned
+zink_heap_idx_from_bits(struct zink_screen *screen, enum zink_heap heap, uint32_t bits)
+{
+   for (unsigned i = 0; i < screen->heap_count[heap]; i++) {
+      if (bits & BITFIELD_BIT(screen->heap_map[heap][i])) {
+         return screen->heap_map[heap][i];
+      }
+   }
+   return UINT32_MAX;
+}
+
 bool
 zink_bo_init(struct zink_screen *screen);
 

diff --git a/src/gallium/drivers/zink/zink_resource.c b/src/gallium/drivers/zink/zink_resource.c
index fc3c1bd..af55ab2 100644 (file)
--- a/src/gallium/drivers/zink/zink_resource.c
+++ b/src/gallium/drivers/zink/zink_resource.c
@@ -943,8 +943,8 @@ resource_object_create(struct zink_screen *screen, const struct pipe_resource *t
    mai.pNext = NULL;
    mai.allocationSize = reqs.size;
    enum zink_heap heap = zink_heap_from_domain_flags(flags, aflags);
-   mai.memoryTypeIndex = screen->heap_map[heap];
-   if (unlikely(!(reqs.memoryTypeBits & BITFIELD_BIT(mai.memoryTypeIndex)))) {
+   mai.memoryTypeIndex = zink_heap_idx_from_bits(screen, heap, reqs.memoryTypeBits);
+   if (mai.memoryTypeIndex == UINT32_MAX) {
       /* not valid based on reqs; demote to more compatible type */
       switch (heap) {
       case ZINK_HEAP_DEVICE_LOCAL_VISIBLE:
@@ -956,9 +956,10 @@ resource_object_create(struct zink_screen *screen, const struct pipe_resource *t
       default:
          break;
       }
-      mai.memoryTypeIndex = screen->heap_map[heap];
-      assert(reqs.memoryTypeBits & BITFIELD_BIT(mai.memoryTypeIndex));
+      mai.memoryTypeIndex = zink_heap_idx_from_bits(screen, heap, reqs.memoryTypeBits);
+      assert(mai.memoryTypeIndex != UINT32_MAX);
    }
+   assert(reqs.memoryTypeBits & BITFIELD_BIT(mai.memoryTypeIndex));
 
    VkMemoryDedicatedAllocateInfo ded_alloc_info = {
       .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,

diff --git a/src/gallium/drivers/zink/zink_screen.c b/src/gallium/drivers/zink/zink_screen.c
index 842f6c0..eafdaa6 100644 (file)
--- a/src/gallium/drivers/zink/zink_screen.c
+++ b/src/gallium/drivers/zink/zink_screen.c
@@ -382,8 +382,10 @@ get_smallest_buffer_heap(struct zink_screen *screen)
    };
    unsigned size = UINT32_MAX;
    for (unsigned i = 0; i < ARRAY_SIZE(heaps); i++) {
-      unsigned heap_idx = screen->info.mem_props.memoryTypes[screen->heap_map[i]].heapIndex;
-      size = MIN2(screen->info.mem_props.memoryHeaps[heap_idx].size, size);
+      for (unsigned j = 0; j < screen->heap_count[i]; j++) {
+         unsigned heap_idx = screen->info.mem_props.memoryTypes[screen->heap_map[i][j]].heapIndex;
+         size = MIN2(screen->info.mem_props.memoryHeaps[heap_idx].size, size);
+      }
    }
    return size;
 }
@@ -2548,30 +2550,35 @@ zink_internal_create_screen(const struct pipe_screen_config *config)
       for (unsigned j = 0; j < screen->info.mem_props.memoryTypeCount; j++) {
          VkMemoryPropertyFlags domains = vk_domain_from_heap(i);
          if ((screen->info.mem_props.memoryTypes[j].propertyFlags & domains) == domains) {
-            assert(screen->heap_map[i] == UINT8_MAX);
-            screen->heap_map[i] = j;
-            break;
+            screen->heap_map[i][screen->heap_count[i]++] = j;
          }
       }
-
+   }
+   /* iterate again to check for missing heaps */
+   for (enum zink_heap i = 0; i < ZINK_HEAP_MAX; i++) {
       /* not found: use compatible heap */
-      if (screen->heap_map[i] == UINT8_MAX) {
+      if (screen->heap_map[i][0] == UINT8_MAX) {
          /* only cached mem has a failure case for now */
          assert(i == ZINK_HEAP_HOST_VISIBLE_CACHED || i == ZINK_HEAP_DEVICE_LOCAL_LAZY ||
                 i == ZINK_HEAP_DEVICE_LOCAL_VISIBLE);
-         if (i == ZINK_HEAP_HOST_VISIBLE_CACHED)
-            screen->heap_map[i] = screen->heap_map[ZINK_HEAP_HOST_VISIBLE_COHERENT];
-         else
-            screen->heap_map[i] = screen->heap_map[ZINK_HEAP_DEVICE_LOCAL];
+         if (i == ZINK_HEAP_HOST_VISIBLE_CACHED) {
+            memcpy(screen->heap_map[i], screen->heap_map[ZINK_HEAP_HOST_VISIBLE_COHERENT], screen->heap_count[ZINK_HEAP_HOST_VISIBLE_COHERENT]);
+            screen->heap_count[i] = screen->heap_count[ZINK_HEAP_HOST_VISIBLE_COHERENT];
+         } else {
+            memcpy(screen->heap_map[i], screen->heap_map[ZINK_HEAP_DEVICE_LOCAL], screen->heap_count[ZINK_HEAP_DEVICE_LOCAL]);
+            screen->heap_count[i] = screen->heap_count[ZINK_HEAP_DEVICE_LOCAL];
+         }
       }
-      screen->heap_flags[i] = screen->info.mem_props.memoryTypes[screen->heap_map[i]].propertyFlags;
    }
    {
-      unsigned vis_vram = screen->heap_map[ZINK_HEAP_DEVICE_LOCAL_VISIBLE];
-      unsigned vram = screen->heap_map[ZINK_HEAP_DEVICE_LOCAL];
+      uint64_t biggest_vis_vram = 0;
+      for (unsigned i = 0; i < screen->heap_count[ZINK_HEAP_DEVICE_LOCAL_VISIBLE]; i++)
+         biggest_vis_vram = MAX2(biggest_vis_vram, screen->info.mem_props.memoryHeaps[screen->info.mem_props.memoryTypes[i].heapIndex].size);
+      uint64_t biggest_vram = 0;
+      for (unsigned i = 0; i < screen->heap_count[ZINK_HEAP_DEVICE_LOCAL]; i++)
+         biggest_vram = MAX2(biggest_vis_vram, screen->info.mem_props.memoryHeaps[screen->info.mem_props.memoryTypes[i].heapIndex].size);
       /* determine if vis vram is roughly equal to total vram */
-      if (screen->info.mem_props.memoryHeaps[screen->info.mem_props.memoryTypes[vis_vram].heapIndex].size >
-          screen->info.mem_props.memoryHeaps[screen->info.mem_props.memoryTypes[vram].heapIndex].size * 0.9)
+      if (biggest_vis_vram > biggest_vram * 0.9)
          screen->resizable_bar = true;
    }
 

diff --git a/src/gallium/drivers/zink/zink_types.h b/src/gallium/drivers/zink/zink_types.h
index 5c27b1d..0ed84e0 100644 (file)
--- a/src/gallium/drivers/zink/zink_types.h
+++ b/src/gallium/drivers/zink/zink_types.h
@@ -1187,8 +1187,8 @@ struct zink_screen {
       unsigned min_alloc_size;
       uint32_t next_bo_unique_id;
    } pb;
-   uint8_t heap_map[VK_MAX_MEMORY_TYPES];
-   VkMemoryPropertyFlags heap_flags[VK_MAX_MEMORY_TYPES];
+   uint8_t heap_map[ZINK_HEAP_MAX][VK_MAX_MEMORY_TYPES];
+   uint8_t heap_count[ZINK_HEAP_MAX];
    bool resizable_bar;
 
    uint64_t total_video_mem;