#include "util/mesa-sha1.h"
#include "util/u_atomic.h"
#include "util/streaming-load-memcpy.h"
+#include "radv_cs.h"
#include "radv_debug.h"
#include "radv_meta.h"
#include "radv_private.h"
#include "aco_interface.h"
#include "sid.h"
#include "vk_format.h"
+#include "vk_sync.h"
+#include "vk_semaphore.h"
#include "aco_shader_info.h"
#include "radv_aco_shader_info.h"
list_add(&block->pool, &device->shader_block_obj_pool);
}
+VkResult
+radv_shader_wait_for_upload(struct radv_device *device, uint64_t seq)
+{
+ if (!seq)
+ return VK_SUCCESS;
+
+ const VkSemaphoreWaitInfo wait_info = {
+ .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
+ .pSemaphores = &device->shader_upload_sem,
+ .semaphoreCount = 1,
+ .pValues = &seq,
+ };
+ return device->vk.dispatch_table.WaitSemaphores(radv_device_to_handle(device), &wait_info,
+ UINT64_MAX);
+}
+
/* Segregated fit allocator, implementing a good-fit allocation policy.
*
* This is an variation of sequential fit allocation with several lists of free blocks ("holes")
MAX2(RADV_SHADER_ALLOC_MIN_ARENA_SIZE
<< MIN2(RADV_SHADER_ALLOC_MAX_ARENA_SIZE_SHIFT, device->shader_arena_shift),
size);
- VkResult result = device->ws->buffer_create(
- device->ws, arena_size, RADV_SHADER_ALLOC_ALIGNMENT, RADEON_DOMAIN_VRAM,
- RADEON_FLAG_NO_INTERPROCESS_SHARING | RADEON_FLAG_32BIT |
+ enum radeon_bo_flag flags = RADEON_FLAG_NO_INTERPROCESS_SHARING | RADEON_FLAG_32BIT;
+ if (device->shader_use_invisible_vram)
+ flags |= RADEON_FLAG_NO_CPU_ACCESS;
+ else
+ flags |=
(device->physical_device->rad_info.cpdma_prefetch_writes_memory ? 0
- : RADEON_FLAG_READ_ONLY),
- RADV_BO_PRIORITY_SHADER, 0, &arena->bo);
+ : RADEON_FLAG_READ_ONLY);
+
+ VkResult result;
+ result =
+ device->ws->buffer_create(device->ws, arena_size, RADV_SHADER_ALLOC_ALIGNMENT,
+ RADEON_DOMAIN_VRAM, flags, RADV_BO_PRIORITY_SHADER, 0, &arena->bo);
if (result != VK_SUCCESS)
goto fail;
radv_rmv_log_bo_allocate(device, arena->bo, arena_size, true);
list_inithead(&arena->entries);
- arena->ptr = (char *)device->ws->buffer_map(arena->bo);
- if (!arena->ptr)
- goto fail;
+ if (!(flags & RADEON_FLAG_NO_CPU_ACCESS)) {
+ arena->ptr = (char *)device->ws->buffer_map(arena->bo);
+ if (!arena->ptr)
+ goto fail;
+ }
alloc = alloc_block_obj(device);
hole = arena_size - size > 0 ? alloc_block_obj(device) : alloc;
mtx_destroy(&device->shader_arena_mutex);
}
+VkResult
+radv_init_shader_upload_queue(struct radv_device *device)
+{
+ if (!device->shader_use_invisible_vram)
+ return VK_SUCCESS;
+
+ VkDevice vk_device = radv_device_to_handle(device);
+ struct radeon_winsys *ws = device->ws;
+
+ const struct vk_device_dispatch_table *disp = &device->vk.dispatch_table;
+ VkResult result = VK_SUCCESS;
+
+ result = ws->ctx_create(ws, RADEON_CTX_PRIORITY_MEDIUM, &device->shader_upload_hw_ctx);
+ if (result != VK_SUCCESS)
+ return result;
+ mtx_init(&device->shader_upload_hw_ctx_mutex, mtx_plain);
+
+ mtx_init(&device->shader_dma_submission_list_mutex, mtx_plain);
+ cnd_init(&device->shader_dma_submission_list_cond);
+ list_inithead(&device->shader_dma_submissions);
+
+ for (unsigned i = 0; i < RADV_SHADER_UPLOAD_CS_COUNT; i++) {
+ struct radv_shader_dma_submission *submission = calloc(1, sizeof(struct radv_shader_dma_submission));
+ submission->cs = ws->cs_create(ws, AMD_IP_SDMA);
+ if (!submission->cs)
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+ list_addtail(&submission->list, &device->shader_dma_submissions);
+ }
+
+ const VkSemaphoreTypeCreateInfo sem_type = {
+ .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
+ .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
+ .initialValue = 0,
+ };
+ const VkSemaphoreCreateInfo sem_create = {
+ .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
+ .pNext = &sem_type,
+ };
+ result = disp->CreateSemaphore(vk_device, &sem_create, NULL, &device->shader_upload_sem);
+ if (result != VK_SUCCESS)
+ return result;
+
+ return VK_SUCCESS;
+}
+
+void
+radv_destroy_shader_upload_queue(struct radv_device *device)
+{
+ if (!device->shader_use_invisible_vram)
+ return;
+
+ struct vk_device_dispatch_table *disp = &device->vk.dispatch_table;
+ struct radeon_winsys *ws = device->ws;
+
+ /* Upload queue should be idle assuming that pipelines are not leaked */
+ if (device->shader_upload_sem)
+ disp->DestroySemaphore(radv_device_to_handle(device), device->shader_upload_sem, NULL);
+
+ list_for_each_entry_safe(struct radv_shader_dma_submission, submission,
+ &device->shader_dma_submissions, list)
+ {
+ if (submission->cs)
+ ws->cs_destroy(submission->cs);
+ if (submission->bo)
+ ws->buffer_destroy(ws, submission->bo);
+ list_del(&submission->list);
+ free(submission);
+ }
+
+ cnd_destroy(&device->shader_dma_submission_list_cond);
+ mtx_destroy(&device->shader_dma_submission_list_mutex);
+
+ if (device->shader_upload_hw_ctx) {
+ mtx_destroy(&device->shader_upload_hw_ctx_mutex);
+ ws->ctx_destroy(device->shader_upload_hw_ctx);
+ }
+}
+
/* For the UMR disassembler. */
#define DEBUGGER_END_OF_CODE_MARKER 0xbf9f0000 /* invalid instruction */
#define DEBUGGER_NUM_MARKERS 5
static bool
radv_shader_binary_upload(struct radv_device *device, const struct radv_shader_binary *binary,
- struct radv_shader *shader)
+ struct radv_shader *shader, void *dest_ptr)
{
- void *dest_ptr;
-
- shader->alloc = radv_alloc_shader_memory(device, shader->code_size, shader);
- if (!shader->alloc)
- return false;
-
- shader->bo = shader->alloc->arena->bo;
- shader->va = radv_buffer_get_va(shader->bo) + shader->alloc->offset;
-
- dest_ptr = shader->alloc->arena->ptr + shader->alloc->offset;
-
if (device->thread_trace.bo) {
shader->code = calloc(shader->code_size, 1);
if (!shader->code) {
return true;
}
+static VkResult
+radv_shader_dma_resize_upload_buf(struct radv_shader_dma_submission *submission,
+ struct radeon_winsys *ws, uint64_t size)
+{
+ if (submission->bo)
+ ws->buffer_destroy(ws, submission->bo);
+
+ VkResult result =
+ ws->buffer_create(ws, size, RADV_SHADER_ALLOC_ALIGNMENT, RADEON_DOMAIN_GTT,
+ RADEON_FLAG_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING |
+ RADEON_FLAG_32BIT | RADEON_FLAG_GTT_WC,
+ RADV_BO_PRIORITY_UPLOAD_BUFFER, 0, &submission->bo);
+ if (result != VK_SUCCESS)
+ return result;
+
+ submission->ptr = ws->buffer_map(submission->bo);
+ submission->bo_size = size;
+
+ return VK_SUCCESS;
+}
+
+struct radv_shader_dma_submission *
+radv_shader_dma_pop_submission(struct radv_device *device)
+{
+ struct radv_shader_dma_submission *submission;
+
+ mtx_lock(&device->shader_dma_submission_list_mutex);
+
+ while (list_is_empty(&device->shader_dma_submissions))
+ cnd_wait(&device->shader_dma_submission_list_cond, &device->shader_dma_submission_list_mutex);
+
+ submission =
+ list_first_entry(&device->shader_dma_submissions, struct radv_shader_dma_submission, list);
+ list_del(&submission->list);
+
+ mtx_unlock(&device->shader_dma_submission_list_mutex);
+
+ return submission;
+}
+
+void
+radv_shader_dma_push_submission(struct radv_device *device,
+ struct radv_shader_dma_submission *submission, uint64_t seq)
+{
+ submission->seq = seq;
+
+ mtx_lock(&device->shader_dma_submission_list_mutex);
+
+ list_addtail(&submission->list, &device->shader_dma_submissions);
+ cnd_signal(&device->shader_dma_submission_list_cond);
+
+ mtx_unlock(&device->shader_dma_submission_list_mutex);
+}
+
+struct radv_shader_dma_submission *
+radv_shader_dma_get_submission(struct radv_device *device, struct radeon_winsys_bo *bo, uint64_t va,
+ uint64_t size)
+{
+ struct radv_shader_dma_submission *submission = radv_shader_dma_pop_submission(device);
+ struct radeon_cmdbuf *cs = submission->cs;
+ struct radeon_winsys *ws = device->ws;
+ VkResult result;
+
+ /* Wait for potentially in-flight submission to settle */
+ result = radv_shader_wait_for_upload(device, submission->seq);
+ if (result != VK_SUCCESS)
+ goto fail;
+
+ ws->cs_reset(cs);
+
+ if (submission->bo_size < size) {
+ result = radv_shader_dma_resize_upload_buf(submission, ws, size);
+ if (result != VK_SUCCESS)
+ goto fail;
+ }
+
+ radv_sdma_copy_buffer(device, cs, radv_buffer_get_va(submission->bo), va, size);
+ radv_cs_add_buffer(ws, cs, submission->bo);
+ radv_cs_add_buffer(ws, cs, bo);
+
+ result = ws->cs_finalize(cs);
+ if (result != VK_SUCCESS)
+ goto fail;
+
+ return submission;
+
+fail:
+ radv_shader_dma_push_submission(device, submission, 0);
+
+ return NULL;
+}
+
+/*
+ * If upload_seq_out is NULL, this function blocks until the DMA is complete. Otherwise, the
+ * semaphore value to wait on device->shader_upload_sem is stored in *upload_seq_out.
+ */
+bool
+radv_shader_dma_submit(struct radv_device *device, struct radv_shader_dma_submission *submission,
+ uint64_t *upload_seq_out)
+{
+ struct radeon_cmdbuf *cs = submission->cs;
+ struct radeon_winsys *ws = device->ws;
+ VkResult result;
+
+ mtx_lock(&device->shader_upload_hw_ctx_mutex);
+
+ uint64_t upload_seq = device->shader_upload_seq + 1;
+
+ struct vk_semaphore *semaphore = vk_semaphore_from_handle(device->shader_upload_sem);
+ struct vk_sync *sync = vk_semaphore_get_active_sync(semaphore);
+ const struct vk_sync_signal signal_info = {
+ .sync = sync,
+ .signal_value = upload_seq,
+ .stage_mask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
+ };
+
+ struct radv_winsys_submit_info submit = {
+ .ip_type = AMD_IP_SDMA,
+ .queue_index = 0,
+ .cs_array = &cs,
+ .cs_count = 1,
+ };
+
+ result = ws->cs_submit(device->shader_upload_hw_ctx, &submit, 0, NULL, 1, &signal_info, false);
+ if (result != VK_SUCCESS)
+ {
+ mtx_unlock(&device->shader_upload_hw_ctx_mutex);
+ radv_shader_dma_push_submission(device, submission, 0);
+ return false;
+ }
+ device->shader_upload_seq = upload_seq;
+ mtx_unlock(&device->shader_upload_hw_ctx_mutex);
+
+ radv_shader_dma_push_submission(device, submission, upload_seq);
+
+ if (upload_seq_out) {
+ *upload_seq_out = upload_seq;
+ } else {
+ result = radv_shader_wait_for_upload(device, upload_seq);
+ if (result != VK_SUCCESS)
+ return false;
+ }
+
+ return true;
+}
+
+
struct radv_shader *
radv_shader_create(struct radv_device *device, const struct radv_shader_binary *binary,
bool keep_shader_info, bool from_cache, const struct radv_shader_args *args)
}
}
- if (!radv_shader_binary_upload(device, binary, shader))
+ shader->alloc = radv_alloc_shader_memory(device, shader->code_size, shader);
+ if (!shader->alloc)
return NULL;
+ shader->bo = shader->alloc->arena->bo;
+ shader->va = radv_buffer_get_va(shader->bo) + shader->alloc->offset;
+
+ if (device->shader_use_invisible_vram) {
+ struct radv_shader_dma_submission *submission =
+ radv_shader_dma_get_submission(device, shader->bo, shader->va, shader->code_size);
+ if (!submission)
+ return NULL;
+
+ if (!radv_shader_binary_upload(device, binary, shader, submission->ptr)) {
+ radv_shader_dma_push_submission(device, submission, 0);
+ return NULL;
+ }
+
+ if (!radv_shader_dma_submit(device, submission, NULL))
+ return NULL;
+ } else {
+ void *dest_ptr = shader->alloc->arena->ptr + shader->alloc->offset;
+
+ if (!radv_shader_binary_upload(device, binary, shader, dest_ptr))
+ return NULL;
+ }
+
return shader;
}
return shader_part;
}
-void
-radv_shader_part_binary_upload(const struct radv_shader_part_binary *binary, void *dest_ptr)
+bool
+radv_shader_part_binary_upload(struct radv_device *device, struct radv_shader_part *shader_part)
{
- memcpy(dest_ptr, binary->data, binary->code_size);
+ const struct radv_shader_part_binary *bin = shader_part->binary;
+ uint32_t code_size = radv_get_shader_binary_size(bin->code_size);
+ struct radv_shader_dma_submission *submission = NULL;
+ void *dest_ptr;
+ if (device->shader_use_invisible_vram) {
+ uint64_t va = radv_buffer_get_va(shader_part->alloc->arena->bo) + shader_part->alloc->offset;
+ submission =
+ radv_shader_dma_get_submission(device, shader_part->alloc->arena->bo, va, code_size);
+ if (!submission)
+ return false;
+
+ dest_ptr = submission->ptr;
+ } else {
+ dest_ptr = shader_part->alloc->arena->ptr + shader_part->alloc->offset;
+ }
+
+ memcpy(dest_ptr, bin->data, bin->code_size);
/* Add end-of-code markers for the UMR disassembler. */
- uint32_t *ptr32 = (uint32_t *)dest_ptr + binary->code_size / 4;
+ uint32_t *ptr32 = (uint32_t *)dest_ptr + code_size / 4;
for (unsigned i = 0; i < DEBUGGER_NUM_MARKERS; i++)
ptr32[i] = DEBUGGER_END_OF_CODE_MARKER;
+
+ if (device->shader_use_invisible_vram) {
+ if (!radv_shader_dma_submit(device, submission, NULL))
+ return false;
+ }
+
+ return true;
}
static char *
prolog->bo = prolog->alloc->arena->bo;
prolog->va = radv_buffer_get_va(prolog->bo) + prolog->alloc->offset;
- void *dest_ptr = prolog->alloc->arena->ptr + prolog->alloc->offset;
- radv_shader_part_binary_upload(binary, dest_ptr);
+ if (!radv_shader_part_binary_upload(device, prolog))
+ goto fail_alloc;
if (options.dump_shader) {
fprintf(stderr, "Vertex prolog");
epilog->bo = epilog->alloc->arena->bo;
epilog->va = radv_buffer_get_va(epilog->bo) + epilog->alloc->offset;
- void *dest_ptr = epilog->alloc->arena->ptr + epilog->alloc->offset;
- radv_shader_part_binary_upload(binary, dest_ptr);
+ if (!radv_shader_part_binary_upload(device, epilog))
+ goto fail_alloc;
if (options.dump_shader) {
fprintf(stderr, "Fragment epilog");
projects / platform / upstream / mesa.git / commitdiff