TU_FROM_HANDLE(tu_image, src_image, srcImage);
TU_FROM_HANDLE(tu_image, dst_image, dstImage);
- tu_bo_list_add(&cmd->bo_list, src_image->bo, MSM_SUBMIT_BO_READ);
- tu_bo_list_add(&cmd->bo_list, dst_image->bo, MSM_SUBMIT_BO_WRITE);
-
for (uint32_t i = 0; i < regionCount; ++i) {
/* can't blit both depth and stencil at once with D32_S8
* TODO: more advanced 3D blit path to support it instead?
TU_FROM_HANDLE(tu_image, dst_image, dstImage);
TU_FROM_HANDLE(tu_buffer, src_buffer, srcBuffer);
- tu_bo_list_add(&cmd->bo_list, src_buffer->bo, MSM_SUBMIT_BO_READ);
- tu_bo_list_add(&cmd->bo_list, dst_image->bo, MSM_SUBMIT_BO_WRITE);
-
for (unsigned i = 0; i < regionCount; ++i)
tu_copy_buffer_to_image(cmd, src_buffer, dst_image, pRegions + i);
}
TU_FROM_HANDLE(tu_image, src_image, srcImage);
TU_FROM_HANDLE(tu_buffer, dst_buffer, dstBuffer);
- tu_bo_list_add(&cmd->bo_list, src_image->bo, MSM_SUBMIT_BO_READ);
- tu_bo_list_add(&cmd->bo_list, dst_buffer->bo, MSM_SUBMIT_BO_WRITE);
-
for (unsigned i = 0; i < regionCount; ++i)
tu_copy_image_to_buffer(cmd, src_image, dst_buffer, pRegions + i);
}
return;
}
- tu_bo_list_add(&cmd->bo_list, staging_image.bo,
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
-
struct tu_image_view staging;
tu_image_view_copy(&staging, &staging_image, src_format,
&staging_subresource, 0, false);
TU_FROM_HANDLE(tu_image, src_image, srcImage);
TU_FROM_HANDLE(tu_image, dst_image, destImage);
- tu_bo_list_add(&cmd->bo_list, src_image->bo, MSM_SUBMIT_BO_READ);
- tu_bo_list_add(&cmd->bo_list, dst_image->bo, MSM_SUBMIT_BO_WRITE);
-
for (uint32_t i = 0; i < regionCount; ++i)
tu_copy_image_to_image(cmd, src_image, dst_image, pRegions + i);
}
TU_FROM_HANDLE(tu_buffer, src_buffer, srcBuffer);
TU_FROM_HANDLE(tu_buffer, dst_buffer, dstBuffer);
- tu_bo_list_add(&cmd->bo_list, src_buffer->bo, MSM_SUBMIT_BO_READ);
- tu_bo_list_add(&cmd->bo_list, dst_buffer->bo, MSM_SUBMIT_BO_WRITE);
-
for (unsigned i = 0; i < regionCount; ++i) {
copy_buffer(cmd,
tu_buffer_iova(dst_buffer) + pRegions[i].dstOffset,
TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
TU_FROM_HANDLE(tu_buffer, buffer, dstBuffer);
- tu_bo_list_add(&cmd->bo_list, buffer->bo, MSM_SUBMIT_BO_WRITE);
-
struct tu_cs_memory tmp;
VkResult result = tu_cs_alloc(&cmd->sub_cs, DIV_ROUND_UP(dataSize, 64), 64, &tmp);
if (result != VK_SUCCESS) {
const struct blit_ops *ops = &r2d_ops;
struct tu_cs *cs = &cmd->cs;
- tu_bo_list_add(&cmd->bo_list, buffer->bo, MSM_SUBMIT_BO_WRITE);
-
if (fillSize == VK_WHOLE_SIZE)
fillSize = buffer->size - dstOffset;
const struct blit_ops *ops = &r2d_ops;
struct tu_cs *cs = &cmd->cs;
- tu_bo_list_add(&cmd->bo_list, src_image->bo, MSM_SUBMIT_BO_READ);
- tu_bo_list_add(&cmd->bo_list, dst_image->bo, MSM_SUBMIT_BO_WRITE);
-
ops->setup(cmd, cs, dst_image->vk_format, VK_IMAGE_ASPECT_COLOR_BIT,
ROTATE_0, false, dst_image->layout[0].ubwc);
{
const struct blit_ops *ops = &r2d_ops;
- tu_bo_list_add(&cmd->bo_list, src->image->bo, MSM_SUBMIT_BO_READ);
- tu_bo_list_add(&cmd->bo_list, dst->image->bo, MSM_SUBMIT_BO_WRITE);
-
assert(src->image->vk_format == dst->image->vk_format);
ops->setup(cmd, cs, dst->image->vk_format, VK_IMAGE_ASPECT_COLOR_BIT,
TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
TU_FROM_HANDLE(tu_image, image, image_h);
- tu_bo_list_add(&cmd->bo_list, image->bo, MSM_SUBMIT_BO_WRITE);
-
for (unsigned i = 0; i < rangeCount; i++)
clear_image(cmd, image, (const VkClearValue*) pColor, pRanges + i, VK_IMAGE_ASPECT_COLOR_BIT);
}
TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
TU_FROM_HANDLE(tu_image, image, image_h);
- tu_bo_list_add(&cmd->bo_list, image->bo, MSM_SUBMIT_BO_WRITE);
-
for (unsigned i = 0; i < rangeCount; i++) {
const VkImageSubresourceRange *range = &pRanges[i];
#include "tu_cs.h"
void
-tu_bo_list_init(struct tu_bo_list *list)
-{
- list->count = list->capacity = 0;
- list->bo_infos = NULL;
-}
-
-void
-tu_bo_list_destroy(struct tu_bo_list *list)
-{
- free(list->bo_infos);
-}
-
-void
-tu_bo_list_reset(struct tu_bo_list *list)
-{
- list->count = 0;
-}
-
-/**
- * \a flags consists of MSM_SUBMIT_BO_FLAGS.
- */
-static uint32_t
-tu_bo_list_add_info(struct tu_bo_list *list,
- const struct drm_msm_gem_submit_bo *bo_info)
-{
- assert(bo_info->handle != 0);
-
- for (uint32_t i = 0; i < list->count; ++i) {
- if (list->bo_infos[i].handle == bo_info->handle) {
- assert(list->bo_infos[i].presumed == bo_info->presumed);
- list->bo_infos[i].flags |= bo_info->flags;
- return i;
- }
- }
-
- /* grow list->bo_infos if needed */
- if (list->count == list->capacity) {
- uint32_t new_capacity = MAX2(2 * list->count, 16);
- struct drm_msm_gem_submit_bo *new_bo_infos = realloc(
- list->bo_infos, new_capacity * sizeof(struct drm_msm_gem_submit_bo));
- if (!new_bo_infos)
- return TU_BO_LIST_FAILED;
- list->bo_infos = new_bo_infos;
- list->capacity = new_capacity;
- }
-
- list->bo_infos[list->count] = *bo_info;
- return list->count++;
-}
-
-uint32_t
-tu_bo_list_add(struct tu_bo_list *list,
- const struct tu_bo *bo,
- uint32_t flags)
-{
- return tu_bo_list_add_info(list, &(struct drm_msm_gem_submit_bo) {
- .flags = flags,
- .handle = bo->gem_handle,
- .presumed = bo->iova,
- });
-}
-
-VkResult
-tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other)
-{
- for (uint32_t i = 0; i < other->count; i++) {
- if (tu_bo_list_add_info(list, other->bo_infos + i) == TU_BO_LIST_FAILED)
- return VK_ERROR_OUT_OF_HOST_MEMORY;
- }
-
- return VK_SUCCESS;
-}
-
-void
tu6_emit_event_write(struct tu_cmd_buffer *cmd,
struct tu_cs *cs,
enum vgt_event_type event)
A6XX_VSC_DRAW_STRM_ADDRESS(.bo = vsc_bo,
.bo_offset = cmd->vsc_prim_strm_pitch * MAX_VSC_PIPES));
- tu_bo_list_add(&cmd->bo_list, vsc_bo, MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
-
tu_cs_sanity_check(cs);
}
cmd_buffer->queue_family_index = TU_QUEUE_GENERAL;
}
- tu_bo_list_init(&cmd_buffer->bo_list);
tu_cs_init(&cmd_buffer->cs, device, TU_CS_MODE_GROW, 4096);
tu_cs_init(&cmd_buffer->draw_cs, device, TU_CS_MODE_GROW, 4096);
tu_cs_init(&cmd_buffer->draw_epilogue_cs, device, TU_CS_MODE_GROW, 4096);
tu_cs_finish(&cmd_buffer->draw_epilogue_cs);
tu_cs_finish(&cmd_buffer->sub_cs);
- tu_bo_list_destroy(&cmd_buffer->bo_list);
vk_object_free(&cmd_buffer->device->vk, &cmd_buffer->pool->alloc, cmd_buffer);
}
{
cmd_buffer->record_result = VK_SUCCESS;
- tu_bo_list_reset(&cmd_buffer->bo_list);
tu_cs_reset(&cmd_buffer->cs);
tu_cs_reset(&cmd_buffer->draw_cs);
tu_cs_reset(&cmd_buffer->draw_epilogue_cs);
cmd->state.vb[firstBinding + i].base = tu_buffer_iova(buf) + pOffsets[i];
cmd->state.vb[firstBinding + i].size = buf->size - pOffsets[i];
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
}
for (uint32_t i = 0; i < MAX_VBS; i++) {
cmd->state.index_va = buf->bo->iova + buf->bo_offset + offset;
cmd->state.max_index_count = (buf->size - offset) >> index_shift;
cmd->state.index_size = index_size;
-
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
}
void
dst[5] = va >> 32;
}
}
-
- for (unsigned j = 0; j < set->layout->buffer_count; ++j) {
- if (set->buffers[j]) {
- tu_bo_list_add(&cmd->bo_list, set->buffers[j],
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
- }
- }
-
- if (set->size > 0) {
- tu_bo_list_add(&cmd->bo_list, &set->pool->bo,
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
- }
}
assert(dyn_idx == dynamicOffsetCount);
tu_cs_emit(cs, size + offset);
cmd->state.streamout_offset[idx] = offset;
-
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_WRITE);
}
tu_cond_exec_end(cs);
TU_FROM_HANDLE(tu_buffer, buf, pCounterBuffers[i]);
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
-
tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(REG_A6XX_VPC_SO_BUFFER_OFFSET(idx)) |
CP_MEM_TO_REG_0_UNK31 |
TU_FROM_HANDLE(tu_buffer, buf, pCounterBuffers[i]);
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_WRITE);
-
/* VPC_SO_FLUSH_BASE has dwords counter, but counter should be in bytes */
tu_cs_emit_pkt7(cs, CP_MEM_TO_REG, 3);
tu_cs_emit(cs, CP_MEM_TO_REG_0_REG(REG_A6XX_CP_SCRATCH_REG(0)) |
tu_emit_cache_flush(cmd_buffer, &cmd_buffer->cs);
}
- tu_bo_list_add(&cmd_buffer->bo_list, &cmd_buffer->device->global_bo,
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
-
- for (uint32_t i = 0; i < cmd_buffer->draw_cs.bo_count; i++) {
- tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_cs.bos[i],
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
- }
-
- for (uint32_t i = 0; i < cmd_buffer->draw_epilogue_cs.bo_count; i++) {
- tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->draw_epilogue_cs.bos[i],
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
- }
-
- for (uint32_t i = 0; i < cmd_buffer->sub_cs.bo_count; i++) {
- tu_bo_list_add(&cmd_buffer->bo_list, cmd_buffer->sub_cs.bos[i],
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
- }
-
tu_cs_end(&cmd_buffer->cs);
tu_cs_end(&cmd_buffer->draw_cs);
tu_cs_end(&cmd_buffer->draw_epilogue_cs);
TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
TU_FROM_HANDLE(tu_pipeline, pipeline, _pipeline);
- for (uint32_t i = 0; i < pipeline->cs.bo_count; i++) {
- tu_bo_list_add(&cmd->bo_list, pipeline->cs.bos[i],
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
- }
-
if (pipelineBindPoint == VK_PIPELINE_BIND_POINT_COMPUTE) {
cmd->state.compute_pipeline = pipeline;
tu_cs_emit_state_ib(&cmd->cs, pipeline->program.state);
for (uint32_t i = 0; i < commandBufferCount; i++) {
TU_FROM_HANDLE(tu_cmd_buffer, secondary, pCmdBuffers[i]);
- result = tu_bo_list_merge(&cmd->bo_list, &secondary->bo_list);
- if (result != VK_SUCCESS) {
- cmd->record_result = result;
- break;
- }
-
if (secondary->usage_flags &
VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) {
assert(tu_cs_is_empty(&secondary->cs));
assert(tu_cs_is_empty(&secondary->draw_cs));
assert(tu_cs_is_empty(&secondary->draw_epilogue_cs));
- for (uint32_t j = 0; j < secondary->cs.bo_count; j++) {
- tu_bo_list_add(&cmd->bo_list, secondary->cs.bos[j],
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
- }
-
tu_cs_add_entries(&cmd->cs, &secondary->cs);
}
tu_set_input_attachments(cmd, cmd->state.subpass);
- for (uint32_t i = 0; i < fb->attachment_count; ++i) {
- const struct tu_image_view *iview = fb->attachments[i].attachment;
- tu_bo_list_add(&cmd->bo_list, iview->image->bo,
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
- }
-
cmd->state.dirty |= TU_CMD_DIRTY_DRAW_STATE;
}
if (result != VK_SUCCESS)
return result;
- tu_bo_list_add(&cmd->bo_list, tess_bo,
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
uint64_t tess_factor_iova = tess_bo->iova;
uint64_t tess_param_iova = tess_factor_iova + tess_factor_size;
tu_cs_emit(cs, drawCount);
tu_cs_emit_qw(cs, buf->bo->iova + buf->bo_offset + offset);
tu_cs_emit(cs, stride);
-
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
}
void
tu_cs_emit(cs, cmd->state.max_index_count);
tu_cs_emit_qw(cs, buf->bo->iova + buf->bo_offset + offset);
tu_cs_emit(cs, stride);
-
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
}
void
tu_cs_emit_qw(cs, buf->bo->iova + buf->bo_offset + offset);
tu_cs_emit_qw(cs, count_buf->bo->iova + count_buf->bo_offset + countBufferOffset);
tu_cs_emit(cs, stride);
-
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
- tu_bo_list_add(&cmd->bo_list, count_buf->bo, MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
}
void
tu_cs_emit_qw(cs, buf->bo->iova + buf->bo_offset + offset);
tu_cs_emit_qw(cs, count_buf->bo->iova + count_buf->bo_offset + countBufferOffset);
tu_cs_emit(cs, stride);
-
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
- tu_bo_list_add(&cmd->bo_list, count_buf->bo, MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
}
void tu_CmdDrawIndirectByteCountEXT(VkCommandBuffer commandBuffer,
tu_cs_emit_qw(cs, buf->bo->iova + buf->bo_offset + counterBufferOffset);
tu_cs_emit(cs, counterOffset);
tu_cs_emit(cs, vertexStride);
-
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
}
struct tu_dispatch_info
if (info->indirect) {
uint64_t iova = tu_buffer_iova(info->indirect) + info->indirect_offset;
- tu_bo_list_add(&cmd->bo_list, info->indirect->bo,
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE);
-
tu_cs_emit_pkt7(cs, CP_EXEC_CS_INDIRECT, 4);
tu_cs_emit(cs, 0x00000000);
tu_cs_emit_qw(cs, iova);
for (uint32_t i = 0; i < info->eventCount; i++) {
TU_FROM_HANDLE(tu_event, event, info->pEvents[i]);
- tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_READ);
-
tu_cs_emit_pkt7(cs, CP_WAIT_REG_MEM, 6);
tu_cs_emit(cs, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ) |
CP_WAIT_REG_MEM_0_POLL_MEMORY);
tu_emit_cache_flush(cmd, cs);
- tu_bo_list_add(&cmd->bo_list, &event->bo, MSM_SUBMIT_BO_WRITE);
-
/* Flags that only require a top-of-pipe event. DrawIndirect parameters are
* read by the CP, so the draw indirect stage counts as top-of-pipe too.
*/
tu_cs_emit(cs, CP_DRAW_PRED_SET_0_SRC(PRED_SRC_MEM) |
CP_DRAW_PRED_SET_0_TEST(inv ? EQ_0_PASS : NE_0_PASS));
tu_cs_emit_qw(cs, global_iova(cmd, predicate));
-
- tu_bo_list_add(&cmd->bo_list, buf->bo, MSM_SUBMIT_BO_READ);
}
void
return VK_ERROR_OUT_OF_HOST_MEMORY;
VkResult result =
- tu_bo_init_new(cs->device, new_bo, size * sizeof(uint32_t));
+ tu_bo_init_new(cs->device, new_bo, size * sizeof(uint32_t), true);
if (result != VK_SUCCESS) {
free(new_bo);
return result;
set_layout->dynamic_ubo = 0;
uint32_t dynamic_offset_count = 0;
- uint32_t buffer_count = 0;
for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
const VkDescriptorSetLayoutBinding *binding = bindings + j;
set_layout->binding[b].type = binding->descriptorType;
set_layout->binding[b].array_size = binding->descriptorCount;
set_layout->binding[b].offset = set_layout->size;
- set_layout->binding[b].buffer_offset = buffer_count;
set_layout->binding[b].dynamic_offset_offset = dynamic_offset_count;
set_layout->binding[b].size = descriptor_size(binding->descriptorType);
set_layout->binding[b].shader_stages = binding->stageFlags;
set_layout->size +=
binding->descriptorCount * set_layout->binding[b].size;
- if (binding->descriptorType != VK_DESCRIPTOR_TYPE_SAMPLER &&
- binding->descriptorType != VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT)
- buffer_count += binding->descriptorCount;
if (binding->descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC ||
binding->descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC) {
if (binding->descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC) {
free(bindings);
set_layout->dynamic_offset_count = dynamic_offset_count;
- set_layout->buffer_count = buffer_count;
*pSetLayout = tu_descriptor_set_layout_to_handle(set_layout);
struct tu_descriptor_set **out_set)
{
struct tu_descriptor_set *set;
- uint32_t buffer_count = layout->buffer_count;
- if (variable_count) {
- unsigned stride = 1;
- if (layout->binding[layout->binding_count - 1].type == VK_DESCRIPTOR_TYPE_SAMPLER ||
- layout->binding[layout->binding_count - 1].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT)
- stride = 0;
- buffer_count = layout->binding[layout->binding_count - 1].buffer_offset +
- *variable_count * stride;
- }
- unsigned dynamic_offset = sizeof(struct tu_descriptor_set) +
- sizeof(struct tu_bo *) * buffer_count;
+ unsigned dynamic_offset = sizeof(struct tu_descriptor_set);
unsigned mem_size = dynamic_offset +
A6XX_TEX_CONST_DWORDS * 4 * layout->dynamic_offset_count;
if (bo_size) {
VkResult ret;
- ret = tu_bo_init_new(device, &pool->bo, bo_size);
+ ret = tu_bo_init_new(device, &pool->bo, bo_size, true);
assert(ret == VK_SUCCESS);
ret = tu_bo_map(device, &pool->bo);
static void write_texel_buffer_descriptor(struct tu_device *device,
struct tu_cmd_buffer *cmd_buffer,
unsigned *dst,
- struct tu_bo **buffer_list,
const VkBufferView buffer_view)
{
TU_FROM_HANDLE(tu_buffer_view, view, buffer_view);
memcpy(dst, view->descriptor, sizeof(view->descriptor));
-
- if (cmd_buffer)
- tu_bo_list_add(&cmd_buffer->bo_list, view->buffer->bo, MSM_SUBMIT_BO_READ);
- else
- *buffer_list = view->buffer->bo;
}
static uint32_t get_range(struct tu_buffer *buf, VkDeviceSize offset,
static void write_buffer_descriptor(struct tu_device *device,
struct tu_cmd_buffer *cmd_buffer,
unsigned *dst,
- struct tu_bo **buffer_list,
const VkDescriptorBufferInfo *buffer_info)
{
TU_FROM_HANDLE(tu_buffer, buffer, buffer_info->buffer);
dst[5] = A6XX_IBO_5_BASE_HI(va >> 32);
for (int i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
dst[i] = 0;
-
- if (cmd_buffer)
- tu_bo_list_add(&cmd_buffer->bo_list, buffer->bo, MSM_SUBMIT_BO_READ);
- else
- *buffer_list = buffer->bo;
}
static void write_ubo_descriptor(struct tu_device *device,
struct tu_cmd_buffer *cmd_buffer,
unsigned *dst,
- struct tu_bo **buffer_list,
const VkDescriptorBufferInfo *buffer_info)
{
TU_FROM_HANDLE(tu_buffer, buffer, buffer_info->buffer);
uint64_t va = tu_buffer_iova(buffer) + buffer_info->offset;
dst[0] = A6XX_UBO_0_BASE_LO(va);
dst[1] = A6XX_UBO_1_BASE_HI(va >> 32) | A6XX_UBO_1_SIZE(range);
-
- if (cmd_buffer)
- tu_bo_list_add(&cmd_buffer->bo_list, buffer->bo, MSM_SUBMIT_BO_READ);
- else
- *buffer_list = buffer->bo;
}
static void
write_image_descriptor(struct tu_device *device,
struct tu_cmd_buffer *cmd_buffer,
unsigned *dst,
- struct tu_bo **buffer_list,
VkDescriptorType descriptor_type,
const VkDescriptorImageInfo *image_info)
{
} else {
memcpy(dst, iview->descriptor, sizeof(iview->descriptor));
}
-
- if (cmd_buffer)
- tu_bo_list_add(&cmd_buffer->bo_list, iview->image->bo, MSM_SUBMIT_BO_READ);
- else
- *buffer_list = iview->image->bo;
}
static void
struct tu_cmd_buffer *cmd_buffer,
unsigned sampler_offset,
unsigned *dst,
- struct tu_bo **buffer_list,
VkDescriptorType descriptor_type,
const VkDescriptorImageInfo *image_info,
bool has_sampler)
{
TU_FROM_HANDLE(tu_sampler, sampler, image_info->sampler);
- write_image_descriptor(device, cmd_buffer, dst, buffer_list,
+ write_image_descriptor(device, cmd_buffer, dst,
descriptor_type, image_info);
/* copy over sampler state */
if (has_sampler) {
const struct tu_descriptor_set_binding_layout *binding_layout =
set->layout->binding + writeset->dstBinding;
uint32_t *ptr = set->mapped_ptr;
- struct tu_bo **buffer_list = set->buffers;
ptr += binding_layout->offset / 4;
ptr += (binding_layout->size / 4) * writeset->dstArrayElement;
- buffer_list += binding_layout->buffer_offset;
- buffer_list += writeset->dstArrayElement;
for (j = 0; j < writeset->descriptorCount; ++j) {
switch(writeset->descriptorType) {
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: {
idx += binding_layout->dynamic_offset_offset;
write_ubo_descriptor(device, cmd_buffer,
set->dynamic_descriptors + A6XX_TEX_CONST_DWORDS * idx,
- buffer_list, writeset->pBufferInfo + j);
+ writeset->pBufferInfo + j);
break;
}
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
- write_ubo_descriptor(device, cmd_buffer, ptr, buffer_list,
- writeset->pBufferInfo + j);
+ write_ubo_descriptor(device, cmd_buffer, ptr, writeset->pBufferInfo + j);
break;
case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
idx += binding_layout->dynamic_offset_offset;
write_buffer_descriptor(device, cmd_buffer,
set->dynamic_descriptors + A6XX_TEX_CONST_DWORDS * idx,
- buffer_list, writeset->pBufferInfo + j);
+ writeset->pBufferInfo + j);
break;
}
case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
- write_buffer_descriptor(device, cmd_buffer, ptr, buffer_list,
- writeset->pBufferInfo + j);
+ write_buffer_descriptor(device, cmd_buffer, ptr, writeset->pBufferInfo + j);
break;
case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
- write_texel_buffer_descriptor(device, cmd_buffer, ptr, buffer_list,
- writeset->pTexelBufferView[j]);
+ write_texel_buffer_descriptor(device, cmd_buffer, ptr, writeset->pTexelBufferView[j]);
break;
case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
- write_image_descriptor(device, cmd_buffer, ptr, buffer_list,
+ write_image_descriptor(device, cmd_buffer, ptr,
writeset->descriptorType,
writeset->pImageInfo + j);
break;
case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
write_combined_image_sampler_descriptor(device, cmd_buffer,
A6XX_TEX_CONST_DWORDS * 4,
- ptr, buffer_list,
+ ptr,
writeset->descriptorType,
writeset->pImageInfo + j,
!binding_layout->immutable_samplers_offset);
break;
}
ptr += binding_layout->size / 4;
- ++buffer_list;
}
}
dst_set->layout->binding + copyset->dstBinding;
uint32_t *src_ptr = src_set->mapped_ptr;
uint32_t *dst_ptr = dst_set->mapped_ptr;
- struct tu_bo **src_buffer_list = src_set->buffers;
- struct tu_bo **dst_buffer_list = dst_set->buffers;
src_ptr += src_binding_layout->offset / 4;
dst_ptr += dst_binding_layout->offset / 4;
src_ptr += src_binding_layout->size * copyset->srcArrayElement / 4;
dst_ptr += dst_binding_layout->size * copyset->dstArrayElement / 4;
- src_buffer_list += src_binding_layout->buffer_offset;
- src_buffer_list += copyset->srcArrayElement;
-
- dst_buffer_list += dst_binding_layout->buffer_offset;
- dst_buffer_list += copyset->dstArrayElement;
-
for (j = 0; j < copyset->descriptorCount; ++j) {
switch (src_binding_layout->type) {
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
src_ptr += src_binding_layout->size / 4;
dst_ptr += dst_binding_layout->size / 4;
-
- if (src_binding_layout->type != VK_DESCRIPTOR_TYPE_SAMPLER) {
- /* Sampler descriptors don't have a buffer list. */
- dst_buffer_list[j] = src_buffer_list[j];
- }
}
}
}
const struct tu_descriptor_set_binding_layout *binding_layout =
set_layout->binding + entry->dstBinding;
- const uint32_t buffer_offset = binding_layout->buffer_offset +
- entry->dstArrayElement;
uint32_t dst_offset, dst_stride;
/* dst_offset is an offset into dynamic_descriptors when the descriptor
.src_stride = entry->stride,
.dst_offset = dst_offset,
.dst_stride = dst_stride,
- .buffer_offset = buffer_offset,
.has_sampler = !binding_layout->immutable_samplers_offset,
};
}
for (uint32_t i = 0; i < templ->entry_count; i++) {
uint32_t *ptr = set->mapped_ptr;
const void *src = ((const char *) pData) + templ->entry[i].src_offset;
- struct tu_bo **buffer_list = set->buffers;
ptr += templ->entry[i].dst_offset;
- buffer_list += templ->entry[i].buffer_offset;
unsigned dst_offset = templ->entry[i].dst_offset;
for (unsigned j = 0; j < templ->entry[i].descriptor_count; ++j) {
switch(templ->entry[i].descriptor_type) {
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: {
assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
write_ubo_descriptor(device, cmd_buffer,
- set->dynamic_descriptors + dst_offset,
- buffer_list, src);
+ set->dynamic_descriptors + dst_offset, src);
break;
}
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
- write_ubo_descriptor(device, cmd_buffer, ptr, buffer_list, src);
+ write_ubo_descriptor(device, cmd_buffer, ptr, src);
break;
case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
write_buffer_descriptor(device, cmd_buffer,
- set->dynamic_descriptors + dst_offset,
- buffer_list, src);
+ set->dynamic_descriptors + dst_offset, src);
break;
}
case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
- write_buffer_descriptor(device, cmd_buffer, ptr, buffer_list, src);
+ write_buffer_descriptor(device, cmd_buffer, ptr, src);
break;
case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
- write_texel_buffer_descriptor(device, cmd_buffer, ptr,
- buffer_list, *(VkBufferView *) src);
+ write_texel_buffer_descriptor(device, cmd_buffer, ptr, *(VkBufferView *) src);
break;
case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: {
- write_image_descriptor(device, cmd_buffer, ptr, buffer_list,
+ write_image_descriptor(device, cmd_buffer, ptr,
templ->entry[i].descriptor_type,
src);
break;
case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
write_combined_image_sampler_descriptor(device, cmd_buffer,
A6XX_TEX_CONST_DWORDS * 4,
- ptr, buffer_list,
+ ptr,
templ->entry[i].descriptor_type,
src,
templ->entry[i].has_sampler);
src = (char *) src + templ->entry[i].src_stride;
ptr += templ->entry[i].dst_stride;
dst_offset += templ->entry[i].dst_stride;
- ++buffer_list;
}
}
}
uint32_t offset;
- /* For descriptors that point to a buffer, index into the array of BO's to
- * be added to the cmdbuffer's used BO list.
- */
- uint32_t buffer_offset;
-
/* Index into the pDynamicOffsets array for dynamic descriptors, as well as
* the array of dynamic descriptors (offsetted by
* tu_pipeline_layout::set::dynamic_offset_start).
*/
uint32_t dynamic_ubo;
- uint32_t buffer_count;
-
bool has_immutable_samplers;
bool has_variable_descriptors;
device->physical_device = physical_device;
device->_lost = false;
+ mtx_init(&device->bo_mutex, mtx_plain);
+
for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
const char *ext_name = pCreateInfo->ppEnabledExtensionNames[i];
int index = tu_get_device_extension_index(ext_name);
if (custom_border_colors)
global_size += TU_BORDER_COLOR_COUNT * sizeof(struct bcolor_entry);
- result = tu_bo_init_new(device, &device->global_bo, global_size);
+ result = tu_bo_init_new(device, &device->global_bo, global_size, false);
if (result != VK_SUCCESS)
goto fail_global_bo;
VkPipelineCache pc = tu_pipeline_cache_to_handle(device->mem_cache);
tu_DestroyPipelineCache(tu_device_to_handle(device), pc, NULL);
+ vk_free(&device->vk.alloc, device->bo_list);
+ vk_free(&device->vk.alloc, device->bo_idx);
vk_free(&device->vk.alloc, device);
}
}
unsigned bo_size = 1ull << size_log2;
- VkResult result = tu_bo_init_new(dev, &dev->scratch_bos[index].bo, bo_size);
+ VkResult result = tu_bo_init_new(dev, &dev->scratch_bos[index].bo, bo_size, false);
if (result != VK_SUCCESS) {
mtx_unlock(&dev->scratch_bos[index].construct_mtx);
return result;
}
} else {
result =
- tu_bo_init_new(device, &mem->bo, pAllocateInfo->allocationSize);
+ tu_bo_init_new(device, &mem->bo, pAllocateInfo->allocationSize, false);
}
if (result != VK_SUCCESS) {
if (!event)
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
- VkResult result = tu_bo_init_new(device, &event->bo, 0x1000);
+ VkResult result = tu_bo_init_new(device, &event->bo, 0x1000, false);
if (result != VK_SUCCESS)
goto fail_alloc;
tu_bo_init(struct tu_device *dev,
struct tu_bo *bo,
uint32_t gem_handle,
- uint64_t size)
+ uint64_t size,
+ bool dump)
{
uint64_t iova = tu_gem_info(dev, gem_handle, MSM_INFO_GET_IOVA);
if (!iova) {
.iova = iova,
};
+ mtx_lock(&dev->bo_mutex);
+ uint32_t idx = dev->bo_count++;
+
+ /* grow the bo list if needed */
+ if (idx >= dev->bo_list_size) {
+ uint32_t new_len = idx + 64;
+ struct drm_msm_gem_submit_bo *new_ptr =
+ vk_realloc(&dev->vk.alloc, dev->bo_list, new_len * sizeof(*dev->bo_list),
+ 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
+ if (!new_ptr) {
+ tu_gem_close(dev, gem_handle);
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+ }
+
+ dev->bo_list = new_ptr;
+ dev->bo_list_size = new_len;
+ }
+
+ /* grow the "bo idx" list (maps gem handles to index in the bo list) */
+ if (bo->gem_handle >= dev->bo_idx_size) {
+ uint32_t new_len = bo->gem_handle + 256;
+ uint32_t *new_ptr =
+ vk_realloc(&dev->vk.alloc, dev->bo_idx, new_len * sizeof(*dev->bo_idx),
+ 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
+ if (!new_ptr) {
+ tu_gem_close(dev, gem_handle);
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+ }
+
+ dev->bo_idx = new_ptr;
+ dev->bo_idx_size = new_len;
+ }
+
+ dev->bo_idx[bo->gem_handle] = idx;
+ dev->bo_list[idx] = (struct drm_msm_gem_submit_bo) {
+ .flags = MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE |
+ COND(dump, MSM_SUBMIT_BO_DUMP),
+ .handle = gem_handle,
+ .presumed = iova,
+ };
+ mtx_unlock(&dev->bo_mutex);
+
return VK_SUCCESS;
}
VkResult
-tu_bo_init_new(struct tu_device *dev, struct tu_bo *bo, uint64_t size)
+tu_bo_init_new(struct tu_device *dev, struct tu_bo *bo, uint64_t size, bool dump)
{
/* TODO: Choose better flags. As of 2018-11-12, freedreno/drm/msm_bo.c
* always sets `flags = MSM_BO_WC`, and we copy that behavior here.
if (ret)
return vk_error(dev->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
- return tu_bo_init(dev, bo, req.handle, size);
+ return tu_bo_init(dev, bo, req.handle, size, dump);
}
VkResult
if (ret)
return vk_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
- return tu_bo_init(dev, bo, gem_handle, size);
+ return tu_bo_init(dev, bo, gem_handle, size, false);
}
int
if (bo->map)
munmap(bo->map, bo->size);
+ mtx_lock(&dev->bo_mutex);
+ uint32_t idx = dev->bo_idx[bo->gem_handle];
+ dev->bo_count--;
+ dev->bo_list[idx] = dev->bo_list[dev->bo_count];
+ dev->bo_idx[dev->bo_list[idx].handle] = idx;
+ mtx_unlock(&dev->bo_mutex);
+
tu_gem_close(dev, bo->gem_handle);
}
const bool last_submit = (i == submitCount - 1);
struct drm_msm_gem_submit_syncobj *in_syncobjs = NULL, *out_syncobjs = NULL;
uint32_t nr_in_syncobjs, nr_out_syncobjs;
- struct tu_bo_list bo_list;
- tu_bo_list_init(&bo_list);
result = tu_get_semaphore_syncobjs(pSubmits[i].pWaitSemaphores,
pSubmits[i].waitSemaphoreCount,
entry_count += cmdbuf->cs.entry_count;
}
+ mtx_lock(&queue->device->bo_mutex);
+
struct drm_msm_gem_submit_cmd cmds[entry_count];
uint32_t entry_idx = 0;
for (uint32_t j = 0; j < submit->commandBufferCount; ++j) {
for (unsigned i = 0; i < cs->entry_count; ++i, ++entry_idx) {
cmds[entry_idx].type = MSM_SUBMIT_CMD_BUF;
cmds[entry_idx].submit_idx =
- tu_bo_list_add(&bo_list, cs->entries[i].bo,
- MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_DUMP);
+ queue->device->bo_idx[cs->entries[i].bo->gem_handle];
cmds[entry_idx].submit_offset = cs->entries[i].offset;
cmds[entry_idx].size = cs->entries[i].size;
cmds[entry_idx].pad = 0;
cmds[entry_idx].nr_relocs = 0;
cmds[entry_idx].relocs = 0;
}
-
- tu_bo_list_merge(&bo_list, &cmdbuf->bo_list);
}
uint32_t flags = MSM_PIPE_3D0;
struct drm_msm_gem_submit req = {
.flags = flags,
.queueid = queue->msm_queue_id,
- .bos = (uint64_t)(uintptr_t) bo_list.bo_infos,
- .nr_bos = bo_list.count,
+ .bos = (uint64_t)(uintptr_t) queue->device->bo_list,
+ .nr_bos = queue->device->bo_count,
.cmds = (uint64_t)(uintptr_t)cmds,
.nr_cmds = entry_count,
.in_syncobjs = (uint64_t)(uintptr_t)in_syncobjs,
strerror(errno));
}
- tu_bo_list_destroy(&bo_list);
+ mtx_unlock(&queue->device->bo_mutex);
free(in_syncobjs);
free(out_syncobjs);
uint32_t vsc_prim_strm_pitch;
BITSET_DECLARE(custom_border_color, TU_BORDER_COLOR_COUNT);
mtx_t mutex;
+
+ /* bo list for submits: */
+ struct drm_msm_gem_submit_bo *bo_list;
+ /* map bo handles to bo list index: */
+ uint32_t *bo_idx;
+ uint32_t bo_count, bo_list_size, bo_idx_size;
+ mtx_t bo_mutex;
};
VkResult _tu_device_set_lost(struct tu_device *device,
}
VkResult
-tu_bo_init_new(struct tu_device *dev, struct tu_bo *bo, uint64_t size);
+tu_bo_init_new(struct tu_device *dev, struct tu_bo *bo, uint64_t size, bool dump);
VkResult
tu_bo_init_dmabuf(struct tu_device *dev,
struct tu_bo *bo,
uint32_t *mapped_ptr;
uint32_t *dynamic_descriptors;
-
- struct tu_bo *buffers[0];
};
struct tu_push_descriptor_set
TU_CMD_BUFFER_STATUS_PENDING,
};
-#ifndef MSM_SUBMIT_BO_READ
-#define MSM_SUBMIT_BO_READ 0x0001
-#define MSM_SUBMIT_BO_WRITE 0x0002
-#define MSM_SUBMIT_BO_DUMP 0x0004
-
-struct drm_msm_gem_submit_bo {
- uint32_t flags; /* in, mask of MSM_SUBMIT_BO_x */
- uint32_t handle; /* in, GEM handle */
- uint64_t presumed; /* in/out, presumed buffer address */
-};
-#endif
-
-struct tu_bo_list
-{
- uint32_t count;
- uint32_t capacity;
- struct drm_msm_gem_submit_bo *bo_infos;
-};
-
-#define TU_BO_LIST_FAILED (~0)
-
-void
-tu_bo_list_init(struct tu_bo_list *list);
-void
-tu_bo_list_destroy(struct tu_bo_list *list);
-void
-tu_bo_list_reset(struct tu_bo_list *list);
-uint32_t
-tu_bo_list_add(struct tu_bo_list *list,
- const struct tu_bo *bo,
- uint32_t flags);
-VkResult
-tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other);
-
struct tu_cmd_buffer
{
struct vk_object_base base;
VkResult record_result;
- struct tu_bo_list bo_list;
struct tu_cs cs;
struct tu_cs draw_cs;
struct tu_cs draw_epilogue_cs;
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
VkResult result = tu_bo_init_new(device, &pool->bo,
- pCreateInfo->queryCount * slot_size);
+ pCreateInfo->queryCount * slot_size, false);
if (result != VK_SUCCESS) {
vk_object_free(&device->vk, pAllocator, pool);
return result;
result_count /* offset */, flags);
}
}
-
- tu_bo_list_add(&cmdbuf->bo_list, buffer->bo, MSM_SUBMIT_BO_WRITE);
}
void
default:
assert(!"Invalid query type");
}
-
- tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
}
void
default:
assert(!"Invalid query type");
}
-
- tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
}
void
default:
assert(!"Invalid query type");
}
-
- tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
}
static void
}
handle_multiview_queries(cmdbuf, pool, query);
-
- tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
}
void
default:
assert(!"Invalid query type");
}
-
- tu_bo_list_add(&cmdbuf->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
}
void
TU_FROM_HANDLE(tu_cmd_buffer, cmd, commandBuffer);
TU_FROM_HANDLE(tu_query_pool, pool, queryPool);
- tu_bo_list_add(&cmd->bo_list, &pool->bo, MSM_SUBMIT_BO_WRITE);
-
/* Inside a render pass, just write the timestamp multiple times so that
* the user gets the last one if we use GMEM. There isn't really much
* better we can do, and this seems to be what the blob does too.