{
struct tu_device *device;
void *mem_ctx;
- struct tu_pipeline_cache *cache;
+ struct vk_pipeline_cache *cache;
struct tu_pipeline_layout *layout;
const VkAllocationCallbacks *alloc;
const VkGraphicsPipelineCreateInfo *create_info;
- struct tu_shader *shaders[MESA_SHADER_FRAGMENT + 1];
- struct ir3_shader_variant *variants[MESA_SHADER_FRAGMENT + 1];
+ struct tu_compiled_shaders *shaders;
struct ir3_shader_variant *binning_variant;
uint64_t shader_iova[MESA_SHADER_FRAGMENT + 1];
uint64_t binning_vs_iova;
}
static void
-tu6_emit_cs_config(struct tu_cs *cs, const struct tu_shader *shader,
+tu6_emit_cs_config(struct tu_cs *cs,
const struct ir3_shader_variant *v,
const struct tu_pvtmem_config *pvtmem,
uint64_t binary_iova)
.gs_state = true,
.fs_state = true,
.gfx_ibo = true));
- for (; stage < ARRAY_SIZE(builder->shaders); stage++) {
- tu6_emit_xs_config(cs, stage, builder->variants[stage]);
+ for (; stage < ARRAY_SIZE(builder->shader_iova); stage++) {
+ tu6_emit_xs_config(cs, stage, builder->shaders->variants[stage]);
}
}
bool binning_pass,
struct tu_pipeline *pipeline)
{
- const struct ir3_shader_variant *vs = builder->variants[MESA_SHADER_VERTEX];
+ const struct ir3_shader_variant *vs = builder->shaders->variants[MESA_SHADER_VERTEX];
const struct ir3_shader_variant *bs = builder->binning_variant;
- const struct ir3_shader_variant *hs = builder->variants[MESA_SHADER_TESS_CTRL];
- const struct ir3_shader_variant *ds = builder->variants[MESA_SHADER_TESS_EVAL];
- const struct ir3_shader_variant *gs = builder->variants[MESA_SHADER_GEOMETRY];
- const struct ir3_shader_variant *fs = builder->variants[MESA_SHADER_FRAGMENT];
+ const struct ir3_shader_variant *hs = builder->shaders->variants[MESA_SHADER_TESS_CTRL];
+ const struct ir3_shader_variant *ds = builder->shaders->variants[MESA_SHADER_TESS_EVAL];
+ const struct ir3_shader_variant *gs = builder->shaders->variants[MESA_SHADER_GEOMETRY];
+ const struct ir3_shader_variant *fs = builder->shaders->variants[MESA_SHADER_FRAGMENT];
gl_shader_stage stage = MESA_SHADER_VERTEX;
uint32_t cps_per_patch = builder->create_info->pTessellationState ?
builder->create_info->pTessellationState->patchControlPoints : 0;
- bool multi_pos_output = builder->shaders[MESA_SHADER_VERTEX]->multi_pos_output;
+ bool multi_pos_output = builder->shaders->multi_pos_output;
/* Don't use the binning pass variant when GS is present because we don't
* support compiling correct binning pass variants with GS.
stage++;
}
- for (; stage < ARRAY_SIZE(builder->shaders); stage++) {
- const struct ir3_shader_variant *xs = builder->variants[stage];
+ for (; stage < ARRAY_SIZE(builder->shaders->variants); stage++) {
+ const struct ir3_shader_variant *xs = builder->shaders->variants[stage];
if (stage == MESA_SHADER_FRAGMENT && binning_pass)
fs = xs = NULL;
struct tu_pipeline *pipeline,
struct tu_pipeline_layout *layout,
struct tu_pipeline_builder *builder,
- struct tu_pipeline_cache *cache,
struct ir3_shader_variant *compute)
{
uint32_t size = 1024 + tu6_load_state_size(pipeline, layout, compute);
/* graphics case: */
if (builder) {
- for (uint32_t i = 0; i < ARRAY_SIZE(builder->variants); i++) {
- if (builder->variants[i]) {
- size += builder->variants[i]->info.size / 4;
+ for (uint32_t i = 0; i < ARRAY_SIZE(builder->shaders->variants); i++) {
+ if (builder->shaders->variants[i]) {
+ size += builder->shaders->variants[i]->info.size / 4;
}
}
size += builder->binning_variant->info.size / 4;
builder->additional_cs_reserve_size = 0;
- for (unsigned i = 0; i < ARRAY_SIZE(builder->variants); i++) {
- struct ir3_shader_variant *variant = builder->variants[i];
+ for (unsigned i = 0; i < ARRAY_SIZE(builder->shaders->variants); i++) {
+ struct ir3_shader_variant *variant = builder->shaders->variants[i];
if (variant) {
builder->additional_cs_reserve_size +=
tu_xs_get_additional_cs_size_dwords(variant);
}
}
+static void
+tu_shader_key_init(struct tu_shader_key *key,
+ const VkPipelineShaderStageCreateInfo *stage_info,
+ struct tu_device *dev)
+{
+ enum ir3_wavesize_option api_wavesize, real_wavesize;
+
+ if (stage_info) {
+ if (stage_info->flags &
+ VK_PIPELINE_SHADER_STAGE_CREATE_ALLOW_VARYING_SUBGROUP_SIZE_BIT_EXT) {
+ api_wavesize = real_wavesize = IR3_SINGLE_OR_DOUBLE;
+ } else {
+ const VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT *size_info =
+ vk_find_struct_const(stage_info->pNext,
+ PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT);
+
+ if (size_info) {
+ if (size_info->requiredSubgroupSize == dev->compiler->threadsize_base) {
+ api_wavesize = IR3_SINGLE_ONLY;
+ } else {
+ assert(size_info->requiredSubgroupSize == dev->compiler->threadsize_base * 2);
+ api_wavesize = IR3_DOUBLE_ONLY;
+ }
+ } else {
+ /* Match the exposed subgroupSize. */
+ api_wavesize = IR3_DOUBLE_ONLY;
+ }
+
+ if (stage_info->flags &
+ VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT_EXT)
+ real_wavesize = api_wavesize;
+ else if (api_wavesize == IR3_SINGLE_ONLY)
+ real_wavesize = IR3_SINGLE_ONLY;
+ else
+ real_wavesize = IR3_SINGLE_OR_DOUBLE;
+ }
+ } else {
+ api_wavesize = real_wavesize = IR3_SINGLE_OR_DOUBLE;
+ }
+
+ key->api_wavesize = api_wavesize;
+ key->real_wavesize = real_wavesize;
+}
+
+static void
+tu_hash_stage(struct mesa_sha1 *ctx,
+ const VkPipelineShaderStageCreateInfo *stage,
+ const struct tu_shader_key *key)
+{
+ VK_FROM_HANDLE(vk_shader_module, module, stage->module);
+ const VkSpecializationInfo *spec_info = stage->pSpecializationInfo;
+
+ _mesa_sha1_update(ctx, module->sha1, sizeof(module->sha1));
+ _mesa_sha1_update(ctx, stage->pName, strlen(stage->pName));
+ if (spec_info && spec_info->mapEntryCount) {
+ _mesa_sha1_update(ctx, spec_info->pMapEntries,
+ spec_info->mapEntryCount * sizeof spec_info->pMapEntries[0]);
+ _mesa_sha1_update(ctx, spec_info->pData, spec_info->dataSize);
+ }
+
+ _mesa_sha1_update(ctx, key, sizeof(*key));
+}
+
+/* Hash flags which can affect ir3 shader compilation which aren't known until
+ * logical device creation.
+ */
+static void
+tu_hash_compiler(struct mesa_sha1 *ctx, const struct ir3_compiler *compiler)
+{
+ _mesa_sha1_update(ctx, &compiler->robust_ubo_access,
+ sizeof(compiler->robust_ubo_access));
+ _mesa_sha1_update(ctx, &ir3_shader_debug, sizeof(ir3_shader_debug));
+}
+
+static void
+tu_hash_shaders(unsigned char *hash,
+ const VkPipelineShaderStageCreateInfo **stages,
+ const struct tu_pipeline_layout *layout,
+ const struct tu_shader_key *keys,
+ const struct ir3_shader_key *ir3_key,
+ const struct ir3_compiler *compiler)
+{
+ struct mesa_sha1 ctx;
+
+ _mesa_sha1_init(&ctx);
+
+ if (layout)
+ _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1));
+
+ _mesa_sha1_update(&ctx, ir3_key, sizeof(ir3_key));
+
+ for (int i = 0; i < MESA_SHADER_STAGES; ++i) {
+ if (stages[i]) {
+ tu_hash_stage(&ctx, stages[i], &keys[i]);
+ }
+ }
+ tu_hash_compiler(&ctx, compiler);
+ _mesa_sha1_final(&ctx, hash);
+}
+
+static void
+tu_hash_compute(unsigned char *hash,
+ const VkPipelineShaderStageCreateInfo *stage,
+ const struct tu_pipeline_layout *layout,
+ const struct tu_shader_key *key,
+ const struct ir3_compiler *compiler)
+{
+ struct mesa_sha1 ctx;
+
+ _mesa_sha1_init(&ctx);
+
+ if (layout)
+ _mesa_sha1_update(&ctx, layout->sha1, sizeof(layout->sha1));
+
+ tu_hash_stage(&ctx, stage, key);
+
+ tu_hash_compiler(&ctx, compiler);
+ _mesa_sha1_final(&ctx, hash);
+}
+
+static bool
+tu_shaders_serialize(struct vk_pipeline_cache_object *object,
+ struct blob *blob);
+
+static struct vk_pipeline_cache_object *
+tu_shaders_deserialize(struct vk_device *device,
+ const void *key_data, size_t key_size,
+ struct blob_reader *blob);
+
+static void
+tu_shaders_destroy(struct vk_pipeline_cache_object *object)
+{
+ struct tu_compiled_shaders *shaders =
+ container_of(object, struct tu_compiled_shaders, base);
+
+ for (unsigned i = 0; i < ARRAY_SIZE(shaders->variants); i++)
+ ralloc_free(shaders->variants[i]);
+
+ vk_pipeline_cache_object_finish(&shaders->base);
+ vk_free(&object->device->alloc, shaders);
+}
+
+const struct vk_pipeline_cache_object_ops tu_shaders_ops = {
+ .serialize = tu_shaders_serialize,
+ .deserialize = tu_shaders_deserialize,
+ .destroy = tu_shaders_destroy,
+};
+
+static struct tu_compiled_shaders *
+tu_shaders_init(struct tu_device *dev, const void *key_data, size_t key_size)
+{
+ VK_MULTIALLOC(ma);
+ VK_MULTIALLOC_DECL(&ma, struct tu_compiled_shaders, shaders, 1);
+ VK_MULTIALLOC_DECL_SIZE(&ma, void, obj_key_data, key_size);
+
+ if (!vk_multialloc_zalloc(&ma, &dev->vk.alloc,
+ VK_SYSTEM_ALLOCATION_SCOPE_DEVICE))
+ return NULL;
+
+ memcpy(obj_key_data, key_data, key_size);
+ vk_pipeline_cache_object_init(&dev->vk, &shaders->base,
+ &tu_shaders_ops, obj_key_data, key_size);
+
+ return shaders;
+}
+
+static bool
+tu_shaders_serialize(struct vk_pipeline_cache_object *object,
+ struct blob *blob)
+{
+ struct tu_compiled_shaders *shaders =
+ container_of(object, struct tu_compiled_shaders, base);
+
+ blob_write_bytes(blob, shaders->push_consts, sizeof(shaders->push_consts));
+ blob_write_uint8(blob, shaders->active_desc_sets);
+ blob_write_uint8(blob, shaders->multi_pos_output);
+
+ for (unsigned i = 0; i < ARRAY_SIZE(shaders->variants); i++) {
+ if (shaders->variants[i]) {
+ blob_write_uint8(blob, 1);
+ ir3_store_variant(blob, shaders->variants[i]);
+ } else {
+ blob_write_uint8(blob, 0);
+ }
+ }
+
+ return true;
+}
+
+static struct vk_pipeline_cache_object *
+tu_shaders_deserialize(struct vk_device *_device,
+ const void *key_data, size_t key_size,
+ struct blob_reader *blob)
+{
+ struct tu_device *dev = container_of(_device, struct tu_device, vk);
+ struct tu_compiled_shaders *shaders =
+ tu_shaders_init(dev, key_data, key_size);
+
+ if (!shaders)
+ return NULL;
+
+ blob_copy_bytes(blob, shaders->push_consts, sizeof(shaders->push_consts));
+ shaders->active_desc_sets = blob_read_uint8(blob);
+ shaders->multi_pos_output = blob_read_uint8(blob);
+
+ for (unsigned i = 0; i < ARRAY_SIZE(shaders->variants); i++) {
+ bool has_shader = blob_read_uint8(blob);
+ if (has_shader) {
+ shaders->variants[i] = ir3_retrieve_variant(blob, dev->compiler, NULL);
+ }
+ }
+
+ return &shaders->base;
+}
+
+static struct tu_compiled_shaders *
+tu_pipeline_cache_lookup(struct vk_pipeline_cache *cache,
+ const void *key_data, size_t key_size)
+{
+ struct vk_pipeline_cache_object *object =
+ vk_pipeline_cache_lookup_object(cache, key_data, key_size,
+ &tu_shaders_ops, NULL);
+ if (object)
+ return container_of(object, struct tu_compiled_shaders, base);
+ else
+ return NULL;
+}
+
+static struct tu_compiled_shaders *
+tu_pipeline_cache_insert(struct vk_pipeline_cache *cache,
+ struct tu_compiled_shaders *shaders)
+{
+ struct vk_pipeline_cache_object *object =
+ vk_pipeline_cache_add_object(cache, &shaders->base);
+ return container_of(object, struct tu_compiled_shaders, base);
+}
+
static VkResult
tu_pipeline_builder_compile_shaders(struct tu_pipeline_builder *builder,
struct tu_pipeline *pipeline)
{
+ VkResult result = VK_SUCCESS;
const struct ir3_compiler *compiler = builder->device->compiler;
const VkPipelineShaderStageCreateInfo *stage_infos[MESA_SHADER_STAGES] = {
NULL
stage_infos[stage] = &builder->create_info->pStages[i];
}
- struct ir3_shader_key key = {};
- tu_pipeline_shader_key_init(&key, pipeline, builder->create_info);
+ struct tu_shader_key keys[ARRAY_SIZE(stage_infos)] = { };
+ for (gl_shader_stage stage = MESA_SHADER_VERTEX;
+ stage < ARRAY_SIZE(keys); stage++) {
+ tu_shader_key_init(&keys[stage], stage_infos[stage], builder->device);
+ }
+
+ struct ir3_shader_key ir3_key = {};
+ tu_pipeline_shader_key_init(&ir3_key, pipeline, builder->create_info);
+
+ keys[MESA_SHADER_VERTEX].multiview_mask = builder->multiview_mask;
+ keys[MESA_SHADER_FRAGMENT].multiview_mask = builder->multiview_mask;
+
+ unsigned char pipeline_sha1[20];
+ tu_hash_shaders(pipeline_sha1, stage_infos, builder->layout, keys, &ir3_key, compiler);
+
+ const bool executable_info = builder->create_info->flags &
+ VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR;
+
+ char *nir_initial_disasm[ARRAY_SIZE(stage_infos)] = { NULL };
+
+ struct tu_compiled_shaders *compiled_shaders;
+
+ if (!executable_info) {
+ compiled_shaders =
+ tu_pipeline_cache_lookup(builder->cache, &pipeline_sha1,
+ sizeof(pipeline_sha1));
- nir_shader *nir[ARRAY_SIZE(builder->shaders)] = { NULL };
+ if (compiled_shaders)
+ goto done;
+ }
+
+ nir_shader *nir[ARRAY_SIZE(stage_infos)] = { NULL };
+
+ struct tu_shader *shaders[ARRAY_SIZE(nir)] = { NULL };
for (gl_shader_stage stage = MESA_SHADER_VERTEX;
stage < ARRAY_SIZE(nir); stage++) {
continue;
nir[stage] = tu_spirv_to_nir(builder->device, builder->mem_ctx, stage_info, stage);
- if (!nir[stage])
- return VK_ERROR_OUT_OF_HOST_MEMORY;
+ if (!nir[stage]) {
+ result = VK_ERROR_OUT_OF_HOST_MEMORY;
+ goto fail;
+ }
}
if (!nir[MESA_SHADER_FRAGMENT]) {
nir[MESA_SHADER_FRAGMENT] = fs_b.shader;
}
- const bool executable_info = builder->create_info->flags &
- VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR;
-
- char *nir_initial_disasm[ARRAY_SIZE(builder->shaders)] = { NULL };
-
if (executable_info) {
for (gl_shader_stage stage = MESA_SHADER_VERTEX;
stage < ARRAY_SIZE(nir); stage++) {
continue;
struct tu_shader *shader =
- tu_shader_create(builder->device, nir[stage], stage_infos[stage],
- builder->multiview_mask, builder->layout,
- builder->alloc);
- if (!shader)
- return VK_ERROR_OUT_OF_HOST_MEMORY;
+ tu_shader_create(builder->device, nir[stage], &keys[stage],
+ builder->layout, builder->alloc);
+ if (!shader) {
+ result = VK_ERROR_OUT_OF_HOST_MEMORY;
+ goto fail;
+ }
/* In SPIR-V generated from GLSL, the primitive mode is specified in the
* tessellation evaluation shader, but in SPIR-V generated from HLSL,
* the mode is specified in the tessellation control shader. */
if ((stage == MESA_SHADER_TESS_EVAL || stage == MESA_SHADER_TESS_CTRL) &&
- key.tessellation == IR3_TESS_NONE) {
- key.tessellation = tu6_get_tessmode(shader);
+ ir3_key.tessellation == IR3_TESS_NONE) {
+ ir3_key.tessellation = tu6_get_tessmode(shader);
}
if (stage > MESA_SHADER_TESS_CTRL) {
if (stage == MESA_SHADER_FRAGMENT) {
- key.tcs_store_primid = key.tcs_store_primid ||
+ ir3_key.tcs_store_primid = ir3_key.tcs_store_primid ||
(nir[stage]->info.inputs_read & (1ull << VARYING_SLOT_PRIMITIVE_ID));
} else {
- key.tcs_store_primid = key.tcs_store_primid ||
+ ir3_key.tcs_store_primid = ir3_key.tcs_store_primid ||
BITSET_TEST(nir[stage]->info.system_values_read, SYSTEM_VALUE_PRIMITIVE_ID);
}
}
* which is set in tu_lower_io. */
desc_sets |= shader->active_desc_sets;
- builder->shaders[stage] = shader;
+ shaders[stage] = shader;
}
- pipeline->active_desc_sets = desc_sets;
- struct tu_shader *last_shader = builder->shaders[MESA_SHADER_GEOMETRY];
+ struct tu_shader *last_shader = shaders[MESA_SHADER_GEOMETRY];
if (!last_shader)
- last_shader = builder->shaders[MESA_SHADER_TESS_EVAL];
+ last_shader = shaders[MESA_SHADER_TESS_EVAL];
if (!last_shader)
- last_shader = builder->shaders[MESA_SHADER_VERTEX];
+ last_shader = shaders[MESA_SHADER_VERTEX];
uint64_t outputs_written = last_shader->ir3_shader->nir->info.outputs_written;
- key.layer_zero = !(outputs_written & VARYING_BIT_LAYER);
- key.view_zero = !(outputs_written & VARYING_BIT_VIEWPORT);
+ ir3_key.layer_zero = !(outputs_written & VARYING_BIT_LAYER);
+ ir3_key.view_zero = !(outputs_written & VARYING_BIT_VIEWPORT);
+
+ compiled_shaders =
+ tu_shaders_init(builder->device, &pipeline_sha1, sizeof(pipeline_sha1));
- pipeline->tess.patch_type = key.tessellation;
+ if (!compiled_shaders) {
+ result = VK_ERROR_OUT_OF_HOST_MEMORY;
+ goto fail;
+ }
+
+ compiled_shaders->active_desc_sets = desc_sets;
+ compiled_shaders->multi_pos_output =
+ shaders[MESA_SHADER_VERTEX]->multi_pos_output;
for (gl_shader_stage stage = MESA_SHADER_VERTEX;
- stage < ARRAY_SIZE(builder->shaders); stage++) {
- if (!builder->shaders[stage])
+ stage < ARRAY_SIZE(shaders); stage++) {
+ if (!shaders[stage])
continue;
- bool created;
- builder->variants[stage] =
- ir3_shader_get_variant(builder->shaders[stage]->ir3_shader,
- &key, false, executable_info, &created);
- if (!builder->variants[stage])
+ compiled_shaders->variants[stage] =
+ ir3_shader_create_variant(shaders[stage]->ir3_shader, &ir3_key,
+ executable_info);
+ if (!compiled_shaders->variants[stage])
return VK_ERROR_OUT_OF_HOST_MEMORY;
+
+ compiled_shaders->push_consts[stage] = shaders[stage]->push_consts;
}
- uint32_t safe_constlens = ir3_trim_constlen(builder->variants, compiler);
+ uint32_t safe_constlens = ir3_trim_constlen(compiled_shaders->variants, compiler);
- key.safe_constlen = true;
+ ir3_key.safe_constlen = true;
for (gl_shader_stage stage = MESA_SHADER_VERTEX;
- stage < ARRAY_SIZE(builder->shaders); stage++) {
- if (!builder->shaders[stage])
+ stage < ARRAY_SIZE(shaders); stage++) {
+ if (!shaders[stage])
continue;
if (safe_constlens & (1 << stage)) {
- bool created;
- builder->variants[stage] =
- ir3_shader_get_variant(builder->shaders[stage]->ir3_shader,
- &key, false, executable_info, &created);
- if (!builder->variants[stage])
- return VK_ERROR_OUT_OF_HOST_MEMORY;
+ ralloc_free(compiled_shaders->variants[stage]);
+ compiled_shaders->variants[stage] =
+ ir3_shader_create_variant(shaders[stage]->ir3_shader, &ir3_key,
+ executable_info);
+ if (!compiled_shaders->variants[stage]) {
+ result = VK_ERROR_OUT_OF_HOST_MEMORY;
+ goto fail;
+ }
}
}
- const struct tu_shader *vs = builder->shaders[MESA_SHADER_VERTEX];
- struct ir3_shader_variant *variant;
-
- if (vs->ir3_shader->stream_output.num_outputs ||
- !ir3_has_binning_vs(&key)) {
- variant = builder->variants[MESA_SHADER_VERTEX];
- } else {
- bool created;
- key.safe_constlen = !!(safe_constlens & (1 << MESA_SHADER_VERTEX));
- variant = ir3_shader_get_variant(vs->ir3_shader, &key,
- true, executable_info, &created);
- if (!variant)
- return VK_ERROR_OUT_OF_HOST_MEMORY;
+ for (gl_shader_stage stage = MESA_SHADER_VERTEX;
+ stage < ARRAY_SIZE(nir); stage++) {
+ if (shaders[stage]) {
+ tu_shader_destroy(builder->device, shaders[stage], builder->alloc);
+ }
}
- builder->binning_variant = variant;
+ compiled_shaders =
+ tu_pipeline_cache_insert(builder->cache, compiled_shaders);
+done:
for (gl_shader_stage stage = MESA_SHADER_VERTEX;
stage < ARRAY_SIZE(nir); stage++) {
- if (builder->variants[stage]) {
- tu_append_executable(pipeline, builder->variants[stage],
+ if (compiled_shaders->variants[stage]) {
+ tu_append_executable(pipeline, compiled_shaders->variants[stage],
nir_initial_disasm[stage]);
}
}
- if (builder->binning_variant != builder->variants[MESA_SHADER_VERTEX]) {
- tu_append_executable(pipeline, builder->binning_variant, NULL);
+ struct ir3_shader_variant *vs =
+ compiled_shaders->variants[MESA_SHADER_VERTEX];
+
+ struct ir3_shader_variant *variant;
+ if (!vs->stream_output.num_outputs && ir3_has_binning_vs(&vs->key)) {
+ tu_append_executable(pipeline, vs->binning, NULL);
+ variant = vs->binning;
+ } else {
+ variant = vs;
+ }
+
+ builder->binning_variant = variant;
+
+ builder->shaders = compiled_shaders;
+
+ pipeline->active_desc_sets = compiled_shaders->active_desc_sets;
+ if (compiled_shaders->variants[MESA_SHADER_TESS_CTRL]) {
+ pipeline->tess.patch_type =
+ compiled_shaders->variants[MESA_SHADER_TESS_CTRL]->key.tessellation;
}
return VK_SUCCESS;
+
+fail:
+ for (gl_shader_stage stage = MESA_SHADER_VERTEX;
+ stage < ARRAY_SIZE(nir); stage++) {
+ if (shaders[stage]) {
+ tu_shader_destroy(builder->device, shaders[stage], builder->alloc);
+ }
+ }
+
+ if (compiled_shaders)
+ vk_pipeline_cache_object_unref(&compiled_shaders->base);
+
+ return result;
}
static void
static void
tu_pipeline_set_linkage(struct tu_program_descriptor_linkage *link,
- struct tu_shader *shader,
+ struct tu_push_constant_range *push_consts,
struct ir3_shader_variant *v)
{
link->const_state = *ir3_const_state(v);
link->constlen = v->constlen;
- link->push_consts = shader->push_consts;
+ link->push_consts = *push_consts;
}
static void
}
pipeline->active_stages = stages;
- for (unsigned i = 0; i < ARRAY_SIZE(builder->shaders); i++) {
- if (!builder->shaders[i])
+ for (unsigned i = 0; i < ARRAY_SIZE(builder->shaders->variants); i++) {
+ if (!builder->shaders->variants[i])
continue;
tu_pipeline_set_linkage(&pipeline->program.link[i],
- builder->shaders[i],
- builder->variants[i]);
+ &builder->shaders->push_consts[i],
+ builder->shaders->variants[i]);
}
}
{
const VkPipelineVertexInputStateCreateInfo *vi_info =
builder->create_info->pVertexInputState;
- const struct ir3_shader_variant *vs = builder->variants[MESA_SHADER_VERTEX];
+ const struct ir3_shader_variant *vs = builder->shaders->variants[MESA_SHADER_VERTEX];
const struct ir3_shader_variant *bs = builder->binning_variant;
/* Bindings may contain holes */
vk_find_struct_const(tess_info->pNext, PIPELINE_TESSELLATION_DOMAIN_ORIGIN_STATE_CREATE_INFO);
pipeline->tess.upper_left_domain_origin = !domain_info ||
domain_info->domainOrigin == VK_TESSELLATION_DOMAIN_ORIGIN_UPPER_LEFT;
- const struct ir3_shader_variant *hs = builder->variants[MESA_SHADER_TESS_CTRL];
+ const struct ir3_shader_variant *hs = builder->shaders->variants[MESA_SHADER_TESS_CTRL];
pipeline->tess.param_stride = hs->output_size * 4;
}
.bfref = ds_info->back.reference & 0xff));
}
- if (builder->shaders[MESA_SHADER_FRAGMENT]) {
- const struct ir3_shader_variant *fs = &builder->shaders[MESA_SHADER_FRAGMENT]->ir3_shader->variants[0];
+ if (builder->shaders->variants[MESA_SHADER_FRAGMENT]) {
+ const struct ir3_shader_variant *fs = builder->shaders->variants[MESA_SHADER_FRAGMENT];
if (fs->has_kill || fs->no_earlyz || fs->writes_pos) {
pipeline->lrz.force_disable_mask |= TU_LRZ_FORCE_DISABLE_WRITE;
}
result = tu_pipeline_builder_compile_shaders(builder, *pipeline);
if (result != VK_SUCCESS) {
vk_object_free(&builder->device->vk, builder->alloc, *pipeline);
- return result;
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
}
result = tu_pipeline_allocate_cs(builder->device, *pipeline,
- builder->layout, builder, builder->cache, NULL);
+ builder->layout, builder, NULL);
if (result != VK_SUCCESS) {
vk_object_free(&builder->device->vk, builder->alloc, *pipeline);
return result;
}
- for (uint32_t i = 0; i < ARRAY_SIZE(builder->variants); i++)
- builder->shader_iova[i] = tu_upload_variant(*pipeline, builder->variants[i]);
+ for (uint32_t i = 0; i < ARRAY_SIZE(builder->shader_iova); i++)
+ builder->shader_iova[i] =
+ tu_upload_variant(*pipeline, builder->shaders->variants[i]);
builder->binning_vs_iova =
tu_upload_variant(*pipeline, builder->binning_variant);
uint32_t pvtmem_size = 0;
bool per_wave = true;
- for (uint32_t i = 0; i < ARRAY_SIZE(builder->variants); i++) {
- if (builder->variants[i]) {
- pvtmem_size = MAX2(pvtmem_size, builder->variants[i]->pvtmem_size);
- if (!builder->variants[i]->pvtmem_per_wave)
+ for (uint32_t i = 0; i < ARRAY_SIZE(builder->shaders->variants); i++) {
+ if (builder->shaders->variants[i]) {
+ pvtmem_size = MAX2(pvtmem_size, builder->shaders->variants[i]->pvtmem_size);
+ if (!builder->shaders->variants[i]->pvtmem_per_wave)
per_wave = false;
}
}
static void
tu_pipeline_builder_finish(struct tu_pipeline_builder *builder)
{
- for (uint32_t i = 0; i < ARRAY_SIZE(builder->shaders); i++) {
- if (!builder->shaders[i])
- continue;
- tu_shader_destroy(builder->device, builder->shaders[i], builder->alloc);
- }
+ if (builder->shaders)
+ vk_pipeline_cache_object_unref(&builder->shaders->base);
ralloc_free(builder->mem_ctx);
}
tu_pipeline_builder_init_graphics(
struct tu_pipeline_builder *builder,
struct tu_device *dev,
- struct tu_pipeline_cache *cache,
+ struct vk_pipeline_cache *cache,
const VkGraphicsPipelineCreateInfo *create_info,
const VkAllocationCallbacks *alloc)
{
VkPipeline *pPipeline)
{
TU_FROM_HANDLE(tu_device, dev, device);
- TU_FROM_HANDLE(tu_pipeline_cache, cache, pipelineCache);
+ TU_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache);
+
+ cache = cache ? cache : dev->mem_cache;
struct tu_pipeline_builder builder;
tu_pipeline_builder_init_graphics(&builder, dev, cache,
VkPipeline *pPipeline)
{
TU_FROM_HANDLE(tu_device, dev, device);
- TU_FROM_HANDLE(tu_pipeline_cache, cache, pipelineCache);
+ TU_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache);
TU_FROM_HANDLE(tu_pipeline_layout, layout, pCreateInfo->layout);
const VkPipelineShaderStageCreateInfo *stage_info = &pCreateInfo->stage;
VkResult result;
+ cache = cache ? cache : dev->mem_cache;
+
struct tu_pipeline *pipeline;
*pPipeline = VK_NULL_HANDLE;
pipeline->executables_mem_ctx = ralloc_context(NULL);
util_dynarray_init(&pipeline->executables, pipeline->executables_mem_ctx);
- struct ir3_shader_key key = {};
+ struct tu_shader_key key;
+ tu_shader_key_init(&key, stage_info, dev);
void *pipeline_mem_ctx = ralloc_context(NULL);
- nir_shader *nir = tu_spirv_to_nir(dev, pipeline_mem_ctx, stage_info, MESA_SHADER_COMPUTE);
+
+ unsigned char pipeline_sha1[20];
+ tu_hash_compute(pipeline_sha1, stage_info, layout, &key, dev->compiler);
+
+ struct tu_compiled_shaders *compiled = NULL;
const bool executable_info = pCreateInfo->flags &
VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR;
- char *nir_initial_disasm = executable_info ?
- nir_shader_as_str(nir, pipeline->executables_mem_ctx) : NULL;
+ if (!executable_info)
+ compiled = tu_pipeline_cache_lookup(cache, pipeline_sha1, sizeof(pipeline_sha1));
- struct tu_shader *shader =
- tu_shader_create(dev, nir, stage_info, 0, layout, pAllocator);
- if (!shader) {
- result = VK_ERROR_OUT_OF_HOST_MEMORY;
- goto fail;
- }
+ char *nir_initial_disasm = NULL;
- pipeline->active_desc_sets = shader->active_desc_sets;
+ if (!compiled) {
+ struct ir3_shader_key ir3_key = {};
- bool created;
- struct ir3_shader_variant *v =
- ir3_shader_get_variant(shader->ir3_shader, &key, false, executable_info, &created);
- if (!v) {
- result = VK_ERROR_OUT_OF_HOST_MEMORY;
- goto fail;
+ nir_shader *nir = tu_spirv_to_nir(dev, pipeline_mem_ctx, stage_info,
+ MESA_SHADER_COMPUTE);
+
+ nir_initial_disasm = executable_info ?
+ nir_shader_as_str(nir, pipeline->executables_mem_ctx) : NULL;
+
+ struct tu_shader *shader =
+ tu_shader_create(dev, nir, &key, layout, pAllocator);
+ if (!shader) {
+ result = VK_ERROR_OUT_OF_HOST_MEMORY;
+ goto fail;
+ }
+
+ compiled = tu_shaders_init(dev, &pipeline_sha1, sizeof(pipeline_sha1));
+ if (!compiled) {
+ tu_shader_destroy(dev, shader, pAllocator);
+ result = VK_ERROR_OUT_OF_HOST_MEMORY;
+ goto fail;
+ }
+
+ compiled->active_desc_sets = shader->active_desc_sets;
+ compiled->push_consts[MESA_SHADER_COMPUTE] = shader->push_consts;
+
+ struct ir3_shader_variant *v =
+ ir3_shader_create_variant(shader->ir3_shader, &ir3_key, executable_info);
+
+ tu_shader_destroy(dev, shader, pAllocator);
+
+ if (!v) {
+ result = VK_ERROR_OUT_OF_HOST_MEMORY;
+ goto fail;
+ }
+
+ compiled->variants[MESA_SHADER_COMPUTE] = v;
+
+ compiled = tu_pipeline_cache_insert(cache, compiled);
}
+ pipeline->active_desc_sets = compiled->active_desc_sets;
+
+ struct ir3_shader_variant *v = compiled->variants[MESA_SHADER_COMPUTE];
+
tu_pipeline_set_linkage(&pipeline->program.link[MESA_SHADER_COMPUTE],
- shader, v);
+ &compiled->push_consts[MESA_SHADER_COMPUTE], v);
- result = tu_pipeline_allocate_cs(dev, pipeline, layout, NULL, cache, v);
+ result = tu_pipeline_allocate_cs(dev, pipeline, layout, NULL, v);
if (result != VK_SUCCESS)
goto fail;
struct tu_cs prog_cs;
uint32_t additional_reserve_size = tu_xs_get_additional_cs_size_dwords(v);
tu_cs_begin_sub_stream(&pipeline->cs, 64 + additional_reserve_size, &prog_cs);
- tu6_emit_cs_config(&prog_cs, shader, v, &pvtmem, shader_iova);
+ tu6_emit_cs_config(&prog_cs, v, &pvtmem, shader_iova);
pipeline->program.state = tu_cs_end_draw_state(&pipeline->cs, &prog_cs);
tu6_emit_load_state(pipeline, layout, true);
tu_append_executable(pipeline, v, nir_initial_disasm);
- tu_shader_destroy(dev, shader, pAllocator);
+ vk_pipeline_cache_object_unref(&compiled->base);
ralloc_free(pipeline_mem_ctx);
*pPipeline = tu_pipeline_to_handle(pipeline);
return VK_SUCCESS;
fail:
- if (shader)
- tu_shader_destroy(dev, shader, pAllocator);
+ if (compiled)
+ vk_pipeline_cache_object_unref(&compiled->base);
ralloc_free(pipeline_mem_ctx);
+++ /dev/null
-/*
- * Copyright © 2015 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- */
-
-#include "tu_private.h"
-
-#include "util/debug.h"
-#include "util/disk_cache.h"
-#include "util/mesa-sha1.h"
-#include "util/u_atomic.h"
-#include "vulkan/util/vk_util.h"
-
-struct cache_entry_variant_info
-{
-};
-
-struct cache_entry
-{
- union {
- unsigned char sha1[20];
- uint32_t sha1_dw[5];
- };
- uint32_t code_sizes[MESA_SHADER_STAGES];
- struct tu_shader_variant *variants[MESA_SHADER_STAGES];
- char code[0];
-};
-
-static void
-tu_pipeline_cache_init(struct tu_pipeline_cache *cache,
- struct tu_device *device)
-{
- cache->device = device;
- pthread_mutex_init(&cache->mutex, NULL);
-
- cache->modified = false;
- cache->kernel_count = 0;
- cache->total_size = 0;
- cache->table_size = 1024;
- const size_t byte_size = cache->table_size * sizeof(cache->hash_table[0]);
- cache->hash_table = malloc(byte_size);
-
- /* We don't consider allocation failure fatal, we just start with a 0-sized
- * cache. Disable caching when we want to keep shader debug info, since
- * we don't get the debug info on cached shaders. */
- if (cache->hash_table == NULL)
- cache->table_size = 0;
- else
- memset(cache->hash_table, 0, byte_size);
-}
-
-static void
-tu_pipeline_cache_finish(struct tu_pipeline_cache *cache)
-{
- for (unsigned i = 0; i < cache->table_size; ++i)
- if (cache->hash_table[i]) {
- vk_free(&cache->alloc, cache->hash_table[i]);
- }
- pthread_mutex_destroy(&cache->mutex);
- free(cache->hash_table);
-}
-
-static uint32_t
-entry_size(struct cache_entry *entry)
-{
- size_t ret = sizeof(*entry);
- for (int i = 0; i < MESA_SHADER_STAGES; ++i)
- if (entry->code_sizes[i])
- ret +=
- sizeof(struct cache_entry_variant_info) + entry->code_sizes[i];
- return ret;
-}
-
-static struct cache_entry *
-tu_pipeline_cache_search_unlocked(struct tu_pipeline_cache *cache,
- const unsigned char *sha1)
-{
- const uint32_t mask = cache->table_size - 1;
- const uint32_t start = (*(uint32_t *) sha1);
-
- if (cache->table_size == 0)
- return NULL;
-
- for (uint32_t i = 0; i < cache->table_size; i++) {
- const uint32_t index = (start + i) & mask;
- struct cache_entry *entry = cache->hash_table[index];
-
- if (!entry)
- return NULL;
-
- if (memcmp(entry->sha1, sha1, sizeof(entry->sha1)) == 0) {
- return entry;
- }
- }
-
- unreachable("hash table should never be full");
-}
-
-static struct cache_entry *
-tu_pipeline_cache_search(struct tu_pipeline_cache *cache,
- const unsigned char *sha1)
-{
- struct cache_entry *entry;
-
- pthread_mutex_lock(&cache->mutex);
-
- entry = tu_pipeline_cache_search_unlocked(cache, sha1);
-
- pthread_mutex_unlock(&cache->mutex);
-
- return entry;
-}
-
-static void
-tu_pipeline_cache_set_entry(struct tu_pipeline_cache *cache,
- struct cache_entry *entry)
-{
- const uint32_t mask = cache->table_size - 1;
- const uint32_t start = entry->sha1_dw[0];
-
- /* We'll always be able to insert when we get here. */
- assert(cache->kernel_count < cache->table_size / 2);
-
- for (uint32_t i = 0; i < cache->table_size; i++) {
- const uint32_t index = (start + i) & mask;
- if (!cache->hash_table[index]) {
- cache->hash_table[index] = entry;
- break;
- }
- }
-
- cache->total_size += entry_size(entry);
- cache->kernel_count++;
-}
-
-static VkResult
-tu_pipeline_cache_grow(struct tu_pipeline_cache *cache)
-{
- const uint32_t table_size = cache->table_size * 2;
- const uint32_t old_table_size = cache->table_size;
- const size_t byte_size = table_size * sizeof(cache->hash_table[0]);
- struct cache_entry **table;
- struct cache_entry **old_table = cache->hash_table;
-
- table = malloc(byte_size);
- if (table == NULL)
- return vk_error(cache, VK_ERROR_OUT_OF_HOST_MEMORY);
-
- cache->hash_table = table;
- cache->table_size = table_size;
- cache->kernel_count = 0;
- cache->total_size = 0;
-
- memset(cache->hash_table, 0, byte_size);
- for (uint32_t i = 0; i < old_table_size; i++) {
- struct cache_entry *entry = old_table[i];
- if (!entry)
- continue;
-
- tu_pipeline_cache_set_entry(cache, entry);
- }
-
- free(old_table);
-
- return VK_SUCCESS;
-}
-
-static void
-tu_pipeline_cache_add_entry(struct tu_pipeline_cache *cache,
- struct cache_entry *entry)
-{
- if (cache->kernel_count == cache->table_size / 2)
- tu_pipeline_cache_grow(cache);
-
- /* Failing to grow that hash table isn't fatal, but may mean we don't
- * have enough space to add this new kernel. Only add it if there's room.
- */
- if (cache->kernel_count < cache->table_size / 2)
- tu_pipeline_cache_set_entry(cache, entry);
-}
-
-static void
-tu_pipeline_cache_load(struct tu_pipeline_cache *cache,
- const void *data,
- size_t size)
-{
- struct tu_device *device = cache->device;
- struct vk_pipeline_cache_header header;
-
- if (size < sizeof(header))
- return;
- memcpy(&header, data, sizeof(header));
- if (header.header_size < sizeof(header))
- return;
- if (header.header_version != VK_PIPELINE_CACHE_HEADER_VERSION_ONE)
- return;
- if (header.vendor_id != 0x5143)
- return;
- if (header.device_id != device->physical_device->dev_id.chip_id)
- return;
- if (memcmp(header.uuid, device->physical_device->cache_uuid,
- VK_UUID_SIZE) != 0)
- return;
-
- char *end = (void *) data + size;
- char *p = (void *) data + header.header_size;
-
- while (end - p >= sizeof(struct cache_entry)) {
- struct cache_entry *entry = (struct cache_entry *) p;
- struct cache_entry *dest_entry;
- size_t size = entry_size(entry);
- if (end - p < size)
- break;
-
- dest_entry =
- vk_alloc(&cache->alloc, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_CACHE);
- if (dest_entry) {
- memcpy(dest_entry, entry, size);
- for (int i = 0; i < MESA_SHADER_STAGES; ++i)
- dest_entry->variants[i] = NULL;
- tu_pipeline_cache_add_entry(cache, dest_entry);
- }
- p += size;
- }
-}
-
-VKAPI_ATTR VkResult VKAPI_CALL
-tu_CreatePipelineCache(VkDevice _device,
- const VkPipelineCacheCreateInfo *pCreateInfo,
- const VkAllocationCallbacks *pAllocator,
- VkPipelineCache *pPipelineCache)
-{
- TU_FROM_HANDLE(tu_device, device, _device);
- struct tu_pipeline_cache *cache;
-
- assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO);
- assert(pCreateInfo->flags == 0);
-
- cache = vk_object_alloc(&device->vk, pAllocator, sizeof(*cache),
- VK_OBJECT_TYPE_PIPELINE_CACHE);
- if (cache == NULL)
- return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
-
- if (pAllocator)
- cache->alloc = *pAllocator;
- else
- cache->alloc = device->vk.alloc;
-
- tu_pipeline_cache_init(cache, device);
-
- if (pCreateInfo->initialDataSize > 0) {
- tu_pipeline_cache_load(cache, pCreateInfo->pInitialData,
- pCreateInfo->initialDataSize);
- }
-
- *pPipelineCache = tu_pipeline_cache_to_handle(cache);
-
- return VK_SUCCESS;
-}
-
-VKAPI_ATTR void VKAPI_CALL
-tu_DestroyPipelineCache(VkDevice _device,
- VkPipelineCache _cache,
- const VkAllocationCallbacks *pAllocator)
-{
- TU_FROM_HANDLE(tu_device, device, _device);
- TU_FROM_HANDLE(tu_pipeline_cache, cache, _cache);
-
- if (!cache)
- return;
- tu_pipeline_cache_finish(cache);
-
- vk_object_free(&device->vk, pAllocator, cache);
-}
-
-VKAPI_ATTR VkResult VKAPI_CALL
-tu_GetPipelineCacheData(VkDevice _device,
- VkPipelineCache _cache,
- size_t *pDataSize,
- void *pData)
-{
- TU_FROM_HANDLE(tu_device, device, _device);
- TU_FROM_HANDLE(tu_pipeline_cache, cache, _cache);
- struct vk_pipeline_cache_header *header;
- VkResult result = VK_SUCCESS;
-
- pthread_mutex_lock(&cache->mutex);
-
- const size_t size = sizeof(*header) + cache->total_size;
- if (pData == NULL) {
- pthread_mutex_unlock(&cache->mutex);
- *pDataSize = size;
- return VK_SUCCESS;
- }
- if (*pDataSize < sizeof(*header)) {
- pthread_mutex_unlock(&cache->mutex);
- *pDataSize = 0;
- return VK_INCOMPLETE;
- }
- void *p = pData, *end = pData + *pDataSize;
- header = p;
- header->header_size = sizeof(*header);
- header->header_version = VK_PIPELINE_CACHE_HEADER_VERSION_ONE;
- header->vendor_id = 0x5143;
- header->device_id = device->physical_device->dev_id.chip_id;
- memcpy(header->uuid, device->physical_device->cache_uuid, VK_UUID_SIZE);
- p += header->header_size;
-
- struct cache_entry *entry;
- for (uint32_t i = 0; i < cache->table_size; i++) {
- if (!cache->hash_table[i])
- continue;
- entry = cache->hash_table[i];
- const uint32_t size = entry_size(entry);
- if (end < p + size) {
- result = VK_INCOMPLETE;
- break;
- }
-
- memcpy(p, entry, size);
- for (int j = 0; j < MESA_SHADER_STAGES; ++j)
- ((struct cache_entry *) p)->variants[j] = NULL;
- p += size;
- }
- *pDataSize = p - pData;
-
- pthread_mutex_unlock(&cache->mutex);
- return result;
-}
-
-static void
-tu_pipeline_cache_merge(struct tu_pipeline_cache *dst,
- struct tu_pipeline_cache *src)
-{
- for (uint32_t i = 0; i < src->table_size; i++) {
- struct cache_entry *entry = src->hash_table[i];
- if (!entry || tu_pipeline_cache_search(dst, entry->sha1))
- continue;
-
- tu_pipeline_cache_add_entry(dst, entry);
-
- src->hash_table[i] = NULL;
- }
-}
-
-VKAPI_ATTR VkResult VKAPI_CALL
-tu_MergePipelineCaches(VkDevice _device,
- VkPipelineCache destCache,
- uint32_t srcCacheCount,
- const VkPipelineCache *pSrcCaches)
-{
- TU_FROM_HANDLE(tu_pipeline_cache, dst, destCache);
-
- for (uint32_t i = 0; i < srcCacheCount; i++) {
- TU_FROM_HANDLE(tu_pipeline_cache, src, pSrcCaches[i]);
-
- tu_pipeline_cache_merge(dst, src);
- }
-
- return VK_SUCCESS;
-}
projects / platform / upstream / mesa.git / commitdiff