static struct radv_shader *
radv_pipeline_create_gs_copy_shader(struct radv_device *device, struct radv_pipeline *pipeline,
+ struct vk_pipeline_cache *cache,
struct radv_pipeline_stage *stages,
const struct radv_pipeline_key *pipeline_key,
const struct radv_pipeline_layout *pipeline_layout,
.optimisations_disabled = pipeline_key->optimisations_disabled,
};
- return radv_shader_nir_to_asm(device, &gs_copy_stage, &nir, 1, &key, keep_executable_info,
+ return radv_shader_nir_to_asm(device, cache, &gs_copy_stage, &nir, 1, &key, keep_executable_info,
keep_statistic_info, gs_copy_binary);
}
static void
radv_pipeline_nir_to_asm(struct radv_device *device, struct radv_graphics_pipeline *pipeline,
- struct radv_pipeline_stage *stages,
+ struct vk_pipeline_cache *cache, struct radv_pipeline_stage *stages,
const struct radv_pipeline_key *pipeline_key,
const struct radv_pipeline_layout *pipeline_layout,
bool keep_executable_info, bool keep_statistic_info,
int64_t stage_start = os_time_get_nano();
pipeline->base.shaders[s] =
- radv_shader_nir_to_asm(device, &stages[s], shaders, shader_count, pipeline_key,
+ radv_shader_nir_to_asm(device, cache, &stages[s], shaders, shader_count, pipeline_key,
keep_executable_info, keep_statistic_info, &binaries[s]);
if (s == MESA_SHADER_GEOMETRY && !stages[s].info.is_ngg) {
- pipeline->base.gs_copy_shader = radv_pipeline_create_gs_copy_shader(device,
- &pipeline->base, stages, pipeline_key, pipeline_layout, keep_executable_info,
- keep_statistic_info, gs_copy_binary);
+ pipeline->base.gs_copy_shader = radv_pipeline_create_gs_copy_shader(
+ device, &pipeline->base, cache, stages, pipeline_key, pipeline_layout,
+ keep_executable_info, keep_statistic_info, gs_copy_binary);
}
stages[s].feedback.duration += os_time_get_nano() - stage_start;
radv_graphics_pipeline_compile(struct radv_graphics_pipeline *pipeline,
const VkGraphicsPipelineCreateInfo *pCreateInfo,
struct radv_pipeline_layout *pipeline_layout,
- struct radv_device *device, struct radv_pipeline_cache *cache,
+ struct radv_device *device, struct vk_pipeline_cache *cache,
const struct radv_pipeline_key *pipeline_key,
VkGraphicsPipelineLibraryFlagBitsEXT lib_flags,
bool fast_linking_enabled)
}
/* Compile NIR shaders to AMD assembly. */
- radv_pipeline_nir_to_asm(device, pipeline, stages, pipeline_key, pipeline_layout, keep_executable_info,
- keep_statistic_info, active_nir_stages, binaries, &gs_copy_binary);
+ radv_pipeline_nir_to_asm(device, pipeline, cache, stages, pipeline_key, pipeline_layout,
+ keep_executable_info, keep_statistic_info, active_nir_stages, binaries,
+ &gs_copy_binary);
if (!radv_pipeline_create_ps_epilog(device, pipeline, pipeline_key, lib_flags, noop_fs,
&ps_epilog_binary))
}
if (!skip_shaders_cache) {
- if (pipeline->base.gs_copy_shader) {
- assert(!binaries[MESA_SHADER_COMPUTE] && !pipeline->base.shaders[MESA_SHADER_COMPUTE]);
- binaries[MESA_SHADER_COMPUTE] = gs_copy_binary;
- pipeline->base.shaders[MESA_SHADER_COMPUTE] = pipeline->base.gs_copy_shader;
- }
-
radv_pipeline_cache_insert_shaders(device, cache, hash, &pipeline->base, binaries,
ps_epilog_binary, NULL, 0);
-
- if (pipeline->base.gs_copy_shader) {
- pipeline->base.gs_copy_shader = pipeline->base.shaders[MESA_SHADER_COMPUTE];
- pipeline->base.shaders[MESA_SHADER_COMPUTE] = NULL;
- binaries[MESA_SHADER_COMPUTE] = NULL;
- }
}
free(gs_copy_binary);
static VkResult
radv_graphics_pipeline_init(struct radv_graphics_pipeline *pipeline, struct radv_device *device,
- struct radv_pipeline_cache *cache,
+ struct vk_pipeline_cache *cache,
const VkGraphicsPipelineCreateInfo *pCreateInfo,
const struct radv_graphics_pipeline_create_info *extra)
{
const VkAllocationCallbacks *pAllocator, VkPipeline *pPipeline)
{
RADV_FROM_HANDLE(radv_device, device, _device);
- RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache);
+ VK_FROM_HANDLE(vk_pipeline_cache, cache, _cache);
struct radv_graphics_pipeline *pipeline;
VkResult result;
static VkResult
radv_graphics_lib_pipeline_init(struct radv_graphics_lib_pipeline *pipeline,
- struct radv_device *device, struct radv_pipeline_cache *cache,
+ struct radv_device *device, struct vk_pipeline_cache *cache,
const VkGraphicsPipelineCreateInfo *pCreateInfo)
{
VkResult result;
const VkGraphicsPipelineCreateInfo *pCreateInfo,
const VkAllocationCallbacks *pAllocator, VkPipeline *pPipeline)
{
- RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache);
+ VK_FROM_HANDLE(vk_pipeline_cache, cache, _cache);
RADV_FROM_HANDLE(radv_device, device, _device);
struct radv_graphics_lib_pipeline *pipeline;
VkResult result;
static VkResult
radv_compute_pipeline_compile(struct radv_compute_pipeline *pipeline,
struct radv_pipeline_layout *pipeline_layout,
- struct radv_device *device, struct radv_pipeline_cache *cache,
+ struct radv_device *device, struct vk_pipeline_cache *cache,
const struct radv_pipeline_key *pipeline_key,
const VkPipelineShaderStageCreateInfo *pStage,
const VkPipelineCreateFlags flags,
/* Compile NIR shader to AMD assembly. */
pipeline->base.shaders[MESA_SHADER_COMPUTE] = radv_shader_nir_to_asm(
- device, &cs_stage, &cs_stage.nir, 1, pipeline_key,
- keep_executable_info, keep_statistic_info, &binaries[MESA_SHADER_COMPUTE]);
+ device, cache, &cs_stage, &cs_stage.nir, 1, pipeline_key, keep_executable_info,
+ keep_statistic_info, &binaries[MESA_SHADER_COMPUTE]);
cs_stage.feedback.duration += os_time_get_nano() - stage_start;
const VkAllocationCallbacks *pAllocator, VkPipeline *pPipeline)
{
RADV_FROM_HANDLE(radv_device, device, _device);
- RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache);
+ VK_FROM_HANDLE(vk_pipeline_cache, cache, _cache);
RADV_FROM_HANDLE(radv_pipeline_layout, pipeline_layout, pCreateInfo->layout);
struct radv_compute_pipeline *pipeline;
VkResult result;
#include "aco_interface.h"
#include "vk_pipeline.h"
-struct cache_entry {
- union {
- unsigned char sha1[SHA1_DIGEST_LENGTH];
- uint32_t sha1_dw[5];
- };
- uint32_t binary_sizes[MESA_VULKAN_SHADER_STAGES];
- uint32_t num_stack_sizes;
- struct radv_shader *shaders[MESA_VULKAN_SHADER_STAGES];
- uint32_t ps_epilog_binary_size;
- struct radv_shader_part *ps_epilog;
- char code[0];
-};
-
-static void
-radv_pipeline_cache_lock(struct radv_pipeline_cache *cache)
-{
- if (cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT)
- return;
-
- mtx_lock(&cache->mutex);
-}
-
-static void
-radv_pipeline_cache_unlock(struct radv_pipeline_cache *cache)
-{
- if (cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT)
- return;
-
- mtx_unlock(&cache->mutex);
-}
-
static bool
radv_is_cache_disabled(struct radv_device *device)
{
(device->physical_device->use_llvm ? 0 : aco_get_codegen_flags());
}
-static void
-radv_pipeline_cache_init(struct radv_pipeline_cache *cache, struct radv_device *device)
-{
- vk_object_base_init(&device->vk, &cache->base, VK_OBJECT_TYPE_PIPELINE_CACHE);
-
- cache->device = device;
- mtx_init(&cache->mutex, mtx_plain);
- cache->flags = 0;
-
- cache->kernel_count = 0;
- cache->total_size = 0;
- cache->table_size = 1024;
- const size_t byte_size = cache->table_size * sizeof(cache->hash_table[0]);
- cache->hash_table = malloc(byte_size);
-
- /* We don't consider allocation failure fatal, we just start with a 0-sized
- * cache. Disable caching when we want to keep shader debug info, since
- * we don't get the debug info on cached shaders. */
- if (cache->hash_table == NULL || radv_is_cache_disabled(device))
- cache->table_size = 0;
- else
- memset(cache->hash_table, 0, byte_size);
-}
-
-static void
-radv_pipeline_cache_finish(struct radv_pipeline_cache *cache)
-{
- for (unsigned i = 0; i < cache->table_size; ++i)
- if (cache->hash_table[i]) {
- for (int j = 0; j < MESA_VULKAN_SHADER_STAGES; ++j) {
- if (cache->hash_table[i]->shaders[j])
- radv_shader_unref(cache->device, cache->hash_table[i]->shaders[j]);
- }
- vk_free(&cache->alloc, cache->hash_table[i]);
- }
- mtx_destroy(&cache->mutex);
- free(cache->hash_table);
-
- vk_object_base_finish(&cache->base);
-}
-
-static uint32_t
-entry_size(const struct cache_entry *entry)
-{
- size_t ret = sizeof(*entry);
- for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i)
- if (entry->binary_sizes[i])
- ret += entry->binary_sizes[i];
- if (entry->ps_epilog_binary_size)
- ret += entry->ps_epilog_binary_size;
- ret += sizeof(struct radv_pipeline_shader_stack_size) * entry->num_stack_sizes;
- ret = align(ret, alignof(struct cache_entry));
- return ret;
-}
-
void
radv_hash_shaders(unsigned char *hash, const struct radv_pipeline_stage *stages,
uint32_t stage_count, const struct radv_pipeline_layout *layout,
if (radv_is_cache_disabled(device))
return radv_shader_create(device, binary);
+ if (!cache)
+ cache = device->mem_cache;
+
uint8_t hash[SHA1_DIGEST_LENGTH];
_mesa_sha1_compute(binary, binary->total_size, hash);
.destroy = radv_pipeline_cache_object_destroy,
};
-static struct cache_entry *
-radv_pipeline_cache_search_unlocked(struct radv_pipeline_cache *cache, const unsigned char *sha1)
-{
- const uint32_t mask = cache->table_size - 1;
- const uint32_t start = (*(uint32_t *)sha1);
-
- if (cache->table_size == 0)
- return NULL;
-
- for (uint32_t i = 0; i < cache->table_size; i++) {
- const uint32_t index = (start + i) & mask;
- struct cache_entry *entry = cache->hash_table[index];
-
- if (!entry)
- return NULL;
-
- if (memcmp(entry->sha1, sha1, sizeof(entry->sha1)) == 0) {
- return entry;
- }
- }
-
- unreachable("hash table should never be full");
-}
-
-static struct cache_entry *
-radv_pipeline_cache_search(struct radv_pipeline_cache *cache, const unsigned char *sha1)
-{
- struct cache_entry *entry;
-
- radv_pipeline_cache_lock(cache);
-
- entry = radv_pipeline_cache_search_unlocked(cache, sha1);
-
- radv_pipeline_cache_unlock(cache);
-
- return entry;
-}
-
-static void
-radv_pipeline_cache_set_entry(struct radv_pipeline_cache *cache, struct cache_entry *entry)
-{
- const uint32_t mask = cache->table_size - 1;
- const uint32_t start = entry->sha1_dw[0];
-
- /* We'll always be able to insert when we get here. */
- assert(cache->kernel_count < cache->table_size / 2);
-
- for (uint32_t i = 0; i < cache->table_size; i++) {
- const uint32_t index = (start + i) & mask;
- if (!cache->hash_table[index]) {
- cache->hash_table[index] = entry;
- break;
- }
- }
-
- cache->total_size += entry_size(entry);
- cache->kernel_count++;
-}
-
-static VkResult
-radv_pipeline_cache_grow(struct radv_pipeline_cache *cache)
-{
- const uint32_t table_size = cache->table_size * 2;
- const uint32_t old_table_size = cache->table_size;
- const size_t byte_size = table_size * sizeof(cache->hash_table[0]);
- struct cache_entry **table;
- struct cache_entry **old_table = cache->hash_table;
-
- table = malloc(byte_size);
- if (table == NULL)
- return vk_error(cache, VK_ERROR_OUT_OF_HOST_MEMORY);
-
- cache->hash_table = table;
- cache->table_size = table_size;
- cache->kernel_count = 0;
- cache->total_size = 0;
-
- memset(cache->hash_table, 0, byte_size);
- for (uint32_t i = 0; i < old_table_size; i++) {
- struct cache_entry *entry = old_table[i];
- if (!entry)
- continue;
-
- radv_pipeline_cache_set_entry(cache, entry);
- }
-
- free(old_table);
-
- return VK_SUCCESS;
-}
-
-static void
-radv_pipeline_cache_add_entry(struct radv_pipeline_cache *cache, struct cache_entry *entry)
-{
- if (cache->kernel_count == cache->table_size / 2)
- radv_pipeline_cache_grow(cache);
-
- /* Failing to grow that hash table isn't fatal, but may mean we don't
- * have enough space to add this new kernel. Only add it if there's room.
- */
- if (cache->kernel_count < cache->table_size / 2)
- radv_pipeline_cache_set_entry(cache, entry);
-}
-
bool
-radv_create_shaders_from_pipeline_cache(struct radv_device *device,
- struct radv_pipeline_cache *cache,
+radv_create_shaders_from_pipeline_cache(struct radv_device *device, struct vk_pipeline_cache *cache,
const unsigned char *sha1, struct radv_pipeline *pipeline,
struct radv_ray_tracing_module *rt_groups,
uint32_t num_rt_groups, bool *found_in_application_cache)
{
- struct cache_entry *entry;
+ *found_in_application_cache = false;
+
+ if (radv_is_cache_disabled(device))
+ return false;
+ bool *found = found_in_application_cache;
if (!cache) {
cache = device->mem_cache;
- *found_in_application_cache = false;
+ found = NULL;
}
- radv_pipeline_cache_lock(cache);
-
- entry = radv_pipeline_cache_search_unlocked(cache, sha1);
+ struct vk_pipeline_cache_object *object =
+ vk_pipeline_cache_lookup_object(cache, sha1, SHA1_DIGEST_LENGTH, &radv_pipeline_ops, found);
- if (!entry) {
- *found_in_application_cache = false;
-
- /* Don't cache when we want debug info, since this isn't
- * present in the cache.
- */
- if (radv_is_cache_disabled(device) || !device->physical_device->vk.disk_cache) {
- radv_pipeline_cache_unlock(cache);
- return false;
- }
+ if (!object)
+ return false;
- uint8_t disk_sha1[SHA1_DIGEST_LENGTH];
- disk_cache_compute_key(device->physical_device->vk.disk_cache, sha1, SHA1_DIGEST_LENGTH, disk_sha1);
+ struct radv_pipeline_cache_object *pipeline_obj =
+ container_of(object, struct radv_pipeline_cache_object, base);
- entry =
- (struct cache_entry *)disk_cache_get(device->physical_device->vk.disk_cache, disk_sha1, NULL);
- if (!entry) {
- radv_pipeline_cache_unlock(cache);
- return false;
+ for (unsigned i = 0; i < pipeline_obj->num_shaders; i++) {
+ gl_shader_stage s = pipeline_obj->shaders[i]->info.stage;
+ if (s == MESA_SHADER_VERTEX && i > 0) {
+ /* The GS copy-shader is a VS placed after all other stages */
+ assert(i == pipeline_obj->num_shaders - 1 && pipeline->shaders[MESA_SHADER_GEOMETRY]);
+ pipeline->gs_copy_shader = radv_shader_ref(pipeline_obj->shaders[i]);
} else {
- size_t size = entry_size(entry);
- struct cache_entry *new_entry =
- vk_alloc(&cache->alloc, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_CACHE);
- if (!new_entry) {
- free(entry);
- radv_pipeline_cache_unlock(cache);
- return false;
- }
-
- memcpy(new_entry, entry, entry_size(entry));
- free(entry);
- entry = new_entry;
-
- if (!(device->instance->debug_flags & RADV_DEBUG_NO_MEMORY_CACHE) ||
- cache != device->mem_cache)
- radv_pipeline_cache_add_entry(cache, new_entry);
+ pipeline->shaders[s] = radv_shader_ref(pipeline_obj->shaders[i]);
}
}
- char *p = entry->code;
- for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) {
- if (!entry->shaders[i] && entry->binary_sizes[i]) {
- struct radv_shader_binary *binary = calloc(1, entry->binary_sizes[i]);
- memcpy(binary, p, entry->binary_sizes[i]);
- p += entry->binary_sizes[i];
-
- entry->shaders[i] = radv_shader_create(device, binary);
-
- free(binary);
- } else if (entry->binary_sizes[i]) {
- p += entry->binary_sizes[i];
- }
- }
-
- for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) {
- if (!entry->shaders[i])
- continue;
-
- pipeline->shaders[i] = entry->shaders[i];
- }
-
- if (pipeline->shaders[MESA_SHADER_GEOMETRY] &&
- !pipeline->shaders[MESA_SHADER_GEOMETRY]->info.is_ngg) {
- /* For the GS copy shader, RADV uses the compute shader slot to avoid a new cache entry. */
- pipeline->gs_copy_shader = pipeline->shaders[MESA_SHADER_COMPUTE];
- pipeline->shaders[MESA_SHADER_COMPUTE] = NULL;
- }
-
- if (!entry->ps_epilog && entry->ps_epilog_binary_size) {
- struct radv_shader_part_binary *binary = calloc(1, entry->ps_epilog_binary_size);
- memcpy(binary, p, entry->ps_epilog_binary_size);
- p += entry->ps_epilog_binary_size;
-
- entry->ps_epilog = radv_shader_part_create(device, binary,
- device->physical_device->ps_wave_size);
-
- free(binary);
- }
-
- if (entry->ps_epilog) {
- if (pipeline->type == RADV_PIPELINE_GRAPHICS) {
- radv_pipeline_to_graphics(pipeline)->ps_epilog = entry->ps_epilog;
- } else {
- radv_pipeline_to_graphics_lib(pipeline)->base.ps_epilog = entry->ps_epilog;
- }
- }
+ if (pipeline_obj->ps_epilog) {
+ assert(num_rt_groups == 0);
+ struct radv_shader_part *ps_epilog = radv_shader_part_ref(pipeline_obj->ps_epilog);
- assert(num_rt_groups == entry->num_stack_sizes);
- for (int i = 0; i < num_rt_groups; ++i) {
- memcpy(&rt_groups[i].stack_size, p, sizeof(struct radv_pipeline_shader_stack_size));
- p += sizeof(struct radv_pipeline_shader_stack_size);
+ if (pipeline->type == RADV_PIPELINE_GRAPHICS)
+ radv_pipeline_to_graphics(pipeline)->ps_epilog = ps_epilog;
+ else
+ radv_pipeline_to_graphics_lib(pipeline)->base.ps_epilog = ps_epilog;
}
- if (device->instance->debug_flags & RADV_DEBUG_NO_MEMORY_CACHE && cache == device->mem_cache)
- vk_free(&cache->alloc, entry);
- else {
- for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i)
- if (entry->shaders[i])
- radv_shader_ref(entry->shaders[i]);
-
- if (entry->ps_epilog)
- radv_shader_part_ref(entry->ps_epilog);
+ if (num_rt_groups) {
+ assert(num_rt_groups == pipeline_obj->num_stack_sizes);
+ assert(pipeline_obj->ps_epilog == NULL);
+ struct radv_pipeline_shader_stack_size *stack_sizes = pipeline_obj->data;
+ for (unsigned i = 0; i < num_rt_groups; i++)
+ rt_groups[i].stack_size = stack_sizes[i];
}
- assert((uintptr_t)p <= (uintptr_t)entry + entry_size(entry));
- radv_pipeline_cache_unlock(cache);
+ vk_pipeline_cache_object_unref(&device->vk, object);
return true;
}
void
-radv_pipeline_cache_insert_shaders(struct radv_device *device, struct radv_pipeline_cache *cache,
+radv_pipeline_cache_insert_shaders(struct radv_device *device, struct vk_pipeline_cache *cache,
const unsigned char *sha1, struct radv_pipeline *pipeline,
struct radv_shader_binary *const *binaries,
struct radv_shader_part_binary *ps_epilog_binary,
const struct radv_ray_tracing_module *rt_groups,
uint32_t num_rt_groups)
{
+ if (radv_is_cache_disabled(device))
+ return;
+
if (!cache)
cache = device->mem_cache;
- radv_pipeline_cache_lock(cache);
- struct cache_entry *entry = radv_pipeline_cache_search_unlocked(cache, sha1);
- if (entry) {
- for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) {
- if (!entry->shaders[i])
- continue;
+ /* Count shaders */
+ unsigned num_shaders = 0;
+ for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i)
+ num_shaders += pipeline->shaders[i] ? 1 : 0;
+ num_shaders += pipeline->gs_copy_shader ? 1 : 0;
- radv_shader_unref(cache->device, pipeline->shaders[i]);
+ unsigned ps_epilog_binary_size = ps_epilog_binary ? ps_epilog_binary->total_size : 0;
- pipeline->shaders[i] = entry->shaders[i];
- radv_shader_ref(pipeline->shaders[i]);
- }
-
- if (entry->ps_epilog) {
- struct radv_graphics_pipeline *graphics_pipeline = radv_pipeline_to_graphics(pipeline);
-
- radv_shader_part_unref(cache->device, graphics_pipeline->ps_epilog);
-
- graphics_pipeline->ps_epilog = entry->ps_epilog;
- radv_shader_part_ref(graphics_pipeline->ps_epilog);
- }
-
- radv_pipeline_cache_unlock(cache);
- return;
- }
+ struct radv_pipeline_cache_object *pipeline_obj;
+ pipeline_obj = radv_pipeline_cache_object_create(&device->vk, num_shaders, sha1, num_rt_groups,
+ ps_epilog_binary_size);
- /* Don't cache when we want debug info, since this isn't
- * present in the cache.
- */
- if (radv_is_cache_disabled(device)) {
- radv_pipeline_cache_unlock(cache);
+ if (!pipeline_obj)
return;
- }
- size_t size = sizeof(*entry) + sizeof(struct radv_pipeline_shader_stack_size) * num_rt_groups;
- for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i)
- if (binaries[i])
- size += binaries[i]->total_size;
- if (ps_epilog_binary)
- size += ps_epilog_binary->total_size;
- const size_t size_without_align = size;
- size = align(size_without_align, alignof(struct cache_entry));
-
- entry = vk_alloc(&cache->alloc, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_CACHE);
- if (!entry) {
- radv_pipeline_cache_unlock(cache);
- return;
+ unsigned idx = 0;
+ for (unsigned i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) {
+ if (pipeline->shaders[i])
+ pipeline_obj->shaders[idx++] = radv_shader_ref(pipeline->shaders[i]);
}
+ /* Place the GS copy-shader after all other stages */
+ if (pipeline->gs_copy_shader)
+ pipeline_obj->shaders[idx++] = radv_shader_ref(pipeline->gs_copy_shader);
- memset(entry, 0, sizeof(*entry));
- memcpy(entry->sha1, sha1, SHA1_DIGEST_LENGTH);
-
- char *p = entry->code;
-
- for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) {
- if (!binaries[i])
- continue;
-
- entry->binary_sizes[i] = binaries[i]->total_size;
-
- memcpy(p, binaries[i], binaries[i]->total_size);
- p += binaries[i]->total_size;
- }
+ assert(idx == num_shaders);
if (ps_epilog_binary) {
- entry->ps_epilog_binary_size = ps_epilog_binary->total_size;
- memcpy(p, ps_epilog_binary, ps_epilog_binary->total_size);
- p += ps_epilog_binary->total_size;
- }
-
- for (int i = 0; i < num_rt_groups; ++i) {
- memcpy(p, &rt_groups[i].stack_size, sizeof(struct radv_pipeline_shader_stack_size));
- p += sizeof(struct radv_pipeline_shader_stack_size);
- }
- entry->num_stack_sizes = num_rt_groups;
-
- // Make valgrind happy by filling the alignment hole at the end.
- assert(p == (char *)entry + size_without_align);
- assert(sizeof(*entry) + (p - entry->code) == size_without_align);
- memset((char *)entry + size_without_align, 0, size - size_without_align);
-
- /* Always add cache items to disk. This will allow collection of
- * compiled shaders by third parties such as steam, even if the app
- * implements its own pipeline cache.
- *
- * Make sure to exclude meta shaders because they are stored in a different cache file.
- */
- if (device->physical_device->vk.disk_cache && cache != radv_pipeline_cache_from_handle(device->meta_state.cache)) {
- uint8_t disk_sha1[SHA1_DIGEST_LENGTH];
- disk_cache_compute_key(device->physical_device->vk.disk_cache, sha1, SHA1_DIGEST_LENGTH, disk_sha1);
-
- disk_cache_put(device->physical_device->vk.disk_cache, disk_sha1, entry, entry_size(entry),
- NULL);
- }
-
- if (device->instance->debug_flags & RADV_DEBUG_NO_MEMORY_CACHE && cache == device->mem_cache) {
- vk_free2(&cache->alloc, NULL, entry);
- radv_pipeline_cache_unlock(cache);
- return;
- }
-
- /* We delay setting the shader so we have reproducible disk cache
- * items.
- */
- for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i) {
- if (!binaries[i])
- continue;
- assert(pipeline->shaders[i]);
-
- entry->shaders[i] = pipeline->shaders[i];
- radv_shader_ref(pipeline->shaders[i]);
- }
-
- if (ps_epilog_binary) {
- struct radv_shader_part *ps_epilog = NULL;
-
- if (pipeline->type == RADV_PIPELINE_GRAPHICS) {
+ memcpy(pipeline_obj->data, ps_epilog_binary, ps_epilog_binary_size);
+ struct radv_shader_part *ps_epilog;
+ if (pipeline->type == RADV_PIPELINE_GRAPHICS)
ps_epilog = radv_pipeline_to_graphics(pipeline)->ps_epilog;
- } else {
+ else
ps_epilog = radv_pipeline_to_graphics_lib(pipeline)->base.ps_epilog;
- }
- entry->ps_epilog = ps_epilog;
- radv_shader_part_ref(ps_epilog);
+ pipeline_obj->ps_epilog = radv_shader_part_ref(ps_epilog);
}
- radv_pipeline_cache_add_entry(cache, entry);
-
- radv_pipeline_cache_unlock(cache);
- return;
-}
-
-static bool
-radv_pipeline_cache_load(struct radv_pipeline_cache *cache, const void *data, size_t size)
-{
- struct radv_device *device = cache->device;
- struct vk_pipeline_cache_header header;
-
- if (size < sizeof(header))
- return false;
- memcpy(&header, data, sizeof(header));
- if (header.header_size < sizeof(header))
- return false;
- if (header.header_version != VK_PIPELINE_CACHE_HEADER_VERSION_ONE)
- return false;
- if (header.vendor_id != ATI_VENDOR_ID)
- return false;
- if (header.device_id != device->physical_device->rad_info.pci_id)
- return false;
- if (memcmp(header.uuid, device->physical_device->cache_uuid, VK_UUID_SIZE) != 0)
- return false;
-
- char *end = (char *)data + size;
- char *p = (char *)data + header.header_size;
-
- while (end - p >= sizeof(struct cache_entry)) {
- struct cache_entry *entry = (struct cache_entry *)p;
- struct cache_entry *dest_entry;
- size_t size_of_entry = entry_size(entry);
- if (end - p < size_of_entry)
- break;
-
- dest_entry = vk_alloc(&cache->alloc, size_of_entry, 8, VK_SYSTEM_ALLOCATION_SCOPE_CACHE);
- if (dest_entry) {
- memcpy(dest_entry, entry, size_of_entry);
- for (int i = 0; i < MESA_VULKAN_SHADER_STAGES; ++i)
- dest_entry->shaders[i] = NULL;
- radv_pipeline_cache_add_entry(cache, dest_entry);
- }
- p += size_of_entry;
- }
-
- return true;
-}
-
-VKAPI_ATTR VkResult VKAPI_CALL
-radv_CreatePipelineCache(VkDevice _device, const VkPipelineCacheCreateInfo *pCreateInfo,
- const VkAllocationCallbacks *pAllocator, VkPipelineCache *pPipelineCache)
-{
- RADV_FROM_HANDLE(radv_device, device, _device);
- struct radv_pipeline_cache *cache;
-
- assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO);
-
- cache = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*cache), 8,
- VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
- if (cache == NULL)
- return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
-
- if (pAllocator)
- cache->alloc = *pAllocator;
- else
- cache->alloc = device->vk.alloc;
-
- radv_pipeline_cache_init(cache, device);
- cache->flags = pCreateInfo->flags;
-
- if (pCreateInfo->initialDataSize > 0) {
- radv_pipeline_cache_load(cache, pCreateInfo->pInitialData, pCreateInfo->initialDataSize);
- }
-
- *pPipelineCache = radv_pipeline_cache_to_handle(cache);
-
- return VK_SUCCESS;
-}
-
-VKAPI_ATTR void VKAPI_CALL
-radv_DestroyPipelineCache(VkDevice _device, VkPipelineCache _cache,
- const VkAllocationCallbacks *pAllocator)
-{
- RADV_FROM_HANDLE(radv_device, device, _device);
- RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache);
-
- if (!cache)
- return;
-
- radv_pipeline_cache_finish(cache);
- vk_free2(&device->vk.alloc, pAllocator, cache);
-}
-
-VKAPI_ATTR VkResult VKAPI_CALL
-radv_GetPipelineCacheData(VkDevice _device, VkPipelineCache _cache, size_t *pDataSize, void *pData)
-{
- RADV_FROM_HANDLE(radv_device, device, _device);
- RADV_FROM_HANDLE(radv_pipeline_cache, cache, _cache);
- struct vk_pipeline_cache_header *header;
- VkResult result = VK_SUCCESS;
-
- radv_pipeline_cache_lock(cache);
-
- const size_t size = sizeof(*header) + cache->total_size;
- if (pData == NULL) {
- radv_pipeline_cache_unlock(cache);
- *pDataSize = size;
- return VK_SUCCESS;
- }
- if (*pDataSize < sizeof(*header)) {
- radv_pipeline_cache_unlock(cache);
- *pDataSize = 0;
- return VK_INCOMPLETE;
- }
- void *p = pData, *end = (char *)pData + *pDataSize;
- header = p;
- header->header_size = align(sizeof(*header), alignof(struct cache_entry));
- header->header_version = VK_PIPELINE_CACHE_HEADER_VERSION_ONE;
- header->vendor_id = ATI_VENDOR_ID;
- header->device_id = device->physical_device->rad_info.pci_id;
- memcpy(header->uuid, device->physical_device->cache_uuid, VK_UUID_SIZE);
- p = (char *)p + header->header_size;
-
- struct cache_entry *entry;
- for (uint32_t i = 0; i < cache->table_size; i++) {
- if (!cache->hash_table[i])
- continue;
- entry = cache->hash_table[i];
- const uint32_t size_of_entry = entry_size(entry);
- if ((char *)end < (char *)p + size_of_entry) {
- result = VK_INCOMPLETE;
- break;
- }
-
- memcpy(p, entry, size_of_entry);
- for (int j = 0; j < MESA_VULKAN_SHADER_STAGES; ++j)
- ((struct cache_entry *)p)->shaders[j] = NULL;
- p = (char *)p + size_of_entry;
- }
- *pDataSize = (char *)p - (char *)pData;
-
- radv_pipeline_cache_unlock(cache);
- return result;
-}
-
-static void
-radv_pipeline_cache_merge(struct radv_pipeline_cache *dst, struct radv_pipeline_cache *src)
-{
- for (uint32_t i = 0; i < src->table_size; i++) {
- struct cache_entry *entry = src->hash_table[i];
- if (!entry || radv_pipeline_cache_search(dst, entry->sha1))
- continue;
-
- radv_pipeline_cache_add_entry(dst, entry);
-
- src->hash_table[i] = NULL;
- }
-}
-
-VKAPI_ATTR VkResult VKAPI_CALL
-radv_MergePipelineCaches(VkDevice _device, VkPipelineCache destCache, uint32_t srcCacheCount,
- const VkPipelineCache *pSrcCaches)
-{
- RADV_FROM_HANDLE(radv_pipeline_cache, dst, destCache);
-
- for (uint32_t i = 0; i < srcCacheCount; i++) {
- RADV_FROM_HANDLE(radv_pipeline_cache, src, pSrcCaches[i]);
-
- radv_pipeline_cache_merge(dst, src);
- }
+ struct radv_pipeline_shader_stack_size *stack_sizes = pipeline_obj->data;
+ for (unsigned i = 0; i < num_rt_groups; i++)
+ stack_sizes[i] = rt_groups[i].stack_size;
- return VK_SUCCESS;
+ /* Add the object to the cache */
+ struct vk_pipeline_cache_object *object =
+ vk_pipeline_cache_add_object(cache, &pipeline_obj->base);
+ vk_pipeline_cache_object_unref(&device->vk, object);
}
projects / platform / upstream / mesa.git / commitdiff