--- /dev/null
+/*
+ * Copyright © 2021 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "vk_pipeline_cache.h"
+
+#include "vk_alloc.h"
+#include "vk_common_entrypoints.h"
+#include "vk_device.h"
+#include "vk_log.h"
+#include "vk_physical_device.h"
+
+#include "util/blob.h"
+#include "util/debug.h"
+#include "util/disk_cache.h"
+#include "util/hash_table.h"
+#include "util/set.h"
+
+struct raw_data_object {
+ struct vk_pipeline_cache_object base;
+
+ const void *data;
+ size_t data_size;
+};
+
+static bool
+raw_data_object_serialize(struct vk_pipeline_cache_object *object,
+ struct blob *blob)
+{
+ struct raw_data_object *data_obj =
+ container_of(object, struct raw_data_object, base);
+
+ blob_write_bytes(blob, data_obj->data, data_obj->data_size);
+
+ return true;
+}
+
+static void
+raw_data_object_destroy(struct vk_pipeline_cache_object *object)
+{
+ struct raw_data_object *data_obj =
+ container_of(object, struct raw_data_object, base);
+
+ vk_free(&data_obj->base.device->alloc, data_obj);
+}
+
+static const struct vk_pipeline_cache_object_ops raw_data_object_ops = {
+ .serialize = raw_data_object_serialize,
+ .destroy = raw_data_object_destroy,
+};
+
+static struct raw_data_object *
+raw_data_object_create(struct vk_device *device,
+ const void *key_data, size_t key_size,
+ const void *data, size_t data_size)
+{
+ VK_MULTIALLOC(ma);
+ VK_MULTIALLOC_DECL(&ma, struct raw_data_object, data_obj, 1);
+ VK_MULTIALLOC_DECL_SIZE(&ma, void, obj_key_data, key_size);
+ VK_MULTIALLOC_DECL_SIZE(&ma, void, obj_data, data_size);
+
+ if (!vk_multialloc_alloc(&ma, &device->alloc,
+ VK_SYSTEM_ALLOCATION_SCOPE_DEVICE))
+ return NULL;
+
+ vk_pipeline_cache_object_init(device, &data_obj->base,
+ &raw_data_object_ops,
+ obj_key_data, key_size);
+ data_obj->data = obj_data;
+ data_obj->data_size = data_size;
+
+ memcpy(obj_key_data, key_data, key_size);
+ memcpy(obj_data, data, data_size);
+
+ return data_obj;
+}
+
+static bool
+object_keys_equal(const void *void_a, const void *void_b)
+{
+ const struct vk_pipeline_cache_object *a = void_a, *b = void_b;
+ if (a->key_size != b->key_size)
+ return false;
+
+ return memcmp(a->key_data, b->key_data, a->key_size) == 0;
+}
+
+static uint32_t
+object_key_hash(const void *void_object)
+{
+ const struct vk_pipeline_cache_object *object = void_object;
+ return _mesa_hash_data(object->key_data, object->key_size);
+}
+
+static void
+vk_pipeline_cache_lock(struct vk_pipeline_cache *cache)
+{
+
+ if (!(cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT))
+ simple_mtx_lock(&cache->lock);
+}
+
+static void
+vk_pipeline_cache_unlock(struct vk_pipeline_cache *cache)
+{
+ if (!(cache->flags & VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT_EXT))
+ simple_mtx_unlock(&cache->lock);
+}
+
+static void
+vk_pipeline_cache_remove_object(struct vk_pipeline_cache *cache,
+ uint32_t hash,
+ struct vk_pipeline_cache_object *object)
+{
+ vk_pipeline_cache_lock(cache);
+ struct set_entry *entry =
+ _mesa_set_search_pre_hashed(cache->object_cache, hash, object);
+ if (entry && entry->key == (const void *)object) {
+ /* Drop the reference owned by the cache */
+ vk_pipeline_cache_object_unref(object);
+
+ _mesa_set_remove(cache->object_cache, entry);
+ }
+ vk_pipeline_cache_unlock(cache);
+
+ /* Drop our reference */
+ vk_pipeline_cache_object_unref(object);
+}
+
+/* Consumes references to both search and replace and produces a reference */
+static struct vk_pipeline_cache_object *
+vk_pipeline_cache_replace_object(struct vk_pipeline_cache *cache,
+ uint32_t hash,
+ struct vk_pipeline_cache_object *search,
+ struct vk_pipeline_cache_object *replace)
+{
+ assert(object_keys_equal(search, replace));
+
+ vk_pipeline_cache_lock(cache);
+ struct set_entry *entry =
+ _mesa_set_search_pre_hashed(cache->object_cache, hash, search);
+
+ struct vk_pipeline_cache_object *found = NULL;
+ if (entry) {
+ if (entry->key == (const void *)search) {
+ /* Drop the reference owned by the cache */
+ vk_pipeline_cache_object_unref(search);
+
+ entry->key = vk_pipeline_cache_object_ref(replace);
+ } else {
+ found = vk_pipeline_cache_object_ref((void *)entry->key);
+ }
+ } else {
+ /* I guess the object was purged? Re-add it to the cache */
+ vk_pipeline_cache_object_ref(replace);
+ _mesa_set_add_pre_hashed(cache->object_cache, hash, replace);
+ }
+ vk_pipeline_cache_unlock(cache);
+
+ vk_pipeline_cache_object_unref(search);
+
+ if (found) {
+ vk_pipeline_cache_object_unref(replace);
+ return found;
+ } else {
+ return replace;
+ }
+}
+
+static bool
+vk_pipeline_cache_object_serialize(struct vk_pipeline_cache *cache,
+ struct vk_pipeline_cache_object *object,
+ struct blob *blob, uint32_t *data_size)
+{
+ if (object->ops->serialize == NULL)
+ return false;
+
+ assert(blob->size == align64(blob->size, VK_PIPELINE_CACHE_BLOB_ALIGN));
+ size_t start = blob->size;
+
+ /* Special case for if we're writing to a NULL blob (just to get the size)
+ * and we already know the data size of the allocation. This should make
+ * the first GetPipelineCacheData() call to get the data size faster in the
+ * common case where a bunch of our objects were loaded from a previous
+ * cache or where we've already serialized the cache once.
+ */
+ if (blob->data == NULL && blob->fixed_allocation) {
+ *data_size = p_atomic_read(&object->data_size);
+ if (*data_size > 0) {
+ blob_write_bytes(blob, NULL, *data_size);
+ return true;
+ }
+ }
+
+ if (!object->ops->serialize(object, blob)) {
+ vk_logw(VK_LOG_OBJS(cache),
+ "Failed to serialize pipeline cache object");
+ return false;
+ }
+
+ size_t size = blob->size - start;
+ if (size > UINT32_MAX) {
+ vk_logw(VK_LOG_OBJS(cache),
+ "Skipping giant (4 GiB or larger) object");
+ return false;
+ }
+
+ if (blob->out_of_memory) {
+ vk_logw(VK_LOG_OBJS(cache),
+ "Insufficient memory for pipeline cache data");
+ return false;
+ }
+
+ *data_size = (uint32_t)size;
+ p_atomic_set(&object->data_size, *data_size);
+
+ return true;
+}
+
+static struct vk_pipeline_cache_object *
+vk_pipeline_cache_object_deserialize(struct vk_pipeline_cache *cache,
+ const void *key_data, uint32_t key_size,
+ const void *data, size_t data_size,
+ const struct vk_pipeline_cache_object_ops *ops)
+{
+ if (unlikely(ops->deserialize == NULL)) {
+ vk_logw(VK_LOG_OBJS(cache),
+ "Pipeline cache object cannot be deserialized");
+ return NULL;
+ }
+
+ struct blob_reader reader;
+ blob_reader_init(&reader, data, data_size);
+
+ struct vk_pipeline_cache_object *object =
+ ops->deserialize(cache->base.device, key_data, key_size, &reader);
+
+ if (object == NULL) {
+ vk_logw(VK_LOG_OBJS(cache),
+ "Deserializing pipeline cache object failed");
+ return NULL;
+ }
+
+ assert(reader.current == reader.end && !reader.overrun);
+ assert(object->device == cache->base.device);
+ assert(object->ops == ops);
+ assert(object->ref_cnt == 1);
+ assert(object->key_size == key_size);
+ assert(memcmp(object->key_data, key_data, key_size) == 0);
+
+ return object;
+}
+
+struct vk_pipeline_cache_object *
+vk_pipeline_cache_lookup_object(struct vk_pipeline_cache *cache,
+ const void *key_data, size_t key_size,
+ const struct vk_pipeline_cache_object_ops *ops,
+ bool *cache_hit)
+{
+ assert(key_size <= UINT32_MAX);
+ assert(ops != NULL);
+
+ if (cache_hit != NULL)
+ *cache_hit = false;
+
+ struct vk_pipeline_cache_object key = {
+ .key_data = key_data,
+ .key_size = key_size,
+ };
+ uint32_t hash = object_key_hash(&key);
+
+ struct vk_pipeline_cache_object *object = NULL;
+
+ if (cache != NULL && cache->object_cache != NULL) {
+ vk_pipeline_cache_lock(cache);
+ struct set_entry *entry =
+ _mesa_set_search_pre_hashed(cache->object_cache, hash, &key);
+ if (entry) {
+ object = vk_pipeline_cache_object_ref((void *)entry->key);
+ if (cache_hit != NULL)
+ *cache_hit = true;
+ }
+ vk_pipeline_cache_unlock(cache);
+ }
+
+ if (object == NULL) {
+#ifdef ENABLE_SHADER_CACHE
+ struct disk_cache *disk_cache = cache->base.device->physical->disk_cache;
+ if (disk_cache != NULL) {
+ cache_key cache_key;
+ disk_cache_compute_key(disk_cache, key_data, key_size, cache_key);
+
+ size_t data_size;
+ uint8_t *data = disk_cache_get(disk_cache, cache_key, &data_size);
+ if (data) {
+ object = vk_pipeline_cache_object_deserialize(cache,
+ key_data, key_size,
+ data, data_size,
+ ops);
+ free(data);
+ if (object != NULL)
+ return vk_pipeline_cache_add_object(cache, object);
+ }
+ }
+#endif
+
+ /* No disk cache or not found in the disk cache */
+ return NULL;
+ }
+
+ if (object->ops == &raw_data_object_ops) {
+ /* The object isn't fully formed yet and we need to deserialize it into
+ * a real object before it can be used.
+ */
+ struct raw_data_object *data_obj =
+ container_of(object, struct raw_data_object, base);
+
+ assert(ops != &raw_data_object_ops);
+
+ struct vk_pipeline_cache_object *real_object =
+ vk_pipeline_cache_object_deserialize(cache,
+ data_obj->base.key_data,
+ data_obj->base.key_size,
+ data_obj->data,
+ data_obj->data_size, ops);
+ if (real_object == NULL) {
+ vk_pipeline_cache_remove_object(cache, hash, object);
+ return NULL;
+ }
+
+ object = vk_pipeline_cache_replace_object(cache, hash, object,
+ real_object);
+ }
+
+ assert(object->ops == ops);
+
+ return object;
+}
+
+struct vk_pipeline_cache_object *
+vk_pipeline_cache_add_object(struct vk_pipeline_cache *cache,
+ struct vk_pipeline_cache_object *object)
+{
+ assert(object->ops != NULL);
+
+ if (cache->object_cache == NULL)
+ return object;
+
+ uint32_t hash = object_key_hash(object);
+
+ vk_pipeline_cache_lock(cache);
+ bool found = false;
+ struct set_entry *entry =
+ _mesa_set_search_or_add_pre_hashed(cache->object_cache,
+ hash, object, &found);
+
+ struct vk_pipeline_cache_object *found_object = NULL;
+ if (found) {
+ found_object = vk_pipeline_cache_object_ref((void *)entry->key);
+ } else {
+ /* The cache now owns a reference */
+ vk_pipeline_cache_object_ref(object);
+ }
+ vk_pipeline_cache_unlock(cache);
+
+ if (found) {
+ vk_pipeline_cache_object_unref(object);
+ return found_object;
+ } else {
+ /* If it wasn't in the object cache, it might not be in the disk cache
+ * either. Better try and add it.
+ */
+
+#ifdef ENABLE_SHADER_CACHE
+ struct disk_cache *disk_cache = cache->base.device->physical->disk_cache;
+ if (object->ops->serialize != NULL && disk_cache) {
+ struct blob blob;
+ blob_init(&blob);
+
+ if (object->ops->serialize(object, &blob) && !blob.out_of_memory) {
+ cache_key cache_key;
+ disk_cache_compute_key(disk_cache, object->key_data,
+ object->key_size, cache_key);
+
+ disk_cache_put(disk_cache, cache_key, blob.data, blob.size, NULL);
+ }
+
+ blob_finish(&blob);
+ }
+#endif
+
+ return object;
+ }
+}
+
+static int32_t
+find_type_for_ops(const struct vk_physical_device *pdevice,
+ const struct vk_pipeline_cache_object_ops *ops)
+{
+ const struct vk_pipeline_cache_object_ops *const *import_ops =
+ pdevice->pipeline_cache_import_ops;
+
+ if (import_ops == NULL)
+ return -1;
+
+ for (int32_t i = 0; import_ops[i]; i++) {
+ if (import_ops[i] == ops)
+ return i;
+ }
+
+ return -1;
+}
+
+static const struct vk_pipeline_cache_object_ops *
+find_ops_for_type(const struct vk_physical_device *pdevice,
+ int32_t type)
+{
+ const struct vk_pipeline_cache_object_ops *const *import_ops =
+ pdevice->pipeline_cache_import_ops;
+
+ if (import_ops == NULL || type < 0)
+ return NULL;
+
+ return import_ops[type];
+}
+
+static void
+vk_pipeline_cache_load(struct vk_pipeline_cache *cache,
+ const void *data, size_t size)
+{
+ struct blob_reader blob;
+ blob_reader_init(&blob, data, size);
+
+ struct vk_pipeline_cache_header header;
+ blob_copy_bytes(&blob, &header, sizeof(header));
+ uint32_t count = blob_read_uint32(&blob);
+ if (blob.overrun)
+ return;
+
+ if (memcmp(&header, &cache->header, sizeof(header)) != 0)
+ return;
+
+ for (uint32_t i = 0; i < count; i++) {
+ int32_t type = blob_read_uint32(&blob);
+ uint32_t key_size = blob_read_uint32(&blob);
+ uint32_t data_size = blob_read_uint32(&blob);
+ const void *key_data = blob_read_bytes(&blob, key_size);
+ blob_reader_align(&blob, VK_PIPELINE_CACHE_BLOB_ALIGN);
+ const void *data = blob_read_bytes(&blob, data_size);
+ if (blob.overrun)
+ break;
+
+ const struct vk_pipeline_cache_object_ops *ops =
+ find_ops_for_type(cache->base.device->physical, type);
+
+ struct vk_pipeline_cache_object *object;
+ if (ops != NULL) {
+ object = vk_pipeline_cache_object_deserialize(cache,
+ key_data, key_size,
+ data, data_size, ops);
+ } else {
+ struct raw_data_object *data_obj =
+ raw_data_object_create(cache->base.device,
+ key_data, key_size,
+ data, data_size);
+ object = data_obj != NULL ? &data_obj->base : NULL;
+ }
+ if (object == NULL)
+ continue;
+
+ object = vk_pipeline_cache_add_object(cache, object);
+ vk_pipeline_cache_object_unref(object);
+ }
+}
+
+struct vk_pipeline_cache *
+vk_pipeline_cache_create(struct vk_device *device,
+ const struct vk_pipeline_cache_create_info *info,
+ const VkAllocationCallbacks *pAllocator)
+{
+ static const struct VkPipelineCacheCreateInfo default_create_info = {
+ .sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO,
+ };
+ struct vk_pipeline_cache *cache;
+
+ const struct VkPipelineCacheCreateInfo *pCreateInfo =
+ info->pCreateInfo != NULL ? info->pCreateInfo : &default_create_info;
+
+ assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO);
+ assert(pCreateInfo->flags == 0);
+
+ cache = vk_object_alloc(device, pAllocator, sizeof(*cache),
+ VK_OBJECT_TYPE_PIPELINE_CACHE);
+ if (cache == NULL)
+ return NULL;
+
+ cache->flags = pCreateInfo->flags;
+
+ struct VkPhysicalDeviceProperties pdevice_props;
+ device->physical->dispatch_table.GetPhysicalDeviceProperties(
+ vk_physical_device_to_handle(device->physical), &pdevice_props);
+
+ cache->header = (struct vk_pipeline_cache_header) {
+ .header_size = sizeof(struct vk_pipeline_cache_header),
+ .header_version = VK_PIPELINE_CACHE_HEADER_VERSION_ONE,
+ .vendor_id = pdevice_props.vendorID,
+ .device_id = pdevice_props.deviceID,
+ };
+ memcpy(cache->header.uuid, pdevice_props.pipelineCacheUUID, VK_UUID_SIZE);
+
+ simple_mtx_init(&cache->lock, mtx_plain);
+
+ if (info->force_enable ||
+ env_var_as_boolean("VK_ENABLE_PIPELINE_CACHE", true)) {
+ cache->object_cache = _mesa_set_create(NULL, object_key_hash,
+ object_keys_equal);
+ }
+
+ if (cache->object_cache && pCreateInfo->initialDataSize > 0) {
+ vk_pipeline_cache_load(cache, pCreateInfo->pInitialData,
+ pCreateInfo->initialDataSize);
+ }
+
+ return cache;
+}
+
+static void
+object_unref_cb(struct set_entry *entry)
+{
+ vk_pipeline_cache_object_unref((void *)entry->key);
+}
+
+void
+vk_pipeline_cache_destroy(struct vk_pipeline_cache *cache,
+ const VkAllocationCallbacks *pAllocator)
+{
+ if (cache->object_cache)
+ _mesa_set_destroy(cache->object_cache, object_unref_cb);
+ simple_mtx_destroy(&cache->lock);
+ vk_object_free(cache->base.device, pAllocator, cache);
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+vk_common_CreatePipelineCache(VkDevice _device,
+ const VkPipelineCacheCreateInfo *pCreateInfo,
+ const VkAllocationCallbacks *pAllocator,
+ VkPipelineCache *pPipelineCache)
+{
+ VK_FROM_HANDLE(vk_device, device, _device);
+ struct vk_pipeline_cache *cache;
+
+ struct vk_pipeline_cache_create_info info = {
+ .pCreateInfo = pCreateInfo,
+ };
+ cache = vk_pipeline_cache_create(device, &info, pAllocator);
+ if (cache == NULL)
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+
+ *pPipelineCache = vk_pipeline_cache_to_handle(cache);
+
+ return VK_SUCCESS;
+}
+
+VKAPI_ATTR void VKAPI_CALL
+vk_common_DestroyPipelineCache(VkDevice device,
+ VkPipelineCache pipelineCache,
+ const VkAllocationCallbacks *pAllocator)
+{
+ VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache);
+
+ if (cache == NULL)
+ return;
+
+ assert(cache->base.device == vk_device_from_handle(device));
+ vk_pipeline_cache_destroy(cache, pAllocator);
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+vk_common_GetPipelineCacheData(VkDevice _device,
+ VkPipelineCache pipelineCache,
+ size_t *pDataSize,
+ void *pData)
+{
+ VK_FROM_HANDLE(vk_device, device, _device);
+ VK_FROM_HANDLE(vk_pipeline_cache, cache, pipelineCache);
+
+ struct blob blob;
+ if (pData) {
+ blob_init_fixed(&blob, pData, *pDataSize);
+ } else {
+ blob_init_fixed(&blob, NULL, SIZE_MAX);
+ }
+
+ blob_write_bytes(&blob, &cache->header, sizeof(cache->header));
+
+ uint32_t count = 0;
+ intptr_t count_offset = blob_reserve_uint32(&blob);
+ if (count_offset < 0) {
+ *pDataSize = 0;
+ blob_finish(&blob);
+ return VK_INCOMPLETE;
+ }
+
+ vk_pipeline_cache_lock(cache);
+
+ VkResult result = VK_SUCCESS;
+ if (cache->object_cache != NULL) {
+ set_foreach(cache->object_cache, entry) {
+ struct vk_pipeline_cache_object *object = (void *)entry->key;
+
+ if (object->ops->serialize == NULL)
+ continue;
+
+ size_t blob_size_save = blob.size;
+
+ int32_t type = find_type_for_ops(device->physical, object->ops);
+ blob_write_uint32(&blob, type);
+ blob_write_uint32(&blob, object->key_size);
+ intptr_t data_size_resv = blob_reserve_uint32(&blob);
+ blob_write_bytes(&blob, object->key_data, object->key_size);
+
+ blob_align(&blob, VK_PIPELINE_CACHE_BLOB_ALIGN);
+
+ uint32_t data_size;
+ if (!vk_pipeline_cache_object_serialize(cache, object,
+ &blob, &data_size)) {
+ blob.size = blob_size_save;
+ if (blob.out_of_memory) {
+ result = VK_INCOMPLETE;
+ break;
+ }
+
+ /* Failed for some other reason; keep going */
+ continue;
+ }
+
+ /* vk_pipeline_cache_object_serialize should have failed */
+ assert(!blob.out_of_memory);
+
+ assert(data_size_resv >= 0);
+ blob_overwrite_uint32(&blob, data_size_resv, data_size);
+ }
+ }
+
+ vk_pipeline_cache_unlock(cache);
+
+ blob_overwrite_uint32(&blob, count_offset, count);
+
+ *pDataSize = blob.size;
+
+ blob_finish(&blob);
+
+ return result;
+}
+
+VKAPI_ATTR VkResult VKAPI_CALL
+vk_common_MergePipelineCaches(VkDevice device,
+ VkPipelineCache dstCache,
+ uint32_t srcCacheCount,
+ const VkPipelineCache *pSrcCaches)
+{
+ VK_FROM_HANDLE(vk_pipeline_cache, dst, dstCache);
+
+ if (!dst->object_cache)
+ return VK_SUCCESS;
+
+ vk_pipeline_cache_lock(dst);
+
+ for (uint32_t i = 0; i < srcCacheCount; i++) {
+ VK_FROM_HANDLE(vk_pipeline_cache, src, pSrcCaches[i]);
+
+ if (!src->object_cache)
+ continue;
+
+ assert(src != dst);
+ if (src == dst)
+ continue;
+
+ vk_pipeline_cache_lock(src);
+
+ set_foreach(src->object_cache, src_entry) {
+ struct vk_pipeline_cache_object *src_object = (void *)src_entry->key;
+
+ bool found_in_dst = false;
+ struct set_entry *dst_entry =
+ _mesa_set_search_or_add_pre_hashed(dst->object_cache,
+ src_entry->hash,
+ src_object, &found_in_dst);
+ if (found_in_dst) {
+ struct vk_pipeline_cache_object *dst_object = (void *)dst_entry->key;
+ if (dst_object->ops == &raw_data_object_ops &&
+ src_object->ops != &raw_data_object_ops) {
+ /* Even though dst has the object, it only has the blob version
+ * which isn't as useful. Replace it with the real object.
+ */
+ vk_pipeline_cache_object_unref(dst_object);
+ dst_entry->key = vk_pipeline_cache_object_ref(src_object);
+ }
+ } else {
+ /* We inserted src_object in dst so it needs a reference */
+ assert(dst_entry->key == (const void *)src_object);
+ vk_pipeline_cache_object_ref(src_object);
+ }
+ }
+
+ vk_pipeline_cache_unlock(src);
+ }
+
+ vk_pipeline_cache_unlock(dst);
+
+ return VK_SUCCESS;
+}
--- /dev/null
+/*
+ * Copyright © 2021 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+#ifndef VK_PIPELINE_CACHE_H
+#define VK_PIPELINE_CACHE_H
+
+#include "vk_object.h"
+#include "vk_util.h"
+
+#include "util/simple_mtx.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* #include "util/blob.h" */
+struct blob;
+struct blob_reader;
+
+/* #include "util/set.h" */
+struct set;
+
+struct vk_pipeline_cache;
+struct vk_pipeline_cache_object;
+
+#define VK_PIPELINE_CACHE_BLOB_ALIGN 8
+
+struct vk_pipeline_cache_object_ops {
+ /** Writes this cache object to the given blob
+ *
+ * Because the cache works with both raw blob data and driver object data
+ * and can't always tell the difference between the two, we have to be very
+ * careful about alignments when [de]serializing. When serialize() is
+ * called, the blob will be aligned to VK_PIPELINE_CACHE_BLOB_ALIGN. The
+ * driver must be careful to not [de]serialize any data types which require
+ * a higher alignment. When deserialize() is called, the blob_reader is
+ * also guaranteed to be aligned to VK_PIPELINE_CACHE_BLOB_ALIGN.
+ *
+ * Returns true on success
+ *
+ * This function is optional. Objects without [de]serialization support
+ * will still be cached in memory but will not be placed in the disk cache
+ * and will not exported to the client when vkGetPipelineCacheData() is
+ * called.
+ */
+ bool (*serialize)(struct vk_pipeline_cache_object *object,
+ struct blob *blob);
+
+ /** Constructs an object from cached data
+ *
+ * See serialize() for details about data alignment.
+ *
+ * returns the created object
+ *
+ * This function is optional.
+ */
+ struct vk_pipeline_cache_object *(*deserialize)(struct vk_device *device,
+ const void *key_data,
+ size_t key_size,
+ struct blob_reader *blob);
+
+ /** Destroys the object
+ *
+ * Called when vk_pipeline_cache_object.ref_cnt hits 0.
+ */
+ void (*destroy)(struct vk_pipeline_cache_object *object);
+};
+
+/** Base struct for cached objects
+ *
+ * A vk_pipeline_cache stores any number of vk_pipeline_cache_object's, each
+ * of which has an associated key of arbitrary size. Cached objects are
+ * reference counted so that they can exist in multiple caches (for example,
+ * when vkMergePipelineCaches() is called) and so that they can persist after
+ * the pipeline cache is destroyed. Each object also has a pointer to a
+ * vk_pipeline_cache_object_ops table which the pipeline cache uses to
+ * [de]serialize the object and clean it up when the reference count hits 0.
+ *
+ * The rest of the details of any given object are entirely up to the driver.
+ * The driver may even have multiple types of objects (distinguished by their
+ * vk_pipeline_cache_object_ops table) in the cache so long as it guarantees
+ * it never has two objects of different types with the same key.
+ */
+struct vk_pipeline_cache_object {
+ struct vk_device *device;
+ const struct vk_pipeline_cache_object_ops *ops;
+ uint32_t ref_cnt;
+
+ uint32_t data_size;
+ const void *key_data;
+ uint32_t key_size;
+};
+
+static inline void
+vk_pipeline_cache_object_init(struct vk_device *device,
+ struct vk_pipeline_cache_object *object,
+ const struct vk_pipeline_cache_object_ops *ops,
+ const void *key_data, uint32_t key_size)
+{
+ object->device = device;
+ object->ops = ops;
+ p_atomic_set(&object->ref_cnt, 1);
+ object->data_size = 0; /* Unknown */
+ object->key_data = key_data;
+ object->key_size = key_size;
+}
+
+static inline void
+vk_pipeline_cache_object_finish(struct vk_pipeline_cache_object *object)
+{
+ assert(p_atomic_read(&object->ref_cnt) <= 1);
+}
+
+static inline struct vk_pipeline_cache_object *
+vk_pipeline_cache_object_ref(struct vk_pipeline_cache_object *object)
+{
+ assert(object && p_atomic_read(&object->ref_cnt) >= 1);
+ p_atomic_inc(&object->ref_cnt);
+ return object;
+}
+
+static inline void
+vk_pipeline_cache_object_unref(struct vk_pipeline_cache_object *object)
+{
+ assert(object && p_atomic_read(&object->ref_cnt) >= 1);
+ if (p_atomic_dec_zero(&object->ref_cnt))
+ object->ops->destroy(object);
+}
+
+/** A generic implementation of VkPipelineCache */
+struct vk_pipeline_cache {
+ struct vk_object_base base;
+
+ /* pCreateInfo::flags */
+ VkPipelineCacheCreateFlags flags;
+
+ struct vk_pipeline_cache_header header;
+
+ /** Protects object_cache */
+ simple_mtx_t lock;
+
+ struct set *object_cache;
+};
+
+VK_DEFINE_NONDISP_HANDLE_CASTS(vk_pipeline_cache, base, VkPipelineCache,
+ VK_OBJECT_TYPE_PIPELINE_CACHE)
+
+struct vk_pipeline_cache_create_info {
+ /* The pCreateInfo for this pipeline cache, if any.
+ *
+ * For driver-internal caches, this is allowed to be NULL.
+ */
+ const VkPipelineCacheCreateInfo *pCreateInfo;
+
+ /** If true, ignore VK_ENABLE_PIPELINE_CACHE and enable anyway */
+ bool force_enable;
+};
+
+struct vk_pipeline_cache *
+vk_pipeline_cache_create(struct vk_device *device,
+ const struct vk_pipeline_cache_create_info *info,
+ const VkAllocationCallbacks *pAllocator);
+void
+vk_pipeline_cache_destroy(struct vk_pipeline_cache *cache,
+ const VkAllocationCallbacks *pAllocator);
+
+/** Attempts to look up an object in the cache by key
+ *
+ * If an object is found in the cache matching the given key, *cache_hit is
+ * set to true and a reference to that object is returned.
+ *
+ * If the driver sets vk_device.disk_cache, we attempt to look up any missing
+ * objects in the disk cache before declaring failure. If an object is found
+ * in the disk cache but not the in-memory cache, *cache_hit is set to false.
+ *
+ * The deserialization of pipeline cache objects found in the cache data
+ * provided via VkPipelineCacheCreateInfo::pInitialData happens during
+ * vk_pipeline_cache_lookup() rather than during vkCreatePipelineCache().
+ * Prior to the first vk_pipeline_cache_lookup() of a given object, it is
+ * stored as an internal raw data object with the same hash. This allows us
+ * to avoid any complex object type tagging in the serialized cache. It does,
+ * however, mean that drivers need to be careful to ensure that objects with
+ * different types (ops) have different keys.
+ *
+ * Returns a reference to the object, if found
+ */
+struct vk_pipeline_cache_object * MUST_CHECK
+vk_pipeline_cache_lookup_object(struct vk_pipeline_cache *cache,
+ const void *key_data, size_t key_size,
+ const struct vk_pipeline_cache_object_ops *ops,
+ bool *cache_hit);
+
+/** Adds an object to the pipeline cache
+ *
+ * This function adds the given object to the pipeline cache. We do not
+ * specify a key here because the key is part of the object. See also
+ * vk_pipeline_cache_object_init().
+ *
+ * This function consumes a reference to the object and returns a reference to
+ * the (possibly different) object in the cache. The intended usage pattern
+ * is as follows:
+ *
+ * key = compute_key();
+ * struct vk_pipeline_cache_object *object =
+ * vk_pipeline_cache_lookup_object(cache, &key, sizeof(key),
+ * &driver_type_ops, &cache_hit);
+ * if (object != NULL)
+ * return container_of(object, driver_type, base);
+ *
+ * object = do_compile();
+ * assert(object != NULL);
+ *
+ * object = vk_pipeline_cache_add_object(cache, object);
+ * return container_of(object, driver_type, base);
+ */
+struct vk_pipeline_cache_object * MUST_CHECK
+vk_pipeline_cache_add_object(struct vk_pipeline_cache *cache,
+ struct vk_pipeline_cache_object *object);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* VK_PIPELINE_CACHE_H */