From 644fcd94864260218037d0b9a0dfd3b00be073d8 Mon Sep 17 00:00:00 2001 From: Timothy Arceri Date: Thu, 1 Oct 2020 20:02:59 +1000 Subject: [PATCH] util/disk_cache: make use of single file cache when env var set When the MESA_DISK_CACHE_SINGLE_FILE environment variable is set we make use of the new single file shader cache implementation. The new cache uses the following directory structure based on the first defined name as follows: $MESA_GLSL_CACHE_DIR/driver_id/gpu_name/foz_cache.foz $MESA_GLSL_CACHE_DIR/driver_id/gpu_name/foz_cache_idx.foz $XDG_CACHE_HOME/mesa_shader_cache_sf/driver_id/gpu_name/foz_cache.foz $XDG_CACHE_HOME/mesa_shader_cache_sf/driver_id/gpu_name/foz_cache_idx.foz /.cache/mesa_shader_cache_sf/driver_id/gpu_name/foz_cache.foz /.cache/mesa_shader_cache_sf/driver_id/gpu_name/foz_cache_idx.foz Where foz_cache_idx.foz is a database of offsets pointing to the location of the shader cache entries in foz_cache.foz This initial implementation doesn't have any max cache size handling and is initially intended to be use by applications such as steam that will handle cache management for us. Acked-by: Pierre-Eric Pelloux-Prayer Part-of: --- src/util/disk_cache.c | 51 ++++++++++++++++++++++++++++++++---------------- src/util/disk_cache_os.c | 39 +++++++++++++++++++++++++++++++++++- src/util/disk_cache_os.h | 18 ++++++++++++++++- 3 files changed, 89 insertions(+), 19 deletions(-) diff --git a/src/util/disk_cache.c b/src/util/disk_cache.c index b568b82..3617fc1 100644 --- a/src/util/disk_cache.c +++ b/src/util/disk_cache.c @@ -103,7 +103,7 @@ disk_cache_create(const char *gpu_name, const char *driver_id, goto path_fail; #endif - char *path = disk_cache_generate_cache_dir(local); + char *path = disk_cache_generate_cache_dir(local, gpu_name, driver_id); if (!path) goto path_fail; @@ -111,6 +111,11 @@ disk_cache_create(const char *gpu_name, const char *driver_id, if (cache->path == NULL) goto path_fail; + if (env_var_as_boolean("MESA_DISK_CACHE_SINGLE_FILE", false)) { + if (!disk_cache_load_cache_index(local, cache, path)) + goto path_fail; + } + if (!disk_cache_mmap_cache_index(local, cache, path)) goto path_fail; @@ -227,6 +232,10 @@ disk_cache_destroy(struct disk_cache *cache) if (cache && !cache->path_init_failed) { util_queue_finish(&cache->cache_queue); util_queue_destroy(&cache->cache_queue); + + if (env_var_as_boolean("MESA_DISK_CACHE_SINGLE_FILE", false)) + foz_destroy(&cache->foz_db); + disk_cache_destroy_mmap(cache); } @@ -315,21 +324,25 @@ cache_put(void *job, int thread_index) char *filename = NULL; struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *) job; - filename = disk_cache_get_cache_filename(dc_job->cache, dc_job->key); - if (filename == NULL) - goto done; - - /* If the cache is too large, evict something else first. */ - while (*dc_job->cache->size + dc_job->size > dc_job->cache->max_size && - i < 8) { - disk_cache_evict_lru_item(dc_job->cache); - i++; - } + if (env_var_as_boolean("MESA_DISK_CACHE_SINGLE_FILE", false)) { + disk_cache_write_item_to_disk_foz(dc_job); + } else { + filename = disk_cache_get_cache_filename(dc_job->cache, dc_job->key); + if (filename == NULL) + goto done; + + /* If the cache is too large, evict something else first. */ + while (*dc_job->cache->size + dc_job->size > dc_job->cache->max_size && + i < 8) { + disk_cache_evict_lru_item(dc_job->cache); + i++; + } - disk_cache_write_item_to_disk(dc_job, filename); + disk_cache_write_item_to_disk(dc_job, filename); done: - free(filename); + free(filename); + } } void @@ -383,11 +396,15 @@ disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size) return blob; } - char *filename = disk_cache_get_cache_filename(cache, key); - if (filename == NULL) - return NULL; + if (env_var_as_boolean("MESA_DISK_CACHE_SINGLE_FILE", false)) { + return disk_cache_load_item_foz(cache, key, size); + } else { + char *filename = disk_cache_get_cache_filename(cache, key); + if (filename == NULL) + return NULL; - return disk_cache_load_item(cache, filename, size); + return disk_cache_load_item(cache, filename, size); + } } void diff --git a/src/util/disk_cache_os.c b/src/util/disk_cache_os.c index 9e59644..865ff7b 100644 --- a/src/util/disk_cache_os.c +++ b/src/util/disk_cache_os.c @@ -841,7 +841,8 @@ disk_cache_write_item_to_disk(struct disk_cache_put_job *dc_job, * /.cache/mesa_shader_cache */ char * -disk_cache_generate_cache_dir(void *mem_ctx) +disk_cache_generate_cache_dir(void *mem_ctx, const char *gpu_name, + const char *driver_id) { char *cache_dir_name = CACHE_DIR_NAME; if (env_var_as_boolean("MESA_DISK_CACHE_SINGLE_FILE", false)) @@ -905,6 +906,16 @@ disk_cache_generate_cache_dir(void *mem_ctx) return NULL; } + if (env_var_as_boolean("MESA_DISK_CACHE_SINGLE_FILE", false)) { + path = concatenate_and_mkdir(mem_ctx, path, driver_id); + if (!path) + return NULL; + + path = concatenate_and_mkdir(mem_ctx, path, gpu_name); + if (!path) + return NULL; + } + return path; } @@ -927,6 +938,32 @@ disk_cache_enabled() return true; } +void * +disk_cache_load_item_foz(struct disk_cache *cache, const cache_key key, + size_t *size) +{ + return foz_read_entry(&cache->foz_db, key, size); +} + +bool +disk_cache_write_item_to_disk_foz(struct disk_cache_put_job *dc_job) +{ + return foz_write_entry(&dc_job->cache->foz_db, dc_job->key, dc_job->data, + dc_job->size); +} + +bool +disk_cache_load_cache_index(void *mem_ctx, struct disk_cache *cache, + char *path) +{ + path = ralloc_asprintf(mem_ctx, "%s/foz_cache", cache->path); + if (path == NULL) + return false; + + /* Load cache index into a hash map (from fossilise files) */ + return foz_prepare(&cache->foz_db, path); +} + bool disk_cache_mmap_cache_index(void *mem_ctx, struct disk_cache *cache, char *path) diff --git a/src/util/disk_cache_os.h b/src/util/disk_cache_os.h index 7ac6a51..69fd0b7 100644 --- a/src/util/disk_cache_os.h +++ b/src/util/disk_cache_os.h @@ -32,6 +32,8 @@ #else +#include "util/fossilize_db.h" + /* Number of bits to mask off from a cache key to get an index. */ #define CACHE_INDEX_KEY_BITS 16 @@ -49,6 +51,8 @@ struct disk_cache { /* Thread queue for compressing and writing cache entries to disk */ struct util_queue cache_queue; + struct foz_db foz_db; + /* Seed for rand, which is used to pick a random directory */ uint64_t seed_xorshift128plus[2]; @@ -90,7 +94,8 @@ struct disk_cache_put_job { }; char * -disk_cache_generate_cache_dir(void *mem_ctx); +disk_cache_generate_cache_dir(void *mem_ctx, const char *gpu_name, + const char *driver_id); void disk_cache_evict_lru_item(struct disk_cache *cache); @@ -99,11 +104,18 @@ void disk_cache_evict_item(struct disk_cache *cache, char *filename); void * +disk_cache_load_item_foz(struct disk_cache *cache, const cache_key key, + size_t *size); + +void * disk_cache_load_item(struct disk_cache *cache, char *filename, size_t *size); char * disk_cache_get_cache_filename(struct disk_cache *cache, const cache_key key); +bool +disk_cache_write_item_to_disk_foz(struct disk_cache_put_job *dc_job); + void disk_cache_write_item_to_disk(struct disk_cache_put_job *dc_job, char *filename); @@ -112,6 +124,10 @@ bool disk_cache_enabled(void); bool +disk_cache_load_cache_index(void *mem_ctx, struct disk_cache *cache, + char *path); + +bool disk_cache_mmap_cache_index(void *mem_ctx, struct disk_cache *cache, char *path); -- 2.7.4