• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright © 2014 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  */
23 
24 #ifdef ENABLE_SHADER_CACHE
25 
26 #include <ctype.h>
27 #include <ftw.h>
28 #include <string.h>
29 #include <stdlib.h>
30 #include <stdio.h>
31 #include <sys/file.h>
32 #include <sys/types.h>
33 #include <sys/stat.h>
34 #include <sys/mman.h>
35 #include <fcntl.h>
36 #include <errno.h>
37 #include <dirent.h>
38 #include <inttypes.h>
39 
40 #include "util/crc32.h"
41 #include "util/debug.h"
42 #include "util/rand_xor.h"
43 #include "util/u_atomic.h"
44 #include "util/mesa-sha1.h"
45 #include "util/ralloc.h"
46 #include "util/compiler.h"
47 
48 #include "disk_cache.h"
49 #include "disk_cache_os.h"
50 
51 /* The cache version should be bumped whenever a change is made to the
52  * structure of cache entries or the index. This will give any 3rd party
53  * applications reading the cache entries a chance to adjust to the changes.
54  *
55  * - The cache version is checked internally when reading a cache entry. If we
56  *   ever have a mismatch we are in big trouble as this means we had a cache
57  *   collision. In case of such an event please check the skys for giant
58  *   asteroids and that the entire Mesa team hasn't been eaten by wolves.
59  *
60  * - There is no strict requirement that cache versions be backwards
61  *   compatible but effort should be taken to limit disruption where possible.
62  */
63 #define CACHE_VERSION 1
64 
65 #define DRV_KEY_CPY(_dst, _src, _src_size) \
66 do {                                       \
67    memcpy(_dst, _src, _src_size);          \
68    _dst += _src_size;                      \
69 } while (0);
70 
71 struct disk_cache *
disk_cache_create(const char * gpu_name,const char * driver_id,uint64_t driver_flags)72 disk_cache_create(const char *gpu_name, const char *driver_id,
73                   uint64_t driver_flags)
74 {
75    void *local;
76    struct disk_cache *cache = NULL;
77    char *max_size_str;
78    uint64_t max_size;
79 
80    uint8_t cache_version = CACHE_VERSION;
81    size_t cv_size = sizeof(cache_version);
82 
83    if (!disk_cache_enabled())
84       return NULL;
85 
86    /* A ralloc context for transient data during this invocation. */
87    local = ralloc_context(NULL);
88    if (local == NULL)
89       goto fail;
90 
91    cache = rzalloc(NULL, struct disk_cache);
92    if (cache == NULL)
93       goto fail;
94 
95    /* Assume failure. */
96    cache->path_init_failed = true;
97 
98 #ifdef ANDROID
99    /* Android needs the "disk cache" to be enabled for
100     * EGL_ANDROID_blob_cache's callbacks to be called, but it doesn't actually
101     * want any storing to disk to happen inside of the driver.
102     */
103    goto path_fail;
104 #endif
105 
106    char *path = disk_cache_generate_cache_dir(local, gpu_name, driver_id);
107    if (!path)
108       goto path_fail;
109 
110    cache->path = ralloc_strdup(cache, path);
111    if (cache->path == NULL)
112       goto path_fail;
113 
114    if (env_var_as_boolean("MESA_DISK_CACHE_SINGLE_FILE", false)) {
115       if (!disk_cache_load_cache_index(local, cache))
116          goto path_fail;
117    }
118 
119    if (!disk_cache_mmap_cache_index(local, cache, path))
120       goto path_fail;
121 
122    max_size = 0;
123 
124    max_size_str = getenv("MESA_SHADER_CACHE_MAX_SIZE");
125 
126    if (!max_size_str) {
127       max_size_str = getenv("MESA_GLSL_CACHE_MAX_SIZE");
128       if (max_size_str)
129          fprintf(stderr,
130                  "*** MESA_GLSL_CACHE_MAX_SIZE is deprecated; "
131                  "use MESA_SHADER_CACHE_MAX_SIZE instead ***\n");
132    }
133 
134    #ifdef MESA_SHADER_CACHE_MAX_SIZE
135    if( !max_size_str ) {
136       max_size_str = MESA_SHADER_CACHE_MAX_SIZE;
137    }
138    #endif
139 
140    if (max_size_str) {
141       char *end;
142       max_size = strtoul(max_size_str, &end, 10);
143       if (end == max_size_str) {
144          max_size = 0;
145       } else {
146          switch (*end) {
147          case 'K':
148          case 'k':
149             max_size *= 1024;
150             break;
151          case 'M':
152          case 'm':
153             max_size *= 1024*1024;
154             break;
155          case '\0':
156          case 'G':
157          case 'g':
158          default:
159             max_size *= 1024*1024*1024;
160             break;
161          }
162       }
163    }
164 
165    /* Default to 1GB for maximum cache size. */
166    if (max_size == 0) {
167       max_size = 1024*1024*1024;
168    }
169 
170    cache->max_size = max_size;
171 
172    /* 4 threads were chosen below because just about all modern CPUs currently
173     * available that run Mesa have *at least* 4 cores. For these CPUs allowing
174     * more threads can result in the queue being processed faster, thus
175     * avoiding excessive memory use due to a backlog of cache entrys building
176     * up in the queue. Since we set the UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY
177     * flag this should have little negative impact on low core systems.
178     *
179     * The queue will resize automatically when it's full, so adding new jobs
180     * doesn't stall.
181     */
182    if (!util_queue_init(&cache->cache_queue, "disk$", 32, 4,
183                         UTIL_QUEUE_INIT_SCALE_THREADS |
184                         UTIL_QUEUE_INIT_RESIZE_IF_FULL |
185                         UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY |
186                         UTIL_QUEUE_INIT_SET_FULL_THREAD_AFFINITY, NULL))
187       goto fail;
188 
189    cache->path_init_failed = false;
190 
191  path_fail:
192 
193    cache->driver_keys_blob_size = cv_size;
194 
195    /* Create driver id keys */
196    size_t id_size = strlen(driver_id) + 1;
197    size_t gpu_name_size = strlen(gpu_name) + 1;
198    cache->driver_keys_blob_size += id_size;
199    cache->driver_keys_blob_size += gpu_name_size;
200 
201    /* We sometimes store entire structs that contains a pointers in the cache,
202     * use pointer size as a key to avoid hard to debug issues.
203     */
204    uint8_t ptr_size = sizeof(void *);
205    size_t ptr_size_size = sizeof(ptr_size);
206    cache->driver_keys_blob_size += ptr_size_size;
207 
208    size_t driver_flags_size = sizeof(driver_flags);
209    cache->driver_keys_blob_size += driver_flags_size;
210 
211    cache->driver_keys_blob =
212       ralloc_size(cache, cache->driver_keys_blob_size);
213    if (!cache->driver_keys_blob)
214       goto fail;
215 
216    uint8_t *drv_key_blob = cache->driver_keys_blob;
217    DRV_KEY_CPY(drv_key_blob, &cache_version, cv_size)
218    DRV_KEY_CPY(drv_key_blob, driver_id, id_size)
219    DRV_KEY_CPY(drv_key_blob, gpu_name, gpu_name_size)
220    DRV_KEY_CPY(drv_key_blob, &ptr_size, ptr_size_size)
221    DRV_KEY_CPY(drv_key_blob, &driver_flags, driver_flags_size)
222 
223    /* Seed our rand function */
224    s_rand_xorshift128plus(cache->seed_xorshift128plus, true);
225 
226    ralloc_free(local);
227 
228    return cache;
229 
230  fail:
231    if (cache)
232       ralloc_free(cache);
233    ralloc_free(local);
234 
235    return NULL;
236 }
237 
238 void
disk_cache_destroy(struct disk_cache * cache)239 disk_cache_destroy(struct disk_cache *cache)
240 {
241    if (cache && !cache->path_init_failed) {
242       util_queue_finish(&cache->cache_queue);
243       util_queue_destroy(&cache->cache_queue);
244 
245       if (env_var_as_boolean("MESA_DISK_CACHE_SINGLE_FILE", false))
246          foz_destroy(&cache->foz_db);
247 
248       disk_cache_destroy_mmap(cache);
249    }
250 
251    ralloc_free(cache);
252 }
253 
254 void
disk_cache_wait_for_idle(struct disk_cache * cache)255 disk_cache_wait_for_idle(struct disk_cache *cache)
256 {
257    util_queue_finish(&cache->cache_queue);
258 }
259 
260 void
disk_cache_remove(struct disk_cache * cache,const cache_key key)261 disk_cache_remove(struct disk_cache *cache, const cache_key key)
262 {
263    char *filename = disk_cache_get_cache_filename(cache, key);
264    if (filename == NULL) {
265       return;
266    }
267 
268    disk_cache_evict_item(cache, filename);
269 }
270 
271 static struct disk_cache_put_job *
create_put_job(struct disk_cache * cache,const cache_key key,void * data,size_t size,struct cache_item_metadata * cache_item_metadata,bool take_ownership)272 create_put_job(struct disk_cache *cache, const cache_key key,
273                void *data, size_t size,
274                struct cache_item_metadata *cache_item_metadata,
275                bool take_ownership)
276 {
277    struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *)
278       malloc(sizeof(struct disk_cache_put_job) + (take_ownership ? 0 : size));
279 
280    if (dc_job) {
281       dc_job->cache = cache;
282       memcpy(dc_job->key, key, sizeof(cache_key));
283       if (take_ownership) {
284          dc_job->data = data;
285       } else {
286          dc_job->data = dc_job + 1;
287          memcpy(dc_job->data, data, size);
288       }
289       dc_job->size = size;
290 
291       /* Copy the cache item metadata */
292       if (cache_item_metadata) {
293          dc_job->cache_item_metadata.type = cache_item_metadata->type;
294          if (cache_item_metadata->type == CACHE_ITEM_TYPE_GLSL) {
295             dc_job->cache_item_metadata.num_keys =
296                cache_item_metadata->num_keys;
297             dc_job->cache_item_metadata.keys = (cache_key *)
298                malloc(cache_item_metadata->num_keys * sizeof(cache_key));
299 
300             if (!dc_job->cache_item_metadata.keys)
301                goto fail;
302 
303             memcpy(dc_job->cache_item_metadata.keys,
304                    cache_item_metadata->keys,
305                    sizeof(cache_key) * cache_item_metadata->num_keys);
306          }
307       } else {
308          dc_job->cache_item_metadata.type = CACHE_ITEM_TYPE_UNKNOWN;
309          dc_job->cache_item_metadata.keys = NULL;
310       }
311    }
312 
313    return dc_job;
314 
315 fail:
316    free(dc_job);
317 
318    return NULL;
319 }
320 
321 static void
destroy_put_job(void * job,void * gdata,int thread_index)322 destroy_put_job(void *job, void *gdata, int thread_index)
323 {
324    if (job) {
325       struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *) job;
326       free(dc_job->cache_item_metadata.keys);
327       free(job);
328    }
329 }
330 
331 static void
destroy_put_job_nocopy(void * job,void * gdata,int thread_index)332 destroy_put_job_nocopy(void *job, void *gdata, int thread_index)
333 {
334    struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *) job;
335    free(dc_job->data);
336    destroy_put_job(job, gdata, thread_index);
337 }
338 
339 static void
cache_put(void * job,void * gdata,int thread_index)340 cache_put(void *job, void *gdata, int thread_index)
341 {
342    assert(job);
343 
344    unsigned i = 0;
345    char *filename = NULL;
346    struct disk_cache_put_job *dc_job = (struct disk_cache_put_job *) job;
347 
348    if (env_var_as_boolean("MESA_DISK_CACHE_SINGLE_FILE", false)) {
349       disk_cache_write_item_to_disk_foz(dc_job);
350    } else {
351       filename = disk_cache_get_cache_filename(dc_job->cache, dc_job->key);
352       if (filename == NULL)
353          goto done;
354 
355       /* If the cache is too large, evict something else first. */
356       while (*dc_job->cache->size + dc_job->size > dc_job->cache->max_size &&
357              i < 8) {
358          disk_cache_evict_lru_item(dc_job->cache);
359          i++;
360       }
361 
362       disk_cache_write_item_to_disk(dc_job, filename);
363 
364 done:
365       free(filename);
366    }
367 }
368 
369 void
disk_cache_put(struct disk_cache * cache,const cache_key key,const void * data,size_t size,struct cache_item_metadata * cache_item_metadata)370 disk_cache_put(struct disk_cache *cache, const cache_key key,
371                const void *data, size_t size,
372                struct cache_item_metadata *cache_item_metadata)
373 {
374    if (cache->blob_put_cb) {
375       cache->blob_put_cb(key, CACHE_KEY_SIZE, data, size);
376       return;
377    }
378 
379    if (cache->path_init_failed)
380       return;
381 
382    struct disk_cache_put_job *dc_job =
383       create_put_job(cache, key, (void*)data, size, cache_item_metadata, false);
384 
385    if (dc_job) {
386       util_queue_fence_init(&dc_job->fence);
387       util_queue_add_job(&cache->cache_queue, dc_job, &dc_job->fence,
388                          cache_put, destroy_put_job, dc_job->size);
389    }
390 }
391 
392 void
disk_cache_put_nocopy(struct disk_cache * cache,const cache_key key,void * data,size_t size,struct cache_item_metadata * cache_item_metadata)393 disk_cache_put_nocopy(struct disk_cache *cache, const cache_key key,
394                       void *data, size_t size,
395                       struct cache_item_metadata *cache_item_metadata)
396 {
397    if (cache->blob_put_cb) {
398       cache->blob_put_cb(key, CACHE_KEY_SIZE, data, size);
399       free(data);
400       return;
401    }
402 
403    if (cache->path_init_failed) {
404       free(data);
405       return;
406    }
407 
408    struct disk_cache_put_job *dc_job =
409       create_put_job(cache, key, data, size, cache_item_metadata, true);
410 
411    if (dc_job) {
412       util_queue_fence_init(&dc_job->fence);
413       util_queue_add_job(&cache->cache_queue, dc_job, &dc_job->fence,
414                          cache_put, destroy_put_job_nocopy, dc_job->size);
415    }
416 }
417 
418 void *
disk_cache_get(struct disk_cache * cache,const cache_key key,size_t * size)419 disk_cache_get(struct disk_cache *cache, const cache_key key, size_t *size)
420 {
421    if (size)
422       *size = 0;
423 
424    if (cache->blob_get_cb) {
425       /* This is what Android EGL defines as the maxValueSize in egl_cache_t
426        * class implementation.
427        */
428       const signed long max_blob_size = 64 * 1024;
429       void *blob = malloc(max_blob_size);
430       if (!blob)
431          return NULL;
432 
433       signed long bytes =
434          cache->blob_get_cb(key, CACHE_KEY_SIZE, blob, max_blob_size);
435 
436       if (!bytes) {
437          free(blob);
438          return NULL;
439       }
440 
441       if (size)
442          *size = bytes;
443       return blob;
444    }
445 
446    if (env_var_as_boolean("MESA_DISK_CACHE_SINGLE_FILE", false)) {
447       return disk_cache_load_item_foz(cache, key, size);
448    } else {
449       char *filename = disk_cache_get_cache_filename(cache, key);
450       if (filename == NULL)
451          return NULL;
452 
453       return disk_cache_load_item(cache, filename, size);
454    }
455 }
456 
457 void
disk_cache_put_key(struct disk_cache * cache,const cache_key key)458 disk_cache_put_key(struct disk_cache *cache, const cache_key key)
459 {
460    const uint32_t *key_chunk = (const uint32_t *) key;
461    int i = CPU_TO_LE32(*key_chunk) & CACHE_INDEX_KEY_MASK;
462    unsigned char *entry;
463 
464    if (cache->blob_put_cb) {
465       cache->blob_put_cb(key, CACHE_KEY_SIZE, key_chunk, sizeof(uint32_t));
466       return;
467    }
468 
469    if (cache->path_init_failed)
470       return;
471 
472    entry = &cache->stored_keys[i * CACHE_KEY_SIZE];
473 
474    memcpy(entry, key, CACHE_KEY_SIZE);
475 }
476 
477 /* This function lets us test whether a given key was previously
478  * stored in the cache with disk_cache_put_key(). The implement is
479  * efficient by not using syscalls or hitting the disk. It's not
480  * race-free, but the races are benign. If we race with someone else
481  * calling disk_cache_put_key, then that's just an extra cache miss and an
482  * extra recompile.
483  */
484 bool
disk_cache_has_key(struct disk_cache * cache,const cache_key key)485 disk_cache_has_key(struct disk_cache *cache, const cache_key key)
486 {
487    const uint32_t *key_chunk = (const uint32_t *) key;
488    int i = CPU_TO_LE32(*key_chunk) & CACHE_INDEX_KEY_MASK;
489    unsigned char *entry;
490 
491    if (cache->blob_get_cb) {
492       uint32_t blob;
493       return cache->blob_get_cb(key, CACHE_KEY_SIZE, &blob, sizeof(uint32_t));
494    }
495 
496    if (cache->path_init_failed)
497       return false;
498 
499    entry = &cache->stored_keys[i * CACHE_KEY_SIZE];
500 
501    return memcmp(entry, key, CACHE_KEY_SIZE) == 0;
502 }
503 
504 void
disk_cache_compute_key(struct disk_cache * cache,const void * data,size_t size,cache_key key)505 disk_cache_compute_key(struct disk_cache *cache, const void *data, size_t size,
506                        cache_key key)
507 {
508    struct mesa_sha1 ctx;
509 
510    _mesa_sha1_init(&ctx);
511    _mesa_sha1_update(&ctx, cache->driver_keys_blob,
512                      cache->driver_keys_blob_size);
513    _mesa_sha1_update(&ctx, data, size);
514    _mesa_sha1_final(&ctx, key);
515 }
516 
517 void
disk_cache_set_callbacks(struct disk_cache * cache,disk_cache_put_cb put,disk_cache_get_cb get)518 disk_cache_set_callbacks(struct disk_cache *cache, disk_cache_put_cb put,
519                          disk_cache_get_cb get)
520 {
521    cache->blob_put_cb = put;
522    cache->blob_get_cb = get;
523 }
524 
525 #endif /* ENABLE_SHADER_CACHE */
526