25  * use with the Mesa shader cache.
39 #include <sys/file.h>
45 #include "mesa-sha1.h"
57 /* Mesa uses 160bit hashes to identify cache entries, a hash of this size
59  * format uses a 64bit hash table to lookup file offsets for reading cache
69       shift--;  in truncate_hash_to_64bits()
75 check_files_opened_successfully(FILE *file, FILE *db_idx)  in check_files_opened_successfully()  argument
77    if (!file) {  in check_files_opened_successfully()
84       if (file)  in check_files_opened_successfully()
85          fclose(file);  in check_files_opened_successfully()
96    if (asprintf(filename, "%s/%s.foz", cache_path, name) == -1)  in create_foz_db_filenames()
99    if (asprintf(idx_filename, "%s/%s_idx.foz", cache_path, name) == -1) {  in create_foz_db_filenames()
109  * to do without locking the file as we assume the file is append only */
111 update_foz_index(struct foz_db *foz_db, FILE *db_idx, unsigned file_idx)  in update_foz_index()
126       /* Corrupt entry. Our process might have been killed before we  in update_foz_index()
140       /* Corrupt entry. Our process might have been killed before we  in update_foz_index()
143       if (offset + header->payload_size > len ||  in update_foz_index()
144           header->payload_size != sizeof(uint64_t))  in update_foz_index()
150       /* read cache item offset from index file */  in update_foz_index()
156       offset += header->payload_size;  in update_foz_index()
159       struct foz_db_entry *entry = ralloc(foz_db->mem_ctx,  in update_foz_index()  local
161       entry->header = *header;  in update_foz_index()
162       entry->file_idx = file_idx;  in update_foz_index()
163       _mesa_sha1_hex_to_sha1(entry->key, hash_str);  in update_foz_index()
165       /* Truncate the entry's hash string to a 64bit hash for use with a  in update_foz_index()
166        * 64bit hash table for looking up file offsets.  in update_foz_index()
171       entry->offset = cache_offset;  in update_foz_index()
173       _mesa_hash_table_u64_insert(foz_db->index_db, key, entry);  in update_foz_index()
181 static int lock_file_with_timeout(FILE *f, int64_t timeout)  in lock_file_with_timeout()
199 load_foz_dbs(struct foz_db *foz_db, FILE *db_idx, uint8_t file_idx,  in load_foz_dbs()
202    /* Scan through the archive and get the list of cache entries. */  in load_foz_dbs()
211       int err = lock_file_with_timeout(foz_db->file[file_idx], 100000000);  in load_foz_dbs()
212       if (err == -1)  in load_foz_dbs()
227                  FOZ_REF_MAGIC_SIZE - 1))  in load_foz_dbs()
230       int version = magic[FOZ_REF_MAGIC_SIZE - 1];  in load_foz_dbs()
236       /* Appending to a fresh file. Make sure we have the magic. */  in load_foz_dbs()
238                  sizeof(stream_reference_magic_and_version), foz_db->file[file_idx]) !=  in load_foz_dbs()
247       fflush(foz_db->file[file_idx]);  in load_foz_dbs()
251    flock(fileno(foz_db->file[file_idx]), LOCK_UN);  in load_foz_dbs()
255    foz_db->alive = true;  in load_foz_dbs()
259    flock(fileno(foz_db->file[file_idx]), LOCK_UN);  in load_foz_dbs()
264 /* Here we open mesa cache foz dbs files. If the files exist we load the index
266  * read cache entries from the foz db containing the actual cache entries.
279    foz_db->file[0] = fopen(filename, "a+b");  in foz_prepare()
280    foz_db->db_idx = fopen(idx_filename, "a+b");  in foz_prepare()
285    if (!check_files_opened_successfully(foz_db->file[0], foz_db->db_idx))  in foz_prepare()
288    simple_mtx_init(&foz_db->mtx, mtx_plain);  in foz_prepare()
289    simple_mtx_init(&foz_db->flock_mtx, mtx_plain);  in foz_prepare()
290    foz_db->mem_ctx = ralloc_context(NULL);  in foz_prepare()
291    foz_db->index_db = _mesa_hash_table_u64_create(NULL);  in foz_prepare()
293    if (!load_foz_dbs(foz_db, foz_db->db_idx, 0, false))  in foz_prepare()
315       foz_db->file[file_idx] = fopen(filename, "rb");  in foz_prepare()
316       FILE *db_idx = fopen(idx_filename, "rb");  in foz_prepare()
321       if (!check_files_opened_successfully(foz_db->file[file_idx], db_idx)) {  in foz_prepare()
323          foz_db->file[file_idx] = NULL;  in foz_prepare()
346    if (foz_db->db_idx)  in foz_destroy()
347       fclose(foz_db->db_idx);  in foz_destroy()
349       if (foz_db->file[i])  in foz_destroy()
350          fclose(foz_db->file[i]);  in foz_destroy()
353    if (foz_db->mem_ctx) {  in foz_destroy()
354       _mesa_hash_table_u64_destroy(foz_db->index_db);  in foz_destroy()
355       ralloc_free(foz_db->mem_ctx);  in foz_destroy()
356       simple_mtx_destroy(&foz_db->flock_mtx);  in foz_destroy()
357       simple_mtx_destroy(&foz_db->mtx);  in foz_destroy()
361 /* Here we lookup a cache entry in the index hash table. If an entry is found
362  * we use the retrieved offset to read the cache entry from disk.
372    if (!foz_db->alive)  in foz_read_entry()
375    simple_mtx_lock(&foz_db->mtx);  in foz_read_entry()
377    struct foz_db_entry *entry =  in foz_read_entry()  local
378       _mesa_hash_table_u64_search(foz_db->index_db, hash);  in foz_read_entry()
379    if (!entry) {  in foz_read_entry()
380       update_foz_index(foz_db, foz_db->db_idx, 0);  in foz_read_entry()
381       entry = _mesa_hash_table_u64_search(foz_db->index_db, hash);  in foz_read_entry()
383    if (!entry) {  in foz_read_entry()
384       simple_mtx_unlock(&foz_db->mtx);  in foz_read_entry()
388    uint8_t file_idx = entry->file_idx;  in foz_read_entry()
389    if (fseek(foz_db->file[file_idx], entry->offset, SEEK_SET) < 0)  in foz_read_entry()
393    if (fread(&entry->header, 1, header_size, foz_db->file[file_idx]) !=  in foz_read_entry()
401       if (cache_key_160bit[i] != entry->key[i])  in foz_read_entry()
405    uint32_t data_sz = entry->header.payload_size;  in foz_read_entry()
407    if (fread(data, 1, data_sz, foz_db->file[file_idx]) != data_sz)  in foz_read_entry()
411    if (entry->header.crc != 0) {  in foz_read_entry()
412       if (util_hash_crc32(data, data_sz) != entry->header.crc)  in foz_read_entry()
416    simple_mtx_unlock(&foz_db->mtx);  in foz_read_entry()
426    /* reading db entry failed. reset the file offset */  in foz_read_entry()
427    simple_mtx_unlock(&foz_db->mtx);  in foz_read_entry()
432 /* Here we write the cache entry to disk and store its offset in the index db.
440    if (!foz_db->alive)  in foz_write_entry()
443    /* The flock is per-fd, not per thread, we do it outside of the main mutex to avoid having to  in foz_write_entry()
445     * conditions between the write threads sharing the same file descriptor. */  in foz_write_entry()
446    simple_mtx_lock(&foz_db->flock_mtx);  in foz_write_entry()
449     * for file contention than mtx contention of significant length. */  in foz_write_entry()
450    int err = lock_file_with_timeout(foz_db->file[0], 1000000000);  in foz_write_entry()
451    if (err == -1)  in foz_write_entry()
454    simple_mtx_lock(&foz_db->mtx);  in foz_write_entry()
456    update_foz_index(foz_db, foz_db->db_idx, 0);  in foz_write_entry()
458    struct foz_db_entry *entry =  in foz_write_entry()  local
459       _mesa_hash_table_u64_search(foz_db->index_db, hash);  in foz_write_entry()
460    if (entry) {  in foz_write_entry()
461       simple_mtx_unlock(&foz_db->mtx);  in foz_write_entry()
462       flock(fileno(foz_db->file[0]), LOCK_UN);  in foz_write_entry()
463       simple_mtx_unlock(&foz_db->flock_mtx);  in foz_write_entry()
467    /* Prepare db entry header and blob ready for writing */  in foz_write_entry()
474    fseek(foz_db->file[0], 0, SEEK_END);  in foz_write_entry()
479    if (fwrite(hash_str, 1, FOSSILIZE_BLOB_HASH_LENGTH, foz_db->file[0]) !=  in foz_write_entry()
483    off_t offset = ftell(foz_db->file[0]);  in foz_write_entry()
485    /* Write db entry header */  in foz_write_entry()
486    if (fwrite(&header, 1, sizeof(header), foz_db->file[0]) != sizeof(header))  in foz_write_entry()
489    /* Now write the db entry blob */  in foz_write_entry()
490    if (fwrite(blob, 1, blob_size, foz_db->file[0]) != blob_size)  in foz_write_entry()
493    /* Flush everything to file to reduce chance of cache corruption */  in foz_write_entry()
494    fflush(foz_db->file[0]);  in foz_write_entry()
497    if (fwrite(hash_str, 1, FOSSILIZE_BLOB_HASH_LENGTH, foz_db->db_idx) !=  in foz_write_entry()
506    if (fwrite(&header, 1, sizeof(header), foz_db->db_idx) !=  in foz_write_entry()
510    if (fwrite(&offset, 1, sizeof(uint64_t), foz_db->db_idx) !=  in foz_write_entry()
514    /* Flush everything to file to reduce chance of cache corruption */  in foz_write_entry()
515    fflush(foz_db->db_idx);  in foz_write_entry()
517    entry = ralloc(foz_db->mem_ctx, struct foz_db_entry);  in foz_write_entry()
518    entry->header = header;  in foz_write_entry()
519    entry->offset = offset;  in foz_write_entry()
520    entry->file_idx = 0;  in foz_write_entry()
521    _mesa_sha1_hex_to_sha1(entry->key, hash_str);  in foz_write_entry()
522    _mesa_hash_table_u64_insert(foz_db->index_db, hash, entry);  in foz_write_entry()
524    simple_mtx_unlock(&foz_db->mtx);  in foz_write_entry()
525    flock(fileno(foz_db->file[0]), LOCK_UN);  in foz_write_entry()
526    simple_mtx_unlock(&foz_db->flock_mtx);  in foz_write_entry()
531    simple_mtx_unlock(&foz_db->mtx);  in foz_write_entry()
533    flock(fileno(foz_db->file[0]), LOCK_UN);  in foz_write_entry()
534    simple_mtx_unlock(&foz_db->flock_mtx);  in foz_write_entry()
542    fprintf(stderr, "Warning: Mesa single file cache selected but Mesa wasn't "  in foz_prepare()
543            "built with single cache file support. Shader cache will be disabled"  in foz_prepare()