1 /*
2 * Copyright 2023 Rose Hudson
3 * Copyright 2022 Amazon.com, Inc. or its affiliates.
4 * Copyright 2018 Intel Corporation
5 * SPDX-License-Identifier: MIT
6 */
7
8 #include <assert.h>
9 #include <stdio.h>
10
11 #include "asahi/compiler/agx_debug.h"
12 #include "compiler/shader_enums.h"
13 #include "util/blob.h"
14 #include "util/build_id.h"
15 #include "util/disk_cache.h"
16 #include "util/mesa-sha1.h"
17 #include "agx_bo.h"
18 #include "agx_device.h"
19 #include "agx_disk_cache.h"
20 #include "agx_state.h"
21
22 /* Flags that are allowed and do not disable the disk cache */
23 #define ALLOWED_FLAGS (AGX_DBG_NO16)
24
25 /**
26 * Compute a disk cache key for the given uncompiled shader and shader key.
27 */
28 static void
agx_disk_cache_compute_key(struct disk_cache * cache,const struct agx_uncompiled_shader * uncompiled,const union asahi_shader_key * shader_key,cache_key cache_key)29 agx_disk_cache_compute_key(struct disk_cache *cache,
30 const struct agx_uncompiled_shader *uncompiled,
31 const union asahi_shader_key *shader_key,
32 cache_key cache_key)
33 {
34 uint8_t data[sizeof(uncompiled->nir_sha1) + sizeof(*shader_key)];
35 int hash_size = sizeof(uncompiled->nir_sha1);
36 int key_size;
37 if (uncompiled->type == PIPE_SHADER_VERTEX ||
38 uncompiled->type == PIPE_SHADER_TESS_EVAL)
39 key_size = sizeof(shader_key->vs);
40 else if (uncompiled->type == PIPE_SHADER_GEOMETRY)
41 key_size = sizeof(shader_key->gs);
42 else if (uncompiled->type == PIPE_SHADER_FRAGMENT)
43 key_size = sizeof(shader_key->fs);
44 else if (uncompiled->type == PIPE_SHADER_COMPUTE ||
45 uncompiled->type == PIPE_SHADER_TESS_CTRL)
46 key_size = 0;
47 else
48 unreachable("Unsupported shader stage");
49
50 memcpy(data, uncompiled->nir_sha1, hash_size);
51
52 if (key_size)
53 memcpy(data + hash_size, shader_key, key_size);
54
55 disk_cache_compute_key(cache, data, hash_size + key_size, cache_key);
56 }
57
58 static void
write_shader(struct blob * blob,const struct agx_compiled_shader * binary,bool is_root_gs)59 write_shader(struct blob *blob, const struct agx_compiled_shader *binary,
60 bool is_root_gs)
61 {
62 blob_write_bytes(blob, &binary->b.info, sizeof(binary->b.info));
63
64 if (binary->b.info.binary_size) {
65 blob_write_bytes(blob, binary->b.binary, binary->b.info.binary_size);
66 }
67
68 blob_write_bytes(blob, &binary->uvs, sizeof(binary->uvs));
69 blob_write_bytes(blob, &binary->attrib_components_read,
70 sizeof(binary->attrib_components_read));
71 blob_write_bytes(blob, &binary->epilog_key, sizeof(binary->epilog_key));
72 blob_write_uint32(blob, binary->push_range_count);
73 blob_write_bytes(blob, binary->push,
74 sizeof(binary->push[0]) * binary->push_range_count);
75
76 if (is_root_gs) {
77 blob_write_uint32(blob, binary->gs_count_words);
78 blob_write_uint32(blob, binary->gs_output_mode);
79 write_shader(blob, binary->pre_gs, false);
80
81 blob_write_uint8(blob, binary->gs_copy != NULL);
82 if (binary->gs_copy)
83 write_shader(blob, binary->gs_copy, false);
84
85 blob_write_uint8(blob, binary->gs_count != NULL);
86 if (binary->gs_count)
87 write_shader(blob, binary->gs_count, false);
88 }
89 }
90
91 static struct agx_compiled_shader *
read_shader(struct agx_screen * screen,struct blob_reader * blob,const struct agx_uncompiled_shader * uncompiled,bool is_root)92 read_shader(struct agx_screen *screen, struct blob_reader *blob,
93 const struct agx_uncompiled_shader *uncompiled, bool is_root)
94 {
95 struct agx_compiled_shader *binary = CALLOC_STRUCT(agx_compiled_shader);
96 binary->stage = uncompiled->type;
97 binary->so = uncompiled;
98
99 blob_copy_bytes(blob, &binary->b.info, sizeof(binary->b.info));
100 size_t size = binary->b.info.binary_size;
101
102 if (uncompiled->type == PIPE_SHADER_VERTEX ||
103 uncompiled->type == PIPE_SHADER_TESS_EVAL ||
104 uncompiled->type == PIPE_SHADER_FRAGMENT) {
105
106 binary->b.binary = malloc(size);
107 blob_copy_bytes(blob, binary->b.binary, size);
108
109 if (size) {
110 binary->bo = agx_bo_create(&screen->dev, size, 0,
111 AGX_BO_EXEC | AGX_BO_LOW_VA, "Executable");
112 memcpy(agx_bo_map(binary->bo), binary->b.binary, size);
113 }
114 } else if (size) {
115 binary->bo = agx_bo_create(&screen->dev, size, 0,
116 AGX_BO_EXEC | AGX_BO_LOW_VA, "Executable");
117 blob_copy_bytes(blob, agx_bo_map(binary->bo), size);
118 }
119
120 blob_copy_bytes(blob, &binary->uvs, sizeof(binary->uvs));
121 blob_copy_bytes(blob, &binary->attrib_components_read,
122 sizeof(binary->attrib_components_read));
123 blob_copy_bytes(blob, &binary->epilog_key, sizeof(binary->epilog_key));
124 binary->push_range_count = blob_read_uint32(blob);
125 blob_copy_bytes(blob, binary->push,
126 sizeof(binary->push[0]) * binary->push_range_count);
127
128 if (is_root && uncompiled->type == PIPE_SHADER_GEOMETRY) {
129 binary->gs_count_words = blob_read_uint32(blob);
130 binary->gs_output_mode = blob_read_uint32(blob);
131 binary->pre_gs = read_shader(screen, blob, uncompiled, false);
132
133 if (blob_read_uint8(blob))
134 binary->gs_copy = read_shader(screen, blob, uncompiled, false);
135
136 if (blob_read_uint8(blob))
137 binary->gs_count = read_shader(screen, blob, uncompiled, false);
138 }
139
140 return binary;
141 }
142
143 /**
144 * Store the given compiled shader in the disk cache.
145 *
146 * This should only be called on newly compiled shaders. No checking is
147 * done to prevent repeated stores of the same shader.
148 */
149 void
agx_disk_cache_store(struct disk_cache * cache,const struct agx_uncompiled_shader * uncompiled,const union asahi_shader_key * key,const struct agx_compiled_shader * binary)150 agx_disk_cache_store(struct disk_cache *cache,
151 const struct agx_uncompiled_shader *uncompiled,
152 const union asahi_shader_key *key,
153 const struct agx_compiled_shader *binary)
154 {
155 #ifdef ENABLE_SHADER_CACHE
156 if (!cache)
157 return;
158
159 cache_key cache_key;
160 agx_disk_cache_compute_key(cache, uncompiled, key, cache_key);
161
162 struct blob blob;
163 blob_init(&blob);
164
165 write_shader(&blob, binary, uncompiled->type == PIPE_SHADER_GEOMETRY);
166
167 disk_cache_put(cache, cache_key, blob.data, blob.size, NULL);
168 blob_finish(&blob);
169 #endif
170 }
171
172 /**
173 * Search for a compiled shader in the disk cache.
174 */
175 struct agx_compiled_shader *
agx_disk_cache_retrieve(struct agx_screen * screen,const struct agx_uncompiled_shader * uncompiled,const union asahi_shader_key * key)176 agx_disk_cache_retrieve(struct agx_screen *screen,
177 const struct agx_uncompiled_shader *uncompiled,
178 const union asahi_shader_key *key)
179 {
180 #ifdef ENABLE_SHADER_CACHE
181 struct disk_cache *cache = screen->disk_cache;
182 if (!cache)
183 return NULL;
184
185 cache_key cache_key;
186 agx_disk_cache_compute_key(cache, uncompiled, key, cache_key);
187
188 size_t size;
189 void *buffer = disk_cache_get(cache, cache_key, &size);
190 if (!buffer)
191 return NULL;
192
193 struct blob_reader blob;
194 blob_reader_init(&blob, buffer, size);
195
196 struct agx_compiled_shader *binary =
197 read_shader(screen, &blob, uncompiled, true);
198
199 free(buffer);
200 return binary;
201 #else
202 return NULL;
203 #endif
204 }
205
206 /**
207 * Initialise the on-disk shader cache.
208 */
209 void
agx_disk_cache_init(struct agx_screen * screen)210 agx_disk_cache_init(struct agx_screen *screen)
211 {
212 #ifdef ENABLE_SHADER_CACHE
213 if (agx_get_compiler_debug() || (screen->dev.debug & ~ALLOWED_FLAGS))
214 return;
215
216 const char *renderer = screen->pscreen.get_name(&screen->pscreen);
217
218 const struct build_id_note *note =
219 build_id_find_nhdr_for_addr(agx_disk_cache_init);
220 assert(note && build_id_length(note) == 20);
221
222 const uint8_t *id_sha1 = build_id_data(note);
223 assert(id_sha1);
224
225 char timestamp[41];
226 _mesa_sha1_format(timestamp, id_sha1);
227
228 uint64_t driver_flags = screen->dev.debug;
229 screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags);
230 #endif
231 }
232