1 /*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "compiler/blob.h"
25 #include "compiler/glsl/ir_uniform.h"
26 #include "compiler/glsl/shader_cache.h"
27 #include "main/mtypes.h"
28 #include "util/build_id.h"
29 #include "util/debug.h"
30 #include "util/disk_cache.h"
31 #include "util/macros.h"
32 #include "util/mesa-sha1.h"
33
34 #include "brw_context.h"
35 #include "brw_program.h"
36 #include "brw_cs.h"
37 #include "brw_gs.h"
38 #include "brw_state.h"
39 #include "brw_vs.h"
40 #include "brw_wm.h"
41
42 static void
gen_shader_sha1(struct brw_context * brw,struct gl_program * prog,gl_shader_stage stage,void * key,unsigned char * out_sha1)43 gen_shader_sha1(struct brw_context *brw, struct gl_program *prog,
44 gl_shader_stage stage, void *key, unsigned char *out_sha1)
45 {
46 char sha1_buf[41];
47 unsigned char sha1[20];
48 char manifest[256];
49 int offset = 0;
50
51 _mesa_sha1_format(sha1_buf, prog->sh.data->sha1);
52 offset += snprintf(manifest, sizeof(manifest), "program: %s\n", sha1_buf);
53
54 _mesa_sha1_compute(key, brw_prog_key_size(stage), sha1);
55 _mesa_sha1_format(sha1_buf, sha1);
56 offset += snprintf(manifest + offset, sizeof(manifest) - offset,
57 "%s_key: %s\n", _mesa_shader_stage_to_abbrev(stage),
58 sha1_buf);
59
60 _mesa_sha1_compute(manifest, strlen(manifest), out_sha1);
61 }
62
63 static void
write_blob_program_data(struct blob * binary,gl_shader_stage stage,const void * program,struct brw_stage_prog_data * prog_data)64 write_blob_program_data(struct blob *binary, gl_shader_stage stage,
65 const void *program,
66 struct brw_stage_prog_data *prog_data)
67 {
68 /* Write prog_data to blob. */
69 blob_write_bytes(binary, prog_data, brw_prog_data_size(stage));
70
71 /* Write program to blob. */
72 blob_write_bytes(binary, program, prog_data->program_size);
73
74 /* Write push params */
75 blob_write_bytes(binary, prog_data->param,
76 sizeof(uint32_t) * prog_data->nr_params);
77
78 /* Write pull params */
79 blob_write_bytes(binary, prog_data->pull_param,
80 sizeof(uint32_t) * prog_data->nr_pull_params);
81 }
82
83 static bool
read_blob_program_data(struct blob_reader * binary,struct gl_program * prog,gl_shader_stage stage,const uint8_t ** program,struct brw_stage_prog_data * prog_data)84 read_blob_program_data(struct blob_reader *binary, struct gl_program *prog,
85 gl_shader_stage stage, const uint8_t **program,
86 struct brw_stage_prog_data *prog_data)
87 {
88 /* Read shader prog_data from blob. */
89 blob_copy_bytes(binary, prog_data, brw_prog_data_size(stage));
90 if (binary->overrun)
91 return false;
92
93 /* Read shader program from blob. */
94 *program = blob_read_bytes(binary, prog_data->program_size);
95
96 /* Read push params */
97 prog_data->param = rzalloc_array(NULL, uint32_t, prog_data->nr_params);
98 blob_copy_bytes(binary, prog_data->param,
99 sizeof(uint32_t) * prog_data->nr_params);
100
101 /* Read pull params */
102 prog_data->pull_param = rzalloc_array(NULL, uint32_t,
103 prog_data->nr_pull_params);
104 blob_copy_bytes(binary, prog_data->pull_param,
105 sizeof(uint32_t) * prog_data->nr_pull_params);
106
107 return (binary->current == binary->end && !binary->overrun);
108 }
109
110 static bool
read_and_upload(struct brw_context * brw,struct disk_cache * cache,struct gl_program * prog,gl_shader_stage stage)111 read_and_upload(struct brw_context *brw, struct disk_cache *cache,
112 struct gl_program *prog, gl_shader_stage stage)
113 {
114 unsigned char binary_sha1[20];
115
116 union brw_any_prog_key prog_key;
117
118 switch (stage) {
119 case MESA_SHADER_VERTEX:
120 brw_vs_populate_key(brw, &prog_key.vs);
121 /* We don't care what instance of the program it is for the disk cache
122 * hash lookup, so set the id to 0 for the sha1 hashing.
123 * program_string_id will be set below.
124 */
125 prog_key.vs.program_string_id = 0;
126 break;
127 case MESA_SHADER_TESS_CTRL:
128 brw_tcs_populate_key(brw, &prog_key.tcs);
129 prog_key.tcs.program_string_id = 0;
130 break;
131 case MESA_SHADER_TESS_EVAL:
132 brw_tes_populate_key(brw, &prog_key.tes);
133 prog_key.tes.program_string_id = 0;
134 break;
135 case MESA_SHADER_GEOMETRY:
136 brw_gs_populate_key(brw, &prog_key.gs);
137 prog_key.gs.program_string_id = 0;
138 break;
139 case MESA_SHADER_FRAGMENT:
140 brw_wm_populate_key(brw, &prog_key.wm);
141 prog_key.wm.program_string_id = 0;
142 break;
143 case MESA_SHADER_COMPUTE:
144 brw_cs_populate_key(brw, &prog_key.cs);
145 prog_key.cs.program_string_id = 0;
146 break;
147 default:
148 unreachable("Unsupported stage!");
149 }
150
151 gen_shader_sha1(brw, prog, stage, &prog_key, binary_sha1);
152
153 size_t buffer_size;
154 uint8_t *buffer = disk_cache_get(cache, binary_sha1, &buffer_size);
155 if (buffer == NULL) {
156 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
157 char sha1_buf[41];
158 _mesa_sha1_format(sha1_buf, binary_sha1);
159 fprintf(stderr, "No cached %s binary found for: %s\n",
160 _mesa_shader_stage_to_abbrev(stage), sha1_buf);
161 }
162 return false;
163 }
164
165 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
166 char sha1_buf[41];
167 _mesa_sha1_format(sha1_buf, binary_sha1);
168 fprintf(stderr, "attempting to populate bo cache with binary: %s\n",
169 sha1_buf);
170 }
171
172 struct blob_reader binary;
173 blob_reader_init(&binary, buffer, buffer_size);
174
175 const uint8_t *program;
176 struct brw_stage_prog_data *prog_data =
177 ralloc_size(NULL, sizeof(union brw_any_prog_data));
178 if (!read_blob_program_data(&binary, prog, stage, &program, prog_data)) {
179 /* Something very bad has gone wrong discard the item from the cache and
180 * rebuild from source.
181 */
182 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
183 fprintf(stderr, "Error reading program from cache (invalid i965 "
184 "cache item)\n");
185 }
186
187 disk_cache_remove(cache, binary_sha1);
188 ralloc_free(prog_data);
189 free(buffer);
190 return false;
191 }
192
193 enum brw_cache_id cache_id;
194 struct brw_stage_state *stage_state;
195
196 switch (stage) {
197 case MESA_SHADER_VERTEX:
198 prog_key.vs.program_string_id = brw_program(prog)->id;
199 cache_id = BRW_CACHE_VS_PROG;
200 stage_state = &brw->vs.base;
201 break;
202 case MESA_SHADER_TESS_CTRL:
203 prog_key.tcs.program_string_id = brw_program(prog)->id;
204 cache_id = BRW_CACHE_TCS_PROG;
205 stage_state = &brw->tcs.base;
206 break;
207 case MESA_SHADER_TESS_EVAL:
208 prog_key.tes.program_string_id = brw_program(prog)->id;
209 cache_id = BRW_CACHE_TES_PROG;
210 stage_state = &brw->tes.base;
211 break;
212 case MESA_SHADER_GEOMETRY:
213 prog_key.gs.program_string_id = brw_program(prog)->id;
214 cache_id = BRW_CACHE_GS_PROG;
215 stage_state = &brw->gs.base;
216 break;
217 case MESA_SHADER_FRAGMENT:
218 prog_key.wm.program_string_id = brw_program(prog)->id;
219 cache_id = BRW_CACHE_FS_PROG;
220 stage_state = &brw->wm.base;
221 break;
222 case MESA_SHADER_COMPUTE:
223 prog_key.cs.program_string_id = brw_program(prog)->id;
224 cache_id = BRW_CACHE_CS_PROG;
225 stage_state = &brw->cs.base;
226 break;
227 default:
228 unreachable("Unsupported stage!");
229 }
230
231 brw_alloc_stage_scratch(brw, stage_state, prog_data->total_scratch);
232
233 brw_upload_cache(&brw->cache, cache_id, &prog_key, brw_prog_key_size(stage),
234 program, prog_data->program_size, prog_data,
235 brw_prog_data_size(stage), &stage_state->prog_offset,
236 &stage_state->prog_data);
237
238 prog->program_written_to_cache = true;
239
240 ralloc_free(prog_data);
241 free(buffer);
242
243 return true;
244 }
245
246 bool
brw_disk_cache_upload_program(struct brw_context * brw,gl_shader_stage stage)247 brw_disk_cache_upload_program(struct brw_context *brw, gl_shader_stage stage)
248 {
249 struct disk_cache *cache = brw->ctx.Cache;
250 if (cache == NULL)
251 return false;
252
253 struct gl_program *prog = brw->ctx._Shader->CurrentProgram[stage];
254 if (prog == NULL)
255 return false;
256
257 /* FIXME: For now we don't read from the cache if transform feedback is
258 * enabled via the API. However the shader cache does support transform
259 * feedback when enabled via in shader xfb qualifiers.
260 */
261 if (prog->sh.LinkedTransformFeedback &&
262 prog->sh.LinkedTransformFeedback->api_enabled)
263 return false;
264
265 if (brw->ctx._Shader->Flags & GLSL_CACHE_FALLBACK)
266 goto fail;
267
268 if (prog->sh.data->LinkStatus != linking_skipped)
269 goto fail;
270
271 if (!read_and_upload(brw, cache, prog, stage))
272 goto fail;
273
274 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
275 fprintf(stderr, "read gen program from cache\n");
276 }
277
278 return true;
279
280 fail:
281 prog->program_written_to_cache = false;
282 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
283 fprintf(stderr, "falling back to nir %s.\n",
284 _mesa_shader_stage_to_abbrev(prog->info.stage));
285 }
286
287 brw_program_deserialize_nir(&brw->ctx, prog, stage);
288
289 return false;
290 }
291
292 static void
write_program_data(struct brw_context * brw,struct gl_program * prog,void * key,struct brw_stage_prog_data * prog_data,uint32_t prog_offset,struct disk_cache * cache,gl_shader_stage stage)293 write_program_data(struct brw_context *brw, struct gl_program *prog,
294 void *key, struct brw_stage_prog_data *prog_data,
295 uint32_t prog_offset, struct disk_cache *cache,
296 gl_shader_stage stage)
297 {
298 struct blob binary;
299 blob_init(&binary);
300
301 const void *program_map = brw->cache.map + prog_offset;
302 /* TODO: Improve perf for non-LLC. It would be best to save it at program
303 * generation time when the program is in normal memory accessible with
304 * cache to the CPU. Another easier change would be to use
305 * _mesa_streaming_load_memcpy to read from the program mapped memory. */
306 write_blob_program_data(&binary, stage, program_map, prog_data);
307
308 unsigned char sha1[20];
309 char buf[41];
310 gen_shader_sha1(brw, prog, stage, key, sha1);
311 _mesa_sha1_format(buf, sha1);
312 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
313 fprintf(stderr, "putting binary in cache: %s\n", buf);
314 }
315
316 disk_cache_put(cache, sha1, binary.data, binary.size, NULL);
317
318 prog->program_written_to_cache = true;
319 blob_finish(&binary);
320 }
321
322 void
brw_disk_cache_write_render_programs(struct brw_context * brw)323 brw_disk_cache_write_render_programs(struct brw_context *brw)
324 {
325 struct disk_cache *cache = brw->ctx.Cache;
326 if (cache == NULL)
327 return;
328
329 struct gl_program *prog =
330 brw->ctx._Shader->CurrentProgram[MESA_SHADER_VERTEX];
331 if (prog && !prog->program_written_to_cache) {
332 struct brw_vs_prog_key vs_key;
333 brw_vs_populate_key(brw, &vs_key);
334 vs_key.program_string_id = 0;
335
336 write_program_data(brw, prog, &vs_key, brw->vs.base.prog_data,
337 brw->vs.base.prog_offset, cache,
338 MESA_SHADER_VERTEX);
339 }
340
341 prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_TESS_CTRL];
342 if (prog && !prog->program_written_to_cache) {
343 struct brw_tcs_prog_key tcs_key;
344 brw_tcs_populate_key(brw, &tcs_key);
345 tcs_key.program_string_id = 0;
346
347 write_program_data(brw, prog, &tcs_key, brw->tcs.base.prog_data,
348 brw->tcs.base.prog_offset, cache,
349 MESA_SHADER_TESS_CTRL);
350 }
351
352 prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_TESS_EVAL];
353 if (prog && !prog->program_written_to_cache) {
354 struct brw_tes_prog_key tes_key;
355 brw_tes_populate_key(brw, &tes_key);
356 tes_key.program_string_id = 0;
357
358 write_program_data(brw, prog, &tes_key, brw->tes.base.prog_data,
359 brw->tes.base.prog_offset, cache,
360 MESA_SHADER_TESS_EVAL);
361 }
362
363 prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_GEOMETRY];
364 if (prog && !prog->program_written_to_cache) {
365 struct brw_gs_prog_key gs_key;
366 brw_gs_populate_key(brw, &gs_key);
367 gs_key.program_string_id = 0;
368
369 write_program_data(brw, prog, &gs_key, brw->gs.base.prog_data,
370 brw->gs.base.prog_offset, cache,
371 MESA_SHADER_GEOMETRY);
372 }
373
374 prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_FRAGMENT];
375 if (prog && !prog->program_written_to_cache) {
376 struct brw_wm_prog_key wm_key;
377 brw_wm_populate_key(brw, &wm_key);
378 wm_key.program_string_id = 0;
379
380 write_program_data(brw, prog, &wm_key, brw->wm.base.prog_data,
381 brw->wm.base.prog_offset, cache,
382 MESA_SHADER_FRAGMENT);
383 }
384 }
385
386 void
brw_disk_cache_write_compute_program(struct brw_context * brw)387 brw_disk_cache_write_compute_program(struct brw_context *brw)
388 {
389 struct disk_cache *cache = brw->ctx.Cache;
390 if (cache == NULL)
391 return;
392
393 struct gl_program *prog =
394 brw->ctx._Shader->CurrentProgram[MESA_SHADER_COMPUTE];
395 if (prog && !prog->program_written_to_cache) {
396 struct brw_cs_prog_key cs_key;
397 brw_cs_populate_key(brw, &cs_key);
398 cs_key.program_string_id = 0;
399
400 write_program_data(brw, prog, &cs_key, brw->cs.base.prog_data,
401 brw->cs.base.prog_offset, cache,
402 MESA_SHADER_COMPUTE);
403 }
404 }
405
406 void
brw_disk_cache_init(struct brw_context * brw)407 brw_disk_cache_init(struct brw_context *brw)
408 {
409 #ifdef ENABLE_SHADER_CACHE
410 if (env_var_as_boolean("MESA_GLSL_CACHE_DISABLE", true))
411 return;
412
413 char renderer[10];
414 MAYBE_UNUSED int len = snprintf(renderer, sizeof(renderer), "i965_%04x",
415 brw->screen->deviceID);
416 assert(len == sizeof(renderer) - 1);
417
418 const struct build_id_note *note =
419 build_id_find_nhdr_for_addr(brw_disk_cache_init);
420 assert(note && build_id_length(note) == 20 /* sha1 */);
421
422 const uint8_t *id_sha1 = build_id_data(note);
423 assert(id_sha1);
424
425 char timestamp[41];
426 _mesa_sha1_format(timestamp, id_sha1);
427
428 brw->ctx.Cache = disk_cache_create(renderer, timestamp, 0);
429 #endif
430 }
431