1 /*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "compiler/glsl/ir_uniform.h"
25 #include "compiler/glsl/shader_cache.h"
26 #include "main/mtypes.h"
27 #include "util/blob.h"
28 #include "util/build_id.h"
29 #include "util/debug.h"
30 #include "util/disk_cache.h"
31 #include "util/macros.h"
32 #include "util/mesa-sha1.h"
33
34 #include "compiler/brw_eu.h"
35 #include "dev/intel_debug.h"
36
37 #include "brw_context.h"
38 #include "brw_program.h"
39 #include "brw_cs.h"
40 #include "brw_gs.h"
41 #include "brw_state.h"
42 #include "brw_vs.h"
43 #include "brw_wm.h"
44
45 static bool
debug_enabled_for_stage(gl_shader_stage stage)46 debug_enabled_for_stage(gl_shader_stage stage)
47 {
48 static const uint64_t stage_debug_flags[] = {
49 DEBUG_VS, DEBUG_TCS, DEBUG_TES, DEBUG_GS, DEBUG_WM, DEBUG_CS,
50 };
51 assert((int)stage >= 0 && stage < ARRAY_SIZE(stage_debug_flags));
52 return INTEL_DEBUG(stage_debug_flags[stage]);
53 }
54
55 static void
intel_shader_sha1(struct gl_program * prog,gl_shader_stage stage,void * key,unsigned char * out_sha1)56 intel_shader_sha1(struct gl_program *prog, gl_shader_stage stage,
57 void *key, unsigned char *out_sha1)
58 {
59 char sha1_buf[41];
60 unsigned char sha1[20];
61 char manifest[256];
62 int offset = 0;
63
64 _mesa_sha1_format(sha1_buf, prog->sh.data->sha1);
65 offset += snprintf(manifest, sizeof(manifest), "program: %s\n", sha1_buf);
66
67 _mesa_sha1_compute(key, brw_prog_key_size(stage), sha1);
68 _mesa_sha1_format(sha1_buf, sha1);
69 offset += snprintf(manifest + offset, sizeof(manifest) - offset,
70 "%s_key: %s\n", _mesa_shader_stage_to_abbrev(stage),
71 sha1_buf);
72
73 _mesa_sha1_compute(manifest, strlen(manifest), out_sha1);
74 }
75
76 static bool
read_blob_program_data(struct blob_reader * binary,struct gl_program * prog,gl_shader_stage stage,const uint8_t ** program,struct brw_stage_prog_data * prog_data)77 read_blob_program_data(struct blob_reader *binary, struct gl_program *prog,
78 gl_shader_stage stage, const uint8_t **program,
79 struct brw_stage_prog_data *prog_data)
80 {
81 return
82 brw_read_blob_program_data(binary, prog, stage, program, prog_data) &&
83 (binary->current == binary->end);
84 }
85
86 static bool
read_and_upload(struct brw_context * brw,struct disk_cache * cache,struct gl_program * prog,gl_shader_stage stage)87 read_and_upload(struct brw_context *brw, struct disk_cache *cache,
88 struct gl_program *prog, gl_shader_stage stage)
89 {
90 unsigned char binary_sha1[20];
91
92 union brw_any_prog_key prog_key;
93
94 switch (stage) {
95 case MESA_SHADER_VERTEX:
96 brw_vs_populate_key(brw, &prog_key.vs);
97 break;
98 case MESA_SHADER_TESS_CTRL:
99 brw_tcs_populate_key(brw, &prog_key.tcs);
100 break;
101 case MESA_SHADER_TESS_EVAL:
102 brw_tes_populate_key(brw, &prog_key.tes);
103 break;
104 case MESA_SHADER_GEOMETRY:
105 brw_gs_populate_key(brw, &prog_key.gs);
106 break;
107 case MESA_SHADER_FRAGMENT:
108 brw_wm_populate_key(brw, &prog_key.wm);
109 break;
110 case MESA_SHADER_COMPUTE:
111 brw_cs_populate_key(brw, &prog_key.cs);
112 break;
113 default:
114 unreachable("Unsupported stage!");
115 }
116
117 /* We don't care what instance of the program it is for the disk cache hash
118 * lookup, so set the id to 0 for the sha1 hashing. program_string_id will
119 * be set below.
120 */
121 prog_key.base.program_string_id = 0;
122
123 intel_shader_sha1(prog, stage, &prog_key, binary_sha1);
124
125 size_t buffer_size;
126 uint8_t *buffer = disk_cache_get(cache, binary_sha1, &buffer_size);
127 if (buffer == NULL) {
128 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
129 char sha1_buf[41];
130 _mesa_sha1_format(sha1_buf, binary_sha1);
131 fprintf(stderr, "No cached %s binary found for: %s\n",
132 _mesa_shader_stage_to_abbrev(stage), sha1_buf);
133 }
134 return false;
135 }
136
137 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
138 char sha1_buf[41];
139 _mesa_sha1_format(sha1_buf, binary_sha1);
140 fprintf(stderr, "attempting to populate bo cache with binary: %s\n",
141 sha1_buf);
142 }
143
144 struct blob_reader binary;
145 blob_reader_init(&binary, buffer, buffer_size);
146
147 const uint8_t *program;
148 struct brw_stage_prog_data *prog_data =
149 ralloc_size(NULL, sizeof(union brw_any_prog_data));
150 if (!read_blob_program_data(&binary, prog, stage, &program, prog_data)) {
151 /* Something very bad has gone wrong discard the item from the cache and
152 * rebuild from source.
153 */
154 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
155 fprintf(stderr, "Error reading program from cache (invalid i965 "
156 "cache item)\n");
157 }
158
159 disk_cache_remove(cache, binary_sha1);
160 ralloc_free(prog_data);
161 free(buffer);
162 return false;
163 }
164
165 enum brw_cache_id cache_id;
166 struct brw_stage_state *stage_state;
167
168 switch (stage) {
169 case MESA_SHADER_VERTEX:
170 cache_id = BRW_CACHE_VS_PROG;
171 stage_state = &brw->vs.base;
172 break;
173 case MESA_SHADER_TESS_CTRL:
174 cache_id = BRW_CACHE_TCS_PROG;
175 stage_state = &brw->tcs.base;
176 break;
177 case MESA_SHADER_TESS_EVAL:
178 cache_id = BRW_CACHE_TES_PROG;
179 stage_state = &brw->tes.base;
180 break;
181 case MESA_SHADER_GEOMETRY:
182 cache_id = BRW_CACHE_GS_PROG;
183 stage_state = &brw->gs.base;
184 break;
185 case MESA_SHADER_FRAGMENT:
186 cache_id = BRW_CACHE_FS_PROG;
187 stage_state = &brw->wm.base;
188 break;
189 case MESA_SHADER_COMPUTE:
190 cache_id = BRW_CACHE_CS_PROG;
191 stage_state = &brw->cs.base;
192 break;
193 default:
194 unreachable("Unsupported stage!");
195 }
196
197 prog_key.base.program_string_id = brw_program(prog)->id;
198
199 brw_alloc_stage_scratch(brw, stage_state, prog_data->total_scratch);
200
201 if (unlikely(debug_enabled_for_stage(stage))) {
202 fprintf(stderr, "NIR for %s program %d loaded from disk shader cache:\n",
203 _mesa_shader_stage_to_abbrev(stage), brw_program(prog)->id);
204 brw_program_deserialize_driver_blob(&brw->ctx, prog, stage);
205 nir_shader *nir = prog->nir;
206 nir_print_shader(nir, stderr);
207 fprintf(stderr, "Native code for %s %s shader %s from disk cache:\n",
208 nir->info.label ? nir->info.label : "unnamed",
209 _mesa_shader_stage_to_string(nir->info.stage), nir->info.name);
210 brw_disassemble_with_labels(&brw->screen->devinfo, program, 0,
211 prog_data->program_size, stderr);
212 }
213
214 brw_upload_cache(&brw->cache, cache_id, &prog_key, brw_prog_key_size(stage),
215 program, prog_data->program_size, prog_data,
216 brw_prog_data_size(stage), &stage_state->prog_offset,
217 &stage_state->prog_data);
218
219 prog->program_written_to_cache = true;
220
221 ralloc_free(prog_data);
222 free(buffer);
223
224 return true;
225 }
226
227 bool
brw_disk_cache_upload_program(struct brw_context * brw,gl_shader_stage stage)228 brw_disk_cache_upload_program(struct brw_context *brw, gl_shader_stage stage)
229 {
230 struct disk_cache *cache = brw->ctx.Cache;
231 if (cache == NULL)
232 return false;
233
234 struct gl_program *prog = brw->ctx._Shader->CurrentProgram[stage];
235 if (prog == NULL)
236 return false;
237
238 if (prog->sh.data->spirv)
239 return false;
240
241 if (brw->ctx._Shader->Flags & GLSL_CACHE_FALLBACK)
242 goto fail;
243
244 if (!read_and_upload(brw, cache, prog, stage))
245 goto fail;
246
247 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
248 fprintf(stderr, "read gen program from cache\n");
249 }
250
251 return true;
252
253 fail:
254 prog->program_written_to_cache = false;
255 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
256 fprintf(stderr, "falling back to nir %s.\n",
257 _mesa_shader_stage_to_abbrev(prog->info.stage));
258 }
259
260 brw_program_deserialize_driver_blob(&brw->ctx, prog, stage);
261
262 return false;
263 }
264
265 static void
write_program_data(struct brw_context * brw,struct gl_program * prog,void * key,struct brw_stage_prog_data * prog_data,uint32_t prog_offset,struct disk_cache * cache,gl_shader_stage stage)266 write_program_data(struct brw_context *brw, struct gl_program *prog,
267 void *key, struct brw_stage_prog_data *prog_data,
268 uint32_t prog_offset, struct disk_cache *cache,
269 gl_shader_stage stage)
270 {
271 struct blob binary;
272 blob_init(&binary);
273
274 const void *program_map = brw->cache.map + prog_offset;
275 /* TODO: Improve perf for non-LLC. It would be best to save it at program
276 * generation time when the program is in normal memory accessible with
277 * cache to the CPU. Another easier change would be to use
278 * _mesa_streaming_load_memcpy to read from the program mapped memory. */
279 brw_write_blob_program_data(&binary, stage, program_map, prog_data);
280
281 unsigned char sha1[20];
282 char buf[41];
283 intel_shader_sha1(prog, stage, key, sha1);
284 _mesa_sha1_format(buf, sha1);
285 if (brw->ctx._Shader->Flags & GLSL_CACHE_INFO) {
286 fprintf(stderr, "putting binary in cache: %s\n", buf);
287 }
288
289 disk_cache_put(cache, sha1, binary.data, binary.size, NULL);
290
291 prog->program_written_to_cache = true;
292 blob_finish(&binary);
293 }
294
295 void
brw_disk_cache_write_render_programs(struct brw_context * brw)296 brw_disk_cache_write_render_programs(struct brw_context *brw)
297 {
298 struct disk_cache *cache = brw->ctx.Cache;
299 if (cache == NULL)
300 return;
301
302 struct gl_program *prog;
303 gl_shader_stage stage;
304 for (stage = MESA_SHADER_VERTEX; stage <= MESA_SHADER_FRAGMENT; stage++) {
305 prog = brw->ctx._Shader->CurrentProgram[stage];
306 if (prog && prog->sh.data->spirv)
307 return;
308 }
309
310 prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_VERTEX];
311 if (prog && !prog->program_written_to_cache) {
312 struct brw_vs_prog_key vs_key;
313 brw_vs_populate_key(brw, &vs_key);
314 vs_key.base.program_string_id = 0;
315
316 write_program_data(brw, prog, &vs_key, brw->vs.base.prog_data,
317 brw->vs.base.prog_offset, cache,
318 MESA_SHADER_VERTEX);
319 }
320
321 prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_TESS_CTRL];
322 if (prog && !prog->program_written_to_cache) {
323 struct brw_tcs_prog_key tcs_key;
324 brw_tcs_populate_key(brw, &tcs_key);
325 tcs_key.base.program_string_id = 0;
326
327 write_program_data(brw, prog, &tcs_key, brw->tcs.base.prog_data,
328 brw->tcs.base.prog_offset, cache,
329 MESA_SHADER_TESS_CTRL);
330 }
331
332 prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_TESS_EVAL];
333 if (prog && !prog->program_written_to_cache) {
334 struct brw_tes_prog_key tes_key;
335 brw_tes_populate_key(brw, &tes_key);
336 tes_key.base.program_string_id = 0;
337
338 write_program_data(brw, prog, &tes_key, brw->tes.base.prog_data,
339 brw->tes.base.prog_offset, cache,
340 MESA_SHADER_TESS_EVAL);
341 }
342
343 prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_GEOMETRY];
344 if (prog && !prog->program_written_to_cache) {
345 struct brw_gs_prog_key gs_key;
346 brw_gs_populate_key(brw, &gs_key);
347 gs_key.base.program_string_id = 0;
348
349 write_program_data(brw, prog, &gs_key, brw->gs.base.prog_data,
350 brw->gs.base.prog_offset, cache,
351 MESA_SHADER_GEOMETRY);
352 }
353
354 prog = brw->ctx._Shader->CurrentProgram[MESA_SHADER_FRAGMENT];
355 if (prog && !prog->program_written_to_cache) {
356 struct brw_wm_prog_key wm_key;
357 brw_wm_populate_key(brw, &wm_key);
358 wm_key.base.program_string_id = 0;
359
360 write_program_data(brw, prog, &wm_key, brw->wm.base.prog_data,
361 brw->wm.base.prog_offset, cache,
362 MESA_SHADER_FRAGMENT);
363 }
364 }
365
366 void
brw_disk_cache_write_compute_program(struct brw_context * brw)367 brw_disk_cache_write_compute_program(struct brw_context *brw)
368 {
369 struct disk_cache *cache = brw->ctx.Cache;
370 if (cache == NULL)
371 return;
372
373 struct gl_program *prog =
374 brw->ctx._Shader->CurrentProgram[MESA_SHADER_COMPUTE];
375
376 if (prog && prog->sh.data->spirv)
377 return;
378
379 if (prog && !prog->program_written_to_cache) {
380 struct brw_cs_prog_key cs_key;
381 brw_cs_populate_key(brw, &cs_key);
382 cs_key.base.program_string_id = 0;
383
384 write_program_data(brw, prog, &cs_key, brw->cs.base.prog_data,
385 brw->cs.base.prog_offset, cache,
386 MESA_SHADER_COMPUTE);
387 }
388 }
389
390 void
brw_disk_cache_init(struct brw_screen * screen)391 brw_disk_cache_init(struct brw_screen *screen)
392 {
393 #ifdef ENABLE_SHADER_CACHE
394 if (INTEL_DEBUG(DEBUG_DISK_CACHE_DISABLE_MASK))
395 return;
396
397 /* array length: print length + null char + 1 extra to verify it is unused */
398 char renderer[11];
399 ASSERTED int len = snprintf(renderer, sizeof(renderer), "i965_%04x",
400 screen->deviceID);
401 assert(len == sizeof(renderer) - 2);
402
403 const struct build_id_note *note =
404 build_id_find_nhdr_for_addr(brw_disk_cache_init);
405 assert(note && build_id_length(note) == 20 /* sha1 */);
406
407 const uint8_t *id_sha1 = build_id_data(note);
408 assert(id_sha1);
409
410 char timestamp[41];
411 _mesa_sha1_format(timestamp, id_sha1);
412
413 const uint64_t driver_flags =
414 brw_get_compiler_config_value(screen->compiler);
415 screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags);
416 #endif
417 }
418