1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 /**
24 * @file iris_program_cache.c
25 *
26 * The in-memory program cache. This is basically a hash table mapping
27 * API-specified shaders and a state key to a compiled variant. It also
28 * takes care of uploading shader assembly into a BO for use on the GPU.
29 */
30
31 #include <stdio.h>
32 #include <errno.h>
33 #include "pipe/p_defines.h"
34 #include "pipe/p_state.h"
35 #include "pipe/p_context.h"
36 #include "pipe/p_screen.h"
37 #include "util/u_atomic.h"
38 #include "util/u_upload_mgr.h"
39 #include "compiler/nir/nir.h"
40 #include "compiler/nir/nir_builder.h"
41 #include "intel/common/intel_disasm.h"
42 #include "intel/compiler/brw_compiler.h"
43 #include "intel/compiler/brw_eu.h"
44 #include "intel/compiler/brw_nir.h"
45 #include "iris_context.h"
46 #include "iris_resource.h"
47
48 struct keybox {
49 uint16_t size;
50 enum iris_program_cache_id cache_id;
51 uint8_t data[0];
52 };
53
54 static struct keybox *
make_keybox(void * mem_ctx,enum iris_program_cache_id cache_id,const void * key,uint32_t key_size)55 make_keybox(void *mem_ctx,
56 enum iris_program_cache_id cache_id,
57 const void *key,
58 uint32_t key_size)
59 {
60 struct keybox *keybox =
61 ralloc_size(mem_ctx, sizeof(struct keybox) + key_size);
62
63 keybox->cache_id = cache_id;
64 keybox->size = key_size;
65 memcpy(keybox->data, key, key_size);
66
67 return keybox;
68 }
69
70 static uint32_t
keybox_hash(const void * void_key)71 keybox_hash(const void *void_key)
72 {
73 const struct keybox *key = void_key;
74 return _mesa_hash_data(&key->cache_id, key->size + sizeof(key->cache_id));
75 }
76
77 static bool
keybox_equals(const void * void_a,const void * void_b)78 keybox_equals(const void *void_a, const void *void_b)
79 {
80 const struct keybox *a = void_a, *b = void_b;
81 if (a->size != b->size)
82 return false;
83
84 return memcmp(a->data, b->data, a->size) == 0;
85 }
86
87 struct iris_compiled_shader *
iris_find_cached_shader(struct iris_context * ice,enum iris_program_cache_id cache_id,uint32_t key_size,const void * key)88 iris_find_cached_shader(struct iris_context *ice,
89 enum iris_program_cache_id cache_id,
90 uint32_t key_size,
91 const void *key)
92 {
93 struct keybox *keybox = make_keybox(NULL, cache_id, key, key_size);
94 struct hash_entry *entry =
95 _mesa_hash_table_search(ice->shaders.cache, keybox);
96
97 ralloc_free(keybox);
98
99 return entry ? entry->data : NULL;
100 }
101
102 void
iris_delete_shader_variant(struct iris_compiled_shader * shader)103 iris_delete_shader_variant(struct iris_compiled_shader *shader)
104 {
105 pipe_resource_reference(&shader->assembly.res, NULL);
106 util_queue_fence_destroy(&shader->ready);
107 ralloc_free(shader);
108 }
109
110 struct iris_compiled_shader *
iris_create_shader_variant(const struct iris_screen * screen,void * mem_ctx,enum iris_program_cache_id cache_id,uint32_t key_size,const void * key)111 iris_create_shader_variant(const struct iris_screen *screen,
112 void *mem_ctx,
113 enum iris_program_cache_id cache_id,
114 uint32_t key_size,
115 const void *key)
116 {
117 #ifndef NDEBUG
118 if (cache_id == IRIS_CACHE_BLORP) {
119 /* Blorp shader must have a mem_ctx. */
120 assert(mem_ctx != NULL);
121 } else if (cache_id == IRIS_CACHE_TCS) {
122 /* Pass-through tessellation control shaders (generated by the driver)
123 * will have a mem_ctx, and other tessellation control shaders will not.
124 */
125 } else {
126 /* Shaders that are neither blorp nor tessellation control must not have
127 * a mem_ctx.
128 */
129 assert(mem_ctx == NULL);
130 }
131 #endif
132
133 struct iris_compiled_shader *shader =
134 rzalloc_size(mem_ctx, sizeof(struct iris_compiled_shader) +
135 screen->vtbl.derived_program_state_size(cache_id));
136
137 pipe_reference_init(&shader->ref, 1);
138 util_queue_fence_init(&shader->ready);
139 util_queue_fence_reset(&shader->ready);
140
141 if (cache_id != IRIS_CACHE_BLORP) {
142 assert(key_size <= sizeof(union iris_any_prog_key));
143 memcpy(&shader->key, key, key_size);
144 }
145
146 return shader;
147 }
148
149 void
iris_upload_shader(struct iris_screen * screen,struct iris_uncompiled_shader * ish,struct iris_compiled_shader * shader,struct hash_table * driver_shaders,struct u_upload_mgr * uploader,enum iris_program_cache_id cache_id,uint32_t key_size,const void * key,const void * assembly)150 iris_upload_shader(struct iris_screen *screen,
151 struct iris_uncompiled_shader *ish,
152 struct iris_compiled_shader *shader,
153 struct hash_table *driver_shaders,
154 struct u_upload_mgr *uploader,
155 enum iris_program_cache_id cache_id,
156 uint32_t key_size,
157 const void *key,
158 const void *assembly)
159 {
160 const struct intel_device_info *devinfo = &screen->devinfo;
161
162 u_upload_alloc(uploader, 0, shader->prog_data->program_size, 64,
163 &shader->assembly.offset, &shader->assembly.res,
164 &shader->map);
165 memcpy(shader->map, assembly, shader->prog_data->program_size);
166
167 struct iris_resource *res = (void *) shader->assembly.res;
168 uint64_t shader_data_addr = res->bo->address +
169 shader->assembly.offset +
170 shader->prog_data->const_data_offset;
171
172 struct brw_shader_reloc_value reloc_values[] = {
173 {
174 .id = BRW_SHADER_RELOC_CONST_DATA_ADDR_LOW,
175 .value = shader_data_addr,
176 },
177 {
178 .id = BRW_SHADER_RELOC_CONST_DATA_ADDR_HIGH,
179 .value = shader_data_addr >> 32,
180 },
181 };
182 brw_write_shader_relocs(&screen->compiler->isa, shader->map,
183 shader->prog_data, reloc_values,
184 ARRAY_SIZE(reloc_values));
185
186 /* Store the 3DSTATE shader packets and other derived state. */
187 screen->vtbl.store_derived_program_state(devinfo, cache_id, shader);
188
189 util_queue_fence_signal(&shader->ready);
190
191 if (!ish) {
192 struct keybox *keybox = make_keybox(shader, cache_id, key, key_size);
193 _mesa_hash_table_insert(driver_shaders, keybox, shader);
194 }
195 }
196
197 bool
iris_blorp_lookup_shader(struct blorp_batch * blorp_batch,const void * key,uint32_t key_size,uint32_t * kernel_out,void * prog_data_out)198 iris_blorp_lookup_shader(struct blorp_batch *blorp_batch,
199 const void *key, uint32_t key_size,
200 uint32_t *kernel_out, void *prog_data_out)
201 {
202 struct blorp_context *blorp = blorp_batch->blorp;
203 struct iris_context *ice = blorp->driver_ctx;
204 struct iris_batch *batch = blorp_batch->driver_batch;
205 struct iris_compiled_shader *shader =
206 iris_find_cached_shader(ice, IRIS_CACHE_BLORP, key_size, key);
207
208 if (!shader)
209 return false;
210
211 struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
212 *kernel_out =
213 iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
214 *((void **) prog_data_out) = shader->prog_data;
215
216 iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE);
217
218 return true;
219 }
220
221 bool
iris_blorp_upload_shader(struct blorp_batch * blorp_batch,uint32_t stage,const void * key,uint32_t key_size,const void * kernel,UNUSED uint32_t kernel_size,const struct brw_stage_prog_data * prog_data_templ,UNUSED uint32_t prog_data_size,uint32_t * kernel_out,void * prog_data_out)222 iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage,
223 const void *key, uint32_t key_size,
224 const void *kernel, UNUSED uint32_t kernel_size,
225 const struct brw_stage_prog_data *prog_data_templ,
226 UNUSED uint32_t prog_data_size,
227 uint32_t *kernel_out, void *prog_data_out)
228 {
229 struct blorp_context *blorp = blorp_batch->blorp;
230 struct iris_context *ice = blorp->driver_ctx;
231 struct iris_batch *batch = blorp_batch->driver_batch;
232 struct iris_screen *screen = batch->screen;
233
234 void *prog_data = ralloc_size(NULL, prog_data_size);
235 memcpy(prog_data, prog_data_templ, prog_data_size);
236
237 struct iris_binding_table bt;
238 memset(&bt, 0, sizeof(bt));
239
240 struct iris_compiled_shader *shader =
241 iris_create_shader_variant(screen, ice->shaders.cache, IRIS_CACHE_BLORP,
242 key_size, key);
243
244 iris_finalize_program(shader, prog_data, NULL, NULL, 0, 0, 0, &bt);
245
246 iris_upload_shader(screen, NULL, shader, ice->shaders.cache,
247 ice->shaders.uploader_driver,
248 IRIS_CACHE_BLORP, key_size, key, kernel);
249
250 struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
251 *kernel_out =
252 iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
253 *((void **) prog_data_out) = shader->prog_data;
254
255 iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE);
256
257 return true;
258 }
259
260 void
iris_init_program_cache(struct iris_context * ice)261 iris_init_program_cache(struct iris_context *ice)
262 {
263 ice->shaders.cache =
264 _mesa_hash_table_create(ice, keybox_hash, keybox_equals);
265
266 ice->shaders.uploader_driver =
267 u_upload_create(&ice->ctx, 16384, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
268 IRIS_RESOURCE_FLAG_SHADER_MEMZONE |
269 IRIS_RESOURCE_FLAG_DEVICE_MEM);
270 ice->shaders.uploader_unsync =
271 u_upload_create(&ice->ctx, 16384, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
272 IRIS_RESOURCE_FLAG_SHADER_MEMZONE |
273 IRIS_RESOURCE_FLAG_DEVICE_MEM);
274 }
275
276 void
iris_destroy_program_cache(struct iris_context * ice)277 iris_destroy_program_cache(struct iris_context *ice)
278 {
279 for (int i = 0; i < MESA_SHADER_STAGES; i++) {
280 iris_shader_variant_reference(&ice->shaders.prog[i], NULL);
281 }
282 iris_shader_variant_reference(&ice->shaders.last_vue_shader, NULL);
283
284 hash_table_foreach(ice->shaders.cache, entry) {
285 struct iris_compiled_shader *shader = entry->data;
286 iris_delete_shader_variant(shader);
287 }
288
289 u_upload_destroy(ice->shaders.uploader_driver);
290 u_upload_destroy(ice->shaders.uploader_unsync);
291
292 ralloc_free(ice->shaders.cache);
293 }
294