1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "util/blob.h"
25 #include "util/hash_table.h"
26 #include "util/debug.h"
27 #include "util/disk_cache.h"
28 #include "util/mesa-sha1.h"
29 #include "nir/nir_serialize.h"
30 #include "anv_private.h"
31 #include "nir/nir_xfb_info.h"
32 #include "vulkan/util/vk_util.h"
33
34 static bool
35 anv_shader_bin_serialize(struct vk_pipeline_cache_object *object,
36 struct blob *blob);
37
38 struct vk_pipeline_cache_object *
39 anv_shader_bin_deserialize(struct vk_device *device,
40 const void *key_data, size_t key_size,
41 struct blob_reader *blob);
42
43 static void
anv_shader_bin_destroy(struct vk_pipeline_cache_object * object)44 anv_shader_bin_destroy(struct vk_pipeline_cache_object *object)
45 {
46 struct anv_device *device =
47 container_of(object->device, struct anv_device, vk);
48 struct anv_shader_bin *shader =
49 container_of(object, struct anv_shader_bin, base);
50
51 anv_state_pool_free(&device->instruction_state_pool, shader->kernel);
52 vk_pipeline_cache_object_finish(&shader->base);
53 vk_free(&device->vk.alloc, shader);
54 }
55
56 static const struct vk_pipeline_cache_object_ops anv_shader_bin_ops = {
57 .serialize = anv_shader_bin_serialize,
58 .deserialize = anv_shader_bin_deserialize,
59 .destroy = anv_shader_bin_destroy,
60 };
61
62 const struct vk_pipeline_cache_object_ops *const anv_cache_import_ops[2] = {
63 &anv_shader_bin_ops,
64 NULL
65 };
66
67 struct anv_shader_bin *
anv_shader_bin_create(struct anv_device * device,gl_shader_stage stage,const void * key_data,uint32_t key_size,const void * kernel_data,uint32_t kernel_size,const struct brw_stage_prog_data * prog_data_in,uint32_t prog_data_size,const struct brw_compile_stats * stats,uint32_t num_stats,const nir_xfb_info * xfb_info_in,const struct anv_pipeline_bind_map * bind_map)68 anv_shader_bin_create(struct anv_device *device,
69 gl_shader_stage stage,
70 const void *key_data, uint32_t key_size,
71 const void *kernel_data, uint32_t kernel_size,
72 const struct brw_stage_prog_data *prog_data_in,
73 uint32_t prog_data_size,
74 const struct brw_compile_stats *stats, uint32_t num_stats,
75 const nir_xfb_info *xfb_info_in,
76 const struct anv_pipeline_bind_map *bind_map)
77 {
78 VK_MULTIALLOC(ma);
79 VK_MULTIALLOC_DECL(&ma, struct anv_shader_bin, shader, 1);
80 VK_MULTIALLOC_DECL_SIZE(&ma, void, obj_key_data, key_size);
81 VK_MULTIALLOC_DECL_SIZE(&ma, struct brw_stage_prog_data, prog_data,
82 prog_data_size);
83 VK_MULTIALLOC_DECL(&ma, struct brw_shader_reloc, prog_data_relocs,
84 prog_data_in->num_relocs);
85 VK_MULTIALLOC_DECL(&ma, uint32_t, prog_data_param, prog_data_in->nr_params);
86
87 VK_MULTIALLOC_DECL_SIZE(&ma, nir_xfb_info, xfb_info,
88 xfb_info_in == NULL ? 0 :
89 nir_xfb_info_size(xfb_info_in->output_count));
90
91 VK_MULTIALLOC_DECL(&ma, struct anv_pipeline_binding, surface_to_descriptor,
92 bind_map->surface_count);
93 VK_MULTIALLOC_DECL(&ma, struct anv_pipeline_binding, sampler_to_descriptor,
94 bind_map->sampler_count);
95
96 if (!vk_multialloc_alloc(&ma, &device->vk.alloc,
97 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE))
98 return NULL;
99
100 memcpy(obj_key_data, key_data, key_size);
101 vk_pipeline_cache_object_init(&device->vk, &shader->base,
102 &anv_shader_bin_ops, obj_key_data, key_size);
103
104 shader->stage = stage;
105
106 shader->kernel =
107 anv_state_pool_alloc(&device->instruction_state_pool, kernel_size, 64);
108 memcpy(shader->kernel.map, kernel_data, kernel_size);
109 shader->kernel_size = kernel_size;
110
111 uint64_t shader_data_addr = INSTRUCTION_STATE_POOL_MIN_ADDRESS +
112 shader->kernel.offset +
113 prog_data_in->const_data_offset;
114
115 int rv_count = 0;
116 struct brw_shader_reloc_value reloc_values[5];
117 reloc_values[rv_count++] = (struct brw_shader_reloc_value) {
118 .id = BRW_SHADER_RELOC_CONST_DATA_ADDR_LOW,
119 .value = shader_data_addr,
120 };
121 reloc_values[rv_count++] = (struct brw_shader_reloc_value) {
122 .id = BRW_SHADER_RELOC_CONST_DATA_ADDR_HIGH,
123 .value = shader_data_addr >> 32,
124 };
125 reloc_values[rv_count++] = (struct brw_shader_reloc_value) {
126 .id = BRW_SHADER_RELOC_SHADER_START_OFFSET,
127 .value = shader->kernel.offset,
128 };
129 if (brw_shader_stage_is_bindless(stage)) {
130 const struct brw_bs_prog_data *bs_prog_data =
131 brw_bs_prog_data_const(prog_data_in);
132 uint64_t resume_sbt_addr = INSTRUCTION_STATE_POOL_MIN_ADDRESS +
133 shader->kernel.offset +
134 bs_prog_data->resume_sbt_offset;
135 reloc_values[rv_count++] = (struct brw_shader_reloc_value) {
136 .id = BRW_SHADER_RELOC_RESUME_SBT_ADDR_LOW,
137 .value = resume_sbt_addr,
138 };
139 reloc_values[rv_count++] = (struct brw_shader_reloc_value) {
140 .id = BRW_SHADER_RELOC_RESUME_SBT_ADDR_HIGH,
141 .value = resume_sbt_addr >> 32,
142 };
143 }
144
145 brw_write_shader_relocs(&device->physical->compiler->isa,
146 shader->kernel.map, prog_data_in,
147 reloc_values, rv_count);
148
149 memcpy(prog_data, prog_data_in, prog_data_size);
150 typed_memcpy(prog_data_relocs, prog_data_in->relocs,
151 prog_data_in->num_relocs);
152 prog_data->relocs = prog_data_relocs;
153 memset(prog_data_param, 0,
154 prog_data->nr_params * sizeof(*prog_data_param));
155 prog_data->param = prog_data_param;
156 shader->prog_data = prog_data;
157 shader->prog_data_size = prog_data_size;
158
159 assert(num_stats <= ARRAY_SIZE(shader->stats));
160 typed_memcpy(shader->stats, stats, num_stats);
161 shader->num_stats = num_stats;
162
163 if (xfb_info_in) {
164 *xfb_info = *xfb_info_in;
165 typed_memcpy(xfb_info->outputs, xfb_info_in->outputs,
166 xfb_info_in->output_count);
167 shader->xfb_info = xfb_info;
168 } else {
169 shader->xfb_info = NULL;
170 }
171
172 shader->bind_map = *bind_map;
173 typed_memcpy(surface_to_descriptor, bind_map->surface_to_descriptor,
174 bind_map->surface_count);
175 shader->bind_map.surface_to_descriptor = surface_to_descriptor;
176 typed_memcpy(sampler_to_descriptor, bind_map->sampler_to_descriptor,
177 bind_map->sampler_count);
178 shader->bind_map.sampler_to_descriptor = sampler_to_descriptor;
179
180 return shader;
181 }
182
183 static bool
anv_shader_bin_serialize(struct vk_pipeline_cache_object * object,struct blob * blob)184 anv_shader_bin_serialize(struct vk_pipeline_cache_object *object,
185 struct blob *blob)
186 {
187 struct anv_shader_bin *shader =
188 container_of(object, struct anv_shader_bin, base);
189
190 blob_write_uint32(blob, shader->stage);
191
192 blob_write_uint32(blob, shader->kernel_size);
193 blob_write_bytes(blob, shader->kernel.map, shader->kernel_size);
194
195 blob_write_uint32(blob, shader->prog_data_size);
196 blob_write_bytes(blob, shader->prog_data, shader->prog_data_size);
197 blob_write_bytes(blob, shader->prog_data->relocs,
198 shader->prog_data->num_relocs *
199 sizeof(shader->prog_data->relocs[0]));
200
201 blob_write_uint32(blob, shader->num_stats);
202 blob_write_bytes(blob, shader->stats,
203 shader->num_stats * sizeof(shader->stats[0]));
204
205 if (shader->xfb_info) {
206 uint32_t xfb_info_size =
207 nir_xfb_info_size(shader->xfb_info->output_count);
208 blob_write_uint32(blob, xfb_info_size);
209 blob_write_bytes(blob, shader->xfb_info, xfb_info_size);
210 } else {
211 blob_write_uint32(blob, 0);
212 }
213
214 blob_write_bytes(blob, shader->bind_map.surface_sha1,
215 sizeof(shader->bind_map.surface_sha1));
216 blob_write_bytes(blob, shader->bind_map.sampler_sha1,
217 sizeof(shader->bind_map.sampler_sha1));
218 blob_write_bytes(blob, shader->bind_map.push_sha1,
219 sizeof(shader->bind_map.push_sha1));
220 blob_write_uint32(blob, shader->bind_map.surface_count);
221 blob_write_uint32(blob, shader->bind_map.sampler_count);
222 blob_write_bytes(blob, shader->bind_map.surface_to_descriptor,
223 shader->bind_map.surface_count *
224 sizeof(*shader->bind_map.surface_to_descriptor));
225 blob_write_bytes(blob, shader->bind_map.sampler_to_descriptor,
226 shader->bind_map.sampler_count *
227 sizeof(*shader->bind_map.sampler_to_descriptor));
228 blob_write_bytes(blob, shader->bind_map.push_ranges,
229 sizeof(shader->bind_map.push_ranges));
230
231 return !blob->out_of_memory;
232 }
233
234 struct vk_pipeline_cache_object *
anv_shader_bin_deserialize(struct vk_device * vk_device,const void * key_data,size_t key_size,struct blob_reader * blob)235 anv_shader_bin_deserialize(struct vk_device *vk_device,
236 const void *key_data, size_t key_size,
237 struct blob_reader *blob)
238 {
239 struct anv_device *device =
240 container_of(vk_device, struct anv_device, vk);
241
242 gl_shader_stage stage = blob_read_uint32(blob);
243
244 uint32_t kernel_size = blob_read_uint32(blob);
245 const void *kernel_data = blob_read_bytes(blob, kernel_size);
246
247 uint32_t prog_data_size = blob_read_uint32(blob);
248 const void *prog_data_bytes = blob_read_bytes(blob, prog_data_size);
249 if (blob->overrun)
250 return NULL;
251
252 union brw_any_prog_data prog_data;
253 memcpy(&prog_data, prog_data_bytes,
254 MIN2(sizeof(prog_data), prog_data_size));
255 prog_data.base.relocs =
256 blob_read_bytes(blob, prog_data.base.num_relocs *
257 sizeof(prog_data.base.relocs[0]));
258
259 uint32_t num_stats = blob_read_uint32(blob);
260 const struct brw_compile_stats *stats =
261 blob_read_bytes(blob, num_stats * sizeof(stats[0]));
262
263 const nir_xfb_info *xfb_info = NULL;
264 uint32_t xfb_size = blob_read_uint32(blob);
265 if (xfb_size)
266 xfb_info = blob_read_bytes(blob, xfb_size);
267
268 struct anv_pipeline_bind_map bind_map;
269 blob_copy_bytes(blob, bind_map.surface_sha1, sizeof(bind_map.surface_sha1));
270 blob_copy_bytes(blob, bind_map.sampler_sha1, sizeof(bind_map.sampler_sha1));
271 blob_copy_bytes(blob, bind_map.push_sha1, sizeof(bind_map.push_sha1));
272 bind_map.surface_count = blob_read_uint32(blob);
273 bind_map.sampler_count = blob_read_uint32(blob);
274 bind_map.surface_to_descriptor = (void *)
275 blob_read_bytes(blob, bind_map.surface_count *
276 sizeof(*bind_map.surface_to_descriptor));
277 bind_map.sampler_to_descriptor = (void *)
278 blob_read_bytes(blob, bind_map.sampler_count *
279 sizeof(*bind_map.sampler_to_descriptor));
280 blob_copy_bytes(blob, bind_map.push_ranges, sizeof(bind_map.push_ranges));
281
282 if (blob->overrun)
283 return NULL;
284
285 struct anv_shader_bin *shader =
286 anv_shader_bin_create(device, stage,
287 key_data, key_size,
288 kernel_data, kernel_size,
289 &prog_data.base, prog_data_size,
290 stats, num_stats, xfb_info, &bind_map);
291 if (shader == NULL)
292 return NULL;
293
294 return &shader->base;
295 }
296
297 struct anv_shader_bin *
anv_device_search_for_kernel(struct anv_device * device,struct vk_pipeline_cache * cache,const void * key_data,uint32_t key_size,bool * user_cache_hit)298 anv_device_search_for_kernel(struct anv_device *device,
299 struct vk_pipeline_cache *cache,
300 const void *key_data, uint32_t key_size,
301 bool *user_cache_hit)
302 {
303 /* Use the default pipeline cache if none is specified */
304 if (cache == NULL)
305 cache = device->default_pipeline_cache;
306
307 bool cache_hit = false;
308 struct vk_pipeline_cache_object *object =
309 vk_pipeline_cache_lookup_object(cache, key_data, key_size,
310 &anv_shader_bin_ops, &cache_hit);
311 if (user_cache_hit != NULL) {
312 *user_cache_hit = object != NULL && cache_hit &&
313 cache != device->default_pipeline_cache;
314 }
315 if (object == NULL)
316 return NULL;
317
318 return container_of(object, struct anv_shader_bin, base);
319 }
320
321 struct anv_shader_bin *
anv_device_upload_kernel(struct anv_device * device,struct vk_pipeline_cache * cache,gl_shader_stage stage,const void * key_data,uint32_t key_size,const void * kernel_data,uint32_t kernel_size,const struct brw_stage_prog_data * prog_data,uint32_t prog_data_size,const struct brw_compile_stats * stats,uint32_t num_stats,const nir_xfb_info * xfb_info,const struct anv_pipeline_bind_map * bind_map)322 anv_device_upload_kernel(struct anv_device *device,
323 struct vk_pipeline_cache *cache,
324 gl_shader_stage stage,
325 const void *key_data, uint32_t key_size,
326 const void *kernel_data, uint32_t kernel_size,
327 const struct brw_stage_prog_data *prog_data,
328 uint32_t prog_data_size,
329 const struct brw_compile_stats *stats,
330 uint32_t num_stats,
331 const nir_xfb_info *xfb_info,
332 const struct anv_pipeline_bind_map *bind_map)
333 {
334 /* Use the default pipeline cache if none is specified */
335 if (cache == NULL)
336 cache = device->default_pipeline_cache;
337
338 struct anv_shader_bin *shader =
339 anv_shader_bin_create(device, stage,
340 key_data, key_size,
341 kernel_data, kernel_size,
342 prog_data, prog_data_size,
343 stats, num_stats,
344 xfb_info, bind_map);
345 if (shader == NULL)
346 return NULL;
347
348 struct vk_pipeline_cache_object *cached =
349 vk_pipeline_cache_add_object(cache, &shader->base);
350
351 return container_of(cached, struct anv_shader_bin, base);
352 }
353
354 #define SHA1_KEY_SIZE 20
355
356 struct nir_shader *
anv_device_search_for_nir(struct anv_device * device,struct vk_pipeline_cache * cache,const nir_shader_compiler_options * nir_options,unsigned char sha1_key[SHA1_KEY_SIZE],void * mem_ctx)357 anv_device_search_for_nir(struct anv_device *device,
358 struct vk_pipeline_cache *cache,
359 const nir_shader_compiler_options *nir_options,
360 unsigned char sha1_key[SHA1_KEY_SIZE],
361 void *mem_ctx)
362 {
363 if (cache == NULL)
364 cache = device->default_pipeline_cache;
365
366 return vk_pipeline_cache_lookup_nir(cache, sha1_key, SHA1_KEY_SIZE,
367 nir_options, NULL, mem_ctx);
368 }
369
370 void
anv_device_upload_nir(struct anv_device * device,struct vk_pipeline_cache * cache,const struct nir_shader * nir,unsigned char sha1_key[SHA1_KEY_SIZE])371 anv_device_upload_nir(struct anv_device *device,
372 struct vk_pipeline_cache *cache,
373 const struct nir_shader *nir,
374 unsigned char sha1_key[SHA1_KEY_SIZE])
375 {
376 if (cache == NULL)
377 cache = device->default_pipeline_cache;
378
379 vk_pipeline_cache_add_nir(cache, sha1_key, SHA1_KEY_SIZE, nir);
380 }
381