1 /*
2 * Copyright © 2020 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "nir.h"
25 #include "nir_serialize.h"
26 #include "nir_spirv.h"
27 #include "util/mesa-sha1.h"
28
29 #ifdef DYNAMIC_LIBCLC_PATH
30 #include <fcntl.h>
31 #include <sys/types.h>
32 #include <sys/stat.h>
33 #include <sys/mman.h>
34 #include <unistd.h>
35 #endif
36
37 #ifdef HAVE_STATIC_LIBCLC_ZSTD
38 #include <zstd.h>
39 #endif
40
41 #ifdef HAVE_STATIC_LIBCLC_SPIRV
42 #include "spirv-mesa3d-.spv.h"
43 #endif
44
45 #ifdef HAVE_STATIC_LIBCLC_SPIRV64
46 #include "spirv64-mesa3d-.spv.h"
47 #endif
48
49 struct clc_file {
50 unsigned bit_size;
51 const char *static_data;
52 size_t static_data_size;
53 const char *sys_path;
54 };
55
56 static const struct clc_file libclc_files[] = {
57 {
58 .bit_size = 32,
59 #ifdef HAVE_STATIC_LIBCLC_SPIRV
60 .static_data = libclc_spirv_mesa3d_spv,
61 .static_data_size = sizeof(libclc_spirv_mesa3d_spv),
62 #endif
63 #ifdef DYNAMIC_LIBCLC_PATH
64 .sys_path = DYNAMIC_LIBCLC_PATH "spirv-mesa3d-.spv",
65 #endif
66 },
67 {
68 .bit_size = 64,
69 #ifdef HAVE_STATIC_LIBCLC_SPIRV64
70 .static_data = libclc_spirv64_mesa3d_spv,
71 .static_data_size = sizeof(libclc_spirv64_mesa3d_spv),
72 #endif
73 #ifdef DYNAMIC_LIBCLC_PATH
74 .sys_path = DYNAMIC_LIBCLC_PATH "spirv64-mesa3d-.spv",
75 #endif
76 },
77 };
78
79 static const struct clc_file *
get_libclc_file(unsigned ptr_bit_size)80 get_libclc_file(unsigned ptr_bit_size)
81 {
82 assert(ptr_bit_size == 32 || ptr_bit_size == 64);
83 return &libclc_files[ptr_bit_size / 64];
84 }
85
86 struct clc_data {
87 const struct clc_file *file;
88
89 unsigned char cache_key[20];
90
91 int fd;
92 const void *data;
93 size_t size;
94 };
95
96 static bool
open_clc_data(struct clc_data * clc,unsigned ptr_bit_size)97 open_clc_data(struct clc_data *clc, unsigned ptr_bit_size)
98 {
99 memset(clc, 0, sizeof(*clc));
100 clc->file = get_libclc_file(ptr_bit_size);
101 clc->fd = -1;
102
103 if (clc->file->static_data) {
104 snprintf((char *)clc->cache_key, sizeof(clc->cache_key),
105 "libclc-spirv%d", ptr_bit_size);
106 return true;
107 }
108
109 #ifdef DYNAMIC_LIBCLC_PATH
110 if (clc->file->sys_path != NULL) {
111 int fd = open(clc->file->sys_path, O_RDONLY);
112 if (fd < 0)
113 return false;
114
115 struct stat stat;
116 int ret = fstat(fd, &stat);
117 if (ret < 0) {
118 fprintf(stderr, "fstat failed on %s: %m\n", clc->file->sys_path);
119 close(fd);
120 return false;
121 }
122
123 struct mesa_sha1 ctx;
124 _mesa_sha1_init(&ctx);
125 _mesa_sha1_update(&ctx, clc->file->sys_path, strlen(clc->file->sys_path));
126 _mesa_sha1_update(&ctx, &stat.st_mtim, sizeof(stat.st_mtim));
127 _mesa_sha1_final(&ctx, clc->cache_key);
128
129 clc->fd = fd;
130
131 return true;
132 }
133 #endif
134
135 return false;
136 }
137
138 #define SPIRV_WORD_SIZE 4
139
140 static bool
map_clc_data(struct clc_data * clc)141 map_clc_data(struct clc_data *clc)
142 {
143 if (clc->file->static_data) {
144 #ifdef HAVE_STATIC_LIBCLC_ZSTD
145 unsigned long long cmp_size =
146 ZSTD_getFrameContentSize(clc->file->static_data,
147 clc->file->static_data_size);
148 if (cmp_size == ZSTD_CONTENTSIZE_UNKNOWN ||
149 cmp_size == ZSTD_CONTENTSIZE_ERROR) {
150 fprintf(stderr, "Could not determine the decompressed size of the "
151 "libclc SPIR-V\n");
152 return false;
153 }
154
155 size_t frame_size =
156 ZSTD_findFrameCompressedSize(clc->file->static_data,
157 clc->file->static_data_size);
158 if (ZSTD_isError(frame_size)) {
159 fprintf(stderr, "Could not determine the size of the first ZSTD frame "
160 "when decompressing libclc SPIR-V: %s\n",
161 ZSTD_getErrorName(frame_size));
162 return false;
163 }
164
165 void *dest = malloc(cmp_size + 1);
166 size_t size = ZSTD_decompress(dest, cmp_size, clc->file->static_data,
167 frame_size);
168 if (ZSTD_isError(size)) {
169 free(dest);
170 fprintf(stderr, "Error decompressing libclc SPIR-V: %s\n",
171 ZSTD_getErrorName(size));
172 return false;
173 }
174
175 clc->data = dest;
176 clc->size = size;
177 #else
178 clc->data = clc->file->static_data;
179 clc->size = clc->file->static_data_size;
180 #endif
181 return true;
182 }
183
184 #ifdef DYNAMIC_LIBCLC_PATH
185 if (clc->file->sys_path != NULL) {
186 off_t len = lseek(clc->fd, 0, SEEK_END);
187 if (len % SPIRV_WORD_SIZE != 0) {
188 fprintf(stderr, "File length isn't a multiple of the word size\n");
189 return false;
190 }
191 clc->size = len;
192
193 clc->data = mmap(NULL, len, PROT_READ, MAP_PRIVATE, clc->fd, 0);
194 if (clc->data == MAP_FAILED) {
195 fprintf(stderr, "Failed to mmap libclc SPIR-V: %m\n");
196 return false;
197 }
198
199 return true;
200 }
201 #endif
202
203 return true;
204 }
205
206 static void
close_clc_data(struct clc_data * clc)207 close_clc_data(struct clc_data *clc)
208 {
209 if (clc->file->static_data) {
210 #ifdef HAVE_STATIC_LIBCLC_ZSTD
211 free((void *)clc->data);
212 #endif
213 return;
214 }
215
216 #ifdef DYNAMIC_LIBCLC_PATH
217 if (clc->file->sys_path != NULL) {
218 if (clc->data)
219 munmap((void *)clc->data, clc->size);
220 close(clc->fd);
221 }
222 #endif
223 }
224
225 /** Returns true if libclc is found
226 *
227 * If libclc is compiled in statically, this always returns true. If we
228 * depend on a dynamic libclc, this opens and tries to stat the file.
229 */
230 bool
nir_can_find_libclc(unsigned ptr_bit_size)231 nir_can_find_libclc(unsigned ptr_bit_size)
232 {
233 struct clc_data clc;
234 if (open_clc_data(&clc, ptr_bit_size)) {
235 close_clc_data(&clc);
236 return true;
237 } else {
238 return false;
239 }
240 }
241
242 /** Adds generic pointer variants of libclc functions
243 *
244 * Libclc currently doesn't contain generic variants for a bunch of functions
245 * like `frexp` but the OpenCL spec with generic pointers requires them. We
246 * really should fix libclc but, in the mean time, we can easily duplicate
247 * every function that works on global memory and make it also work on generic
248 * memory.
249 */
250 static void
libclc_add_generic_variants(nir_shader * shader)251 libclc_add_generic_variants(nir_shader *shader)
252 {
253 nir_foreach_function(func, shader) {
254 /* These don't need generic variants */
255 if (strstr(func->name, "async_work_group_strided_copy"))
256 continue;
257
258 char *U3AS1 = strstr(func->name, "U3AS1");
259 if (U3AS1 == NULL)
260 continue;
261
262 ptrdiff_t offset_1 = U3AS1 - func->name + 4;
263 assert(offset_1 < strlen(func->name) && func->name[offset_1] == '1');
264
265 char *generic_name = ralloc_strdup(shader, func->name);
266 assert(generic_name[offset_1] == '1');
267 generic_name[offset_1] = '4';
268
269 nir_function *gfunc = nir_function_create(shader, generic_name);
270 gfunc->num_params = func->num_params;
271 gfunc->params = ralloc_array(shader, nir_parameter, gfunc->num_params);
272 for (unsigned i = 0; i < gfunc->num_params; i++)
273 gfunc->params[i] = func->params[i];
274
275 gfunc->impl = nir_function_impl_clone(shader, func->impl);
276 gfunc->impl->function = gfunc;
277
278 /* Rewrite any global pointers to generic */
279 nir_foreach_block(block, gfunc->impl) {
280 nir_foreach_instr(instr, block) {
281 if (instr->type != nir_instr_type_deref)
282 continue;
283
284 nir_deref_instr *deref = nir_instr_as_deref(instr);
285 if (!nir_deref_mode_may_be(deref, nir_var_mem_global))
286 continue;
287
288 assert(deref->type != nir_deref_type_var);
289 assert(nir_deref_mode_is(deref, nir_var_mem_global));
290
291 deref->modes = nir_var_mem_generic;
292 }
293 }
294
295 nir_metadata_preserve(gfunc->impl, nir_metadata_none);
296 }
297 }
298
299 nir_shader *
nir_load_libclc_shader(unsigned ptr_bit_size,struct disk_cache * disk_cache,const struct spirv_to_nir_options * spirv_options,const nir_shader_compiler_options * nir_options)300 nir_load_libclc_shader(unsigned ptr_bit_size,
301 struct disk_cache *disk_cache,
302 const struct spirv_to_nir_options *spirv_options,
303 const nir_shader_compiler_options *nir_options)
304 {
305 assert(ptr_bit_size ==
306 nir_address_format_bit_size(spirv_options->global_addr_format));
307
308 struct clc_data clc;
309 if (!open_clc_data(&clc, ptr_bit_size))
310 return NULL;
311
312 #ifdef ENABLE_SHADER_CACHE
313 cache_key cache_key;
314 if (disk_cache) {
315 disk_cache_compute_key(disk_cache, clc.cache_key,
316 sizeof(clc.cache_key), cache_key);
317
318 size_t buffer_size;
319 uint8_t *buffer = disk_cache_get(disk_cache, cache_key, &buffer_size);
320 if (buffer) {
321 struct blob_reader blob;
322 blob_reader_init(&blob, buffer, buffer_size);
323 nir_shader *nir = nir_deserialize(NULL, nir_options, &blob);
324 free(buffer);
325 close_clc_data(&clc);
326 return nir;
327 }
328 }
329 #endif
330
331 if (!map_clc_data(&clc)) {
332 close_clc_data(&clc);
333 return NULL;
334 }
335
336 struct spirv_to_nir_options spirv_lib_options = *spirv_options;
337 spirv_lib_options.create_library = true;
338
339 assert(clc.size % SPIRV_WORD_SIZE == 0);
340 nir_shader *nir = spirv_to_nir(clc.data, clc.size / SPIRV_WORD_SIZE,
341 NULL, 0, MESA_SHADER_KERNEL, NULL,
342 &spirv_lib_options, nir_options);
343 nir_validate_shader(nir, "after nir_load_clc_shader");
344
345 /* nir_inline_libclc will assume that the functions in this shader are
346 * already ready to lower. This means we need to inline any function_temp
347 * initializers and lower any early returns.
348 */
349 nir->info.internal = true;
350 NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp);
351 NIR_PASS_V(nir, nir_lower_returns);
352
353 NIR_PASS_V(nir, libclc_add_generic_variants);
354
355 /* TODO: One day, we may want to run some optimizations on the libclc
356 * shader once and cache them to save time in each shader call.
357 */
358
359 #ifdef ENABLE_SHADER_CACHE
360 if (disk_cache) {
361 struct blob blob;
362 blob_init(&blob);
363 nir_serialize(&blob, nir, false);
364 disk_cache_put(disk_cache, cache_key, blob.data, blob.size, NULL);
365 }
366 #endif
367
368 close_clc_data(&clc);
369 return nir;
370 }
371