1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28
29 /**
30 * @file
31 * Helpers for emiting intrinsic calls.
32 *
33 * LLVM vanilla IR doesn't represent all basic arithmetic operations we care
34 * about, and it is often necessary to resort target-specific intrinsics for
35 * performance, convenience.
36 *
37 * Ideally we would like to stay away from target specific intrinsics and
38 * move all the instruction selection logic into upstream LLVM where it belongs.
39 *
40 * These functions are also used for calling C functions provided by us from
41 * generated LLVM code.
42 *
43 * @author Jose Fonseca <jfonseca@vmware.com>
44 */
45
46 #include <llvm/Config/llvm-config.h>
47
48 #include "util/u_debug.h"
49 #include "util/u_string.h"
50 #include "util/bitscan.h"
51
52 #include "lp_bld_const.h"
53 #include "lp_bld_intr.h"
54 #include "lp_bld_type.h"
55 #include "lp_bld_pack.h"
56 #include "lp_bld_debug.h"
57
58
59 void
lp_format_intrinsic(char * name,size_t size,const char * name_root,LLVMTypeRef type)60 lp_format_intrinsic(char *name,
61 size_t size,
62 const char *name_root,
63 LLVMTypeRef type)
64 {
65 unsigned length = 0;
66 unsigned width;
67 char c;
68
69 LLVMTypeKind kind = LLVMGetTypeKind(type);
70 if (kind == LLVMVectorTypeKind) {
71 length = LLVMGetVectorSize(type);
72 type = LLVMGetElementType(type);
73 kind = LLVMGetTypeKind(type);
74 }
75
76 switch (kind) {
77 case LLVMIntegerTypeKind:
78 c = 'i';
79 width = LLVMGetIntTypeWidth(type);
80 break;
81 case LLVMFloatTypeKind:
82 c = 'f';
83 width = 32;
84 break;
85 case LLVMDoubleTypeKind:
86 c = 'f';
87 width = 64;
88 break;
89 case LLVMHalfTypeKind:
90 c = 'f';
91 width = 16;
92 break;
93 default:
94 unreachable("unexpected LLVMTypeKind");
95 }
96
97 if (length) {
98 snprintf(name, size, "%s.v%u%c%u", name_root, length, c, width);
99 } else {
100 snprintf(name, size, "%s.%c%u", name_root, c, width);
101 }
102 }
103
104
105 LLVMValueRef
lp_declare_intrinsic(LLVMModuleRef module,const char * name,LLVMTypeRef ret_type,LLVMTypeRef * arg_types,unsigned num_args)106 lp_declare_intrinsic(LLVMModuleRef module,
107 const char *name,
108 LLVMTypeRef ret_type,
109 LLVMTypeRef *arg_types,
110 unsigned num_args)
111 {
112 LLVMTypeRef function_type;
113 LLVMValueRef function;
114
115 assert(!LLVMGetNamedFunction(module, name));
116
117 function_type = LLVMFunctionType(ret_type, arg_types, num_args, 0);
118 function = LLVMAddFunction(module, name, function_type);
119
120 LLVMSetFunctionCallConv(function, LLVMCCallConv);
121 LLVMSetLinkage(function, LLVMExternalLinkage);
122
123 assert(LLVMIsDeclaration(function));
124
125 return function;
126 }
127
128
129 #if LLVM_VERSION_MAJOR < 4
lp_attr_to_llvm_attr(enum lp_func_attr attr)130 static LLVMAttribute lp_attr_to_llvm_attr(enum lp_func_attr attr)
131 {
132 switch (attr) {
133 case LP_FUNC_ATTR_ALWAYSINLINE: return LLVMAlwaysInlineAttribute;
134 case LP_FUNC_ATTR_INREG: return LLVMInRegAttribute;
135 case LP_FUNC_ATTR_NOALIAS: return LLVMNoAliasAttribute;
136 case LP_FUNC_ATTR_NOUNWIND: return LLVMNoUnwindAttribute;
137 case LP_FUNC_ATTR_READNONE: return LLVMReadNoneAttribute;
138 case LP_FUNC_ATTR_READONLY: return LLVMReadOnlyAttribute;
139 default:
140 _debug_printf("Unhandled function attribute: %x\n", attr);
141 return 0;
142 }
143 }
144
145 #else
146
attr_to_str(enum lp_func_attr attr)147 static const char *attr_to_str(enum lp_func_attr attr)
148 {
149 switch (attr) {
150 case LP_FUNC_ATTR_ALWAYSINLINE: return "alwaysinline";
151 case LP_FUNC_ATTR_INREG: return "inreg";
152 case LP_FUNC_ATTR_NOALIAS: return "noalias";
153 case LP_FUNC_ATTR_NOUNWIND: return "nounwind";
154 case LP_FUNC_ATTR_READNONE: return "readnone";
155 case LP_FUNC_ATTR_READONLY: return "readonly";
156 case LP_FUNC_ATTR_WRITEONLY: return "writeonly";
157 case LP_FUNC_ATTR_INACCESSIBLE_MEM_ONLY: return "inaccessiblememonly";
158 case LP_FUNC_ATTR_CONVERGENT: return "convergent";
159 default:
160 _debug_printf("Unhandled function attribute: %x\n", attr);
161 return 0;
162 }
163 }
164
165 #endif
166
167 void
lp_add_function_attr(LLVMValueRef function_or_call,int attr_idx,enum lp_func_attr attr)168 lp_add_function_attr(LLVMValueRef function_or_call,
169 int attr_idx, enum lp_func_attr attr)
170 {
171
172 #if LLVM_VERSION_MAJOR < 4
173 LLVMAttribute llvm_attr = lp_attr_to_llvm_attr(attr);
174 if (LLVMIsAFunction(function_or_call)) {
175 if (attr_idx == -1) {
176 LLVMAddFunctionAttr(function_or_call, llvm_attr);
177 } else {
178 LLVMAddAttribute(LLVMGetParam(function_or_call, attr_idx - 1), llvm_attr);
179 }
180 } else {
181 LLVMAddInstrAttribute(function_or_call, attr_idx, llvm_attr);
182 }
183 #else
184
185 LLVMModuleRef module;
186 if (LLVMIsAFunction(function_or_call)) {
187 module = LLVMGetGlobalParent(function_or_call);
188 } else {
189 LLVMBasicBlockRef bb = LLVMGetInstructionParent(function_or_call);
190 LLVMValueRef function = LLVMGetBasicBlockParent(bb);
191 module = LLVMGetGlobalParent(function);
192 }
193 LLVMContextRef ctx = LLVMGetModuleContext(module);
194
195 const char *attr_name = attr_to_str(attr);
196 unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name,
197 strlen(attr_name));
198 LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(ctx, kind_id, 0);
199
200 if (LLVMIsAFunction(function_or_call))
201 LLVMAddAttributeAtIndex(function_or_call, attr_idx, llvm_attr);
202 else
203 LLVMAddCallSiteAttribute(function_or_call, attr_idx, llvm_attr);
204 #endif
205 }
206
207 static void
lp_add_func_attributes(LLVMValueRef function,unsigned attrib_mask)208 lp_add_func_attributes(LLVMValueRef function, unsigned attrib_mask)
209 {
210 /* NoUnwind indicates that the intrinsic never raises a C++ exception.
211 * Set it for all intrinsics.
212 */
213 attrib_mask |= LP_FUNC_ATTR_NOUNWIND;
214 attrib_mask &= ~LP_FUNC_ATTR_LEGACY;
215
216 while (attrib_mask) {
217 enum lp_func_attr attr = 1u << u_bit_scan(&attrib_mask);
218 lp_add_function_attr(function, -1, attr);
219 }
220 }
221
222 LLVMValueRef
lp_build_intrinsic(LLVMBuilderRef builder,const char * name,LLVMTypeRef ret_type,LLVMValueRef * args,unsigned num_args,unsigned attr_mask)223 lp_build_intrinsic(LLVMBuilderRef builder,
224 const char *name,
225 LLVMTypeRef ret_type,
226 LLVMValueRef *args,
227 unsigned num_args,
228 unsigned attr_mask)
229 {
230 LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
231 LLVMValueRef function, call;
232 bool set_callsite_attrs = LLVM_VERSION_MAJOR >= 4 &&
233 !(attr_mask & LP_FUNC_ATTR_LEGACY);
234
235 function = LLVMGetNamedFunction(module, name);
236 if(!function) {
237 LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS];
238 unsigned i;
239
240 assert(num_args <= LP_MAX_FUNC_ARGS);
241
242 for(i = 0; i < num_args; ++i) {
243 assert(args[i]);
244 arg_types[i] = LLVMTypeOf(args[i]);
245 }
246
247 function = lp_declare_intrinsic(module, name, ret_type, arg_types, num_args);
248
249 /*
250 * If llvm removes an intrinsic we use, we'll hit this abort (rather
251 * than a call to address zero in the jited code).
252 */
253 if (LLVMGetIntrinsicID(function) == 0) {
254 _debug_printf("llvm (version " MESA_LLVM_VERSION_STRING
255 ") found no intrinsic for %s, going to crash...\n",
256 name);
257 abort();
258 }
259
260 if (!set_callsite_attrs)
261 lp_add_func_attributes(function, attr_mask);
262
263 if (gallivm_debug & GALLIVM_DEBUG_IR) {
264 lp_debug_dump_value(function);
265 }
266 }
267
268 call = LLVMBuildCall(builder, function, args, num_args, "");
269 if (set_callsite_attrs)
270 lp_add_func_attributes(call, attr_mask);
271 return call;
272 }
273
274
275 LLVMValueRef
lp_build_intrinsic_unary(LLVMBuilderRef builder,const char * name,LLVMTypeRef ret_type,LLVMValueRef a)276 lp_build_intrinsic_unary(LLVMBuilderRef builder,
277 const char *name,
278 LLVMTypeRef ret_type,
279 LLVMValueRef a)
280 {
281 return lp_build_intrinsic(builder, name, ret_type, &a, 1, 0);
282 }
283
284
285 LLVMValueRef
lp_build_intrinsic_binary(LLVMBuilderRef builder,const char * name,LLVMTypeRef ret_type,LLVMValueRef a,LLVMValueRef b)286 lp_build_intrinsic_binary(LLVMBuilderRef builder,
287 const char *name,
288 LLVMTypeRef ret_type,
289 LLVMValueRef a,
290 LLVMValueRef b)
291 {
292 LLVMValueRef args[2];
293
294 args[0] = a;
295 args[1] = b;
296
297 return lp_build_intrinsic(builder, name, ret_type, args, 2, 0);
298 }
299
300
301 /**
302 * Call intrinsic with arguments adapted to intrinsic vector length.
303 *
304 * Split vectors which are too large for the hw, or expand them if they
305 * are too small, so a caller calling a function which might use intrinsics
306 * doesn't need to do splitting/expansion on its own.
307 * This only supports intrinsics where src and dst types match.
308 */
309 LLVMValueRef
lp_build_intrinsic_binary_anylength(struct gallivm_state * gallivm,const char * name,struct lp_type src_type,unsigned intr_size,LLVMValueRef a,LLVMValueRef b)310 lp_build_intrinsic_binary_anylength(struct gallivm_state *gallivm,
311 const char *name,
312 struct lp_type src_type,
313 unsigned intr_size,
314 LLVMValueRef a,
315 LLVMValueRef b)
316 {
317 unsigned i;
318 struct lp_type intrin_type = src_type;
319 LLVMBuilderRef builder = gallivm->builder;
320 LLVMValueRef i32undef = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
321 LLVMValueRef anative, bnative;
322 unsigned intrin_length = intr_size / src_type.width;
323
324 intrin_type.length = intrin_length;
325
326 if (intrin_length > src_type.length) {
327 LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
328 LLVMValueRef constvec, tmp;
329
330 for (i = 0; i < src_type.length; i++) {
331 elems[i] = lp_build_const_int32(gallivm, i);
332 }
333 for (; i < intrin_length; i++) {
334 elems[i] = i32undef;
335 }
336 if (src_type.length == 1) {
337 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, intrin_type);
338 a = LLVMBuildBitCast(builder, a, LLVMVectorType(elem_type, 1), "");
339 b = LLVMBuildBitCast(builder, b, LLVMVectorType(elem_type, 1), "");
340 }
341 constvec = LLVMConstVector(elems, intrin_length);
342 anative = LLVMBuildShuffleVector(builder, a, a, constvec, "");
343 bnative = LLVMBuildShuffleVector(builder, b, b, constvec, "");
344 tmp = lp_build_intrinsic_binary(builder, name,
345 lp_build_vec_type(gallivm, intrin_type),
346 anative, bnative);
347 if (src_type.length > 1) {
348 constvec = LLVMConstVector(elems, src_type.length);
349 return LLVMBuildShuffleVector(builder, tmp, tmp, constvec, "");
350 }
351 else {
352 return LLVMBuildExtractElement(builder, tmp, elems[0], "");
353 }
354 }
355 else if (intrin_length < src_type.length) {
356 unsigned num_vec = src_type.length / intrin_length;
357 LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
358
359 /* don't support arbitrary size here as this is so yuck */
360 if (src_type.length % intrin_length) {
361 /* FIXME: This is something which should be supported
362 * but there doesn't seem to be any need for it currently
363 * so crash and burn.
364 */
365 debug_printf("%s: should handle arbitrary vector size\n",
366 __FUNCTION__);
367 assert(0);
368 return NULL;
369 }
370
371 for (i = 0; i < num_vec; i++) {
372 anative = lp_build_extract_range(gallivm, a, i*intrin_length,
373 intrin_length);
374 bnative = lp_build_extract_range(gallivm, b, i*intrin_length,
375 intrin_length);
376 tmp[i] = lp_build_intrinsic_binary(builder, name,
377 lp_build_vec_type(gallivm, intrin_type),
378 anative, bnative);
379 }
380 return lp_build_concat(gallivm, tmp, intrin_type, num_vec);
381 }
382 else {
383 return lp_build_intrinsic_binary(builder, name,
384 lp_build_vec_type(gallivm, src_type),
385 a, b);
386 }
387 }
388
389
390 LLVMValueRef
lp_build_intrinsic_map(struct gallivm_state * gallivm,const char * name,LLVMTypeRef ret_type,LLVMValueRef * args,unsigned num_args)391 lp_build_intrinsic_map(struct gallivm_state *gallivm,
392 const char *name,
393 LLVMTypeRef ret_type,
394 LLVMValueRef *args,
395 unsigned num_args)
396 {
397 LLVMBuilderRef builder = gallivm->builder;
398 LLVMTypeRef ret_elem_type = LLVMGetElementType(ret_type);
399 unsigned n = LLVMGetVectorSize(ret_type);
400 unsigned i, j;
401 LLVMValueRef res;
402
403 assert(num_args <= LP_MAX_FUNC_ARGS);
404
405 res = LLVMGetUndef(ret_type);
406 for(i = 0; i < n; ++i) {
407 LLVMValueRef index = lp_build_const_int32(gallivm, i);
408 LLVMValueRef arg_elems[LP_MAX_FUNC_ARGS];
409 LLVMValueRef res_elem;
410 for(j = 0; j < num_args; ++j)
411 arg_elems[j] = LLVMBuildExtractElement(builder, args[j], index, "");
412 res_elem = lp_build_intrinsic(builder, name, ret_elem_type, arg_elems, num_args, 0);
413 res = LLVMBuildInsertElement(builder, res, res_elem, index, "");
414 }
415
416 return res;
417 }
418
419
420 LLVMValueRef
lp_build_intrinsic_map_unary(struct gallivm_state * gallivm,const char * name,LLVMTypeRef ret_type,LLVMValueRef a)421 lp_build_intrinsic_map_unary(struct gallivm_state *gallivm,
422 const char *name,
423 LLVMTypeRef ret_type,
424 LLVMValueRef a)
425 {
426 return lp_build_intrinsic_map(gallivm, name, ret_type, &a, 1);
427 }
428
429
430 LLVMValueRef
lp_build_intrinsic_map_binary(struct gallivm_state * gallivm,const char * name,LLVMTypeRef ret_type,LLVMValueRef a,LLVMValueRef b)431 lp_build_intrinsic_map_binary(struct gallivm_state *gallivm,
432 const char *name,
433 LLVMTypeRef ret_type,
434 LLVMValueRef a,
435 LLVMValueRef b)
436 {
437 LLVMValueRef args[2];
438
439 args[0] = a;
440 args[1] = b;
441
442 return lp_build_intrinsic_map(gallivm, name, ret_type, args, 2);
443 }
444
445
446