1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28
29 /**
30 * @file
31 * Helpers for emiting intrinsic calls.
32 *
33 * LLVM vanilla IR doesn't represent all basic arithmetic operations we care
34 * about, and it is often necessary to resort target-specific intrinsics for
35 * performance, convenience.
36 *
37 * Ideally we would like to stay away from target specific intrinsics and
38 * move all the instruction selection logic into upstream LLVM where it belongs.
39 *
40 * These functions are also used for calling C functions provided by us from
41 * generated LLVM code.
42 *
43 * @author Jose Fonseca <jfonseca@vmware.com>
44 */
45
46
47 #include "util/u_debug.h"
48 #include "util/u_string.h"
49 #include "util/bitscan.h"
50
51 #include "lp_bld_const.h"
52 #include "lp_bld_intr.h"
53 #include "lp_bld_type.h"
54 #include "lp_bld_pack.h"
55 #include "lp_bld_debug.h"
56
57
58 void
lp_format_intrinsic(char * name,size_t size,const char * name_root,LLVMTypeRef type)59 lp_format_intrinsic(char *name,
60 size_t size,
61 const char *name_root,
62 LLVMTypeRef type)
63 {
64 unsigned length = 0;
65 unsigned width;
66 char c;
67
68 LLVMTypeKind kind = LLVMGetTypeKind(type);
69 if (kind == LLVMVectorTypeKind) {
70 length = LLVMGetVectorSize(type);
71 type = LLVMGetElementType(type);
72 kind = LLVMGetTypeKind(type);
73 }
74
75 switch (kind) {
76 case LLVMIntegerTypeKind:
77 c = 'i';
78 width = LLVMGetIntTypeWidth(type);
79 break;
80 case LLVMFloatTypeKind:
81 c = 'f';
82 width = 32;
83 break;
84 case LLVMDoubleTypeKind:
85 c = 'f';
86 width = 64;
87 break;
88 default:
89 unreachable("unexpected LLVMTypeKind");
90 }
91
92 if (length) {
93 util_snprintf(name, size, "%s.v%u%c%u", name_root, length, c, width);
94 } else {
95 util_snprintf(name, size, "%s.%c%u", name_root, c, width);
96 }
97 }
98
99
100 LLVMValueRef
lp_declare_intrinsic(LLVMModuleRef module,const char * name,LLVMTypeRef ret_type,LLVMTypeRef * arg_types,unsigned num_args)101 lp_declare_intrinsic(LLVMModuleRef module,
102 const char *name,
103 LLVMTypeRef ret_type,
104 LLVMTypeRef *arg_types,
105 unsigned num_args)
106 {
107 LLVMTypeRef function_type;
108 LLVMValueRef function;
109
110 assert(!LLVMGetNamedFunction(module, name));
111
112 function_type = LLVMFunctionType(ret_type, arg_types, num_args, 0);
113 function = LLVMAddFunction(module, name, function_type);
114
115 LLVMSetFunctionCallConv(function, LLVMCCallConv);
116 LLVMSetLinkage(function, LLVMExternalLinkage);
117
118 assert(LLVMIsDeclaration(function));
119
120 return function;
121 }
122
123
124 #if HAVE_LLVM < 0x0400
lp_attr_to_llvm_attr(enum lp_func_attr attr)125 static LLVMAttribute lp_attr_to_llvm_attr(enum lp_func_attr attr)
126 {
127 switch (attr) {
128 case LP_FUNC_ATTR_ALWAYSINLINE: return LLVMAlwaysInlineAttribute;
129 case LP_FUNC_ATTR_BYVAL: return LLVMByValAttribute;
130 case LP_FUNC_ATTR_INREG: return LLVMInRegAttribute;
131 case LP_FUNC_ATTR_NOALIAS: return LLVMNoAliasAttribute;
132 case LP_FUNC_ATTR_NOUNWIND: return LLVMNoUnwindAttribute;
133 case LP_FUNC_ATTR_READNONE: return LLVMReadNoneAttribute;
134 case LP_FUNC_ATTR_READONLY: return LLVMReadOnlyAttribute;
135 default:
136 _debug_printf("Unhandled function attribute: %x\n", attr);
137 return 0;
138 }
139 }
140
141 #else
142
attr_to_str(enum lp_func_attr attr)143 static const char *attr_to_str(enum lp_func_attr attr)
144 {
145 switch (attr) {
146 case LP_FUNC_ATTR_ALWAYSINLINE: return "alwaysinline";
147 case LP_FUNC_ATTR_BYVAL: return "byval";
148 case LP_FUNC_ATTR_INREG: return "inreg";
149 case LP_FUNC_ATTR_NOALIAS: return "noalias";
150 case LP_FUNC_ATTR_NOUNWIND: return "nounwind";
151 case LP_FUNC_ATTR_READNONE: return "readnone";
152 case LP_FUNC_ATTR_READONLY: return "readonly";
153 case LP_FUNC_ATTR_WRITEONLY: return "writeonly";
154 case LP_FUNC_ATTR_INACCESSIBLE_MEM_ONLY: return "inaccessiblememonly";
155 case LP_FUNC_ATTR_CONVERGENT: return "convergent";
156 default:
157 _debug_printf("Unhandled function attribute: %x\n", attr);
158 return 0;
159 }
160 }
161
162 #endif
163
164 void
lp_add_function_attr(LLVMValueRef function_or_call,int attr_idx,enum lp_func_attr attr)165 lp_add_function_attr(LLVMValueRef function_or_call,
166 int attr_idx, enum lp_func_attr attr)
167 {
168
169 #if HAVE_LLVM < 0x0400
170 LLVMAttribute llvm_attr = lp_attr_to_llvm_attr(attr);
171 if (LLVMIsAFunction(function_or_call)) {
172 if (attr_idx == -1) {
173 LLVMAddFunctionAttr(function_or_call, llvm_attr);
174 } else {
175 LLVMAddAttribute(LLVMGetParam(function_or_call, attr_idx - 1), llvm_attr);
176 }
177 } else {
178 LLVMAddInstrAttribute(function_or_call, attr_idx, llvm_attr);
179 }
180 #else
181
182 LLVMModuleRef module;
183 if (LLVMIsAFunction(function_or_call)) {
184 module = LLVMGetGlobalParent(function_or_call);
185 } else {
186 LLVMBasicBlockRef bb = LLVMGetInstructionParent(function_or_call);
187 LLVMValueRef function = LLVMGetBasicBlockParent(bb);
188 module = LLVMGetGlobalParent(function);
189 }
190 LLVMContextRef ctx = LLVMGetModuleContext(module);
191
192 const char *attr_name = attr_to_str(attr);
193 unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name,
194 strlen(attr_name));
195 LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(ctx, kind_id, 0);
196
197 if (LLVMIsAFunction(function_or_call))
198 LLVMAddAttributeAtIndex(function_or_call, attr_idx, llvm_attr);
199 else
200 LLVMAddCallSiteAttribute(function_or_call, attr_idx, llvm_attr);
201 #endif
202 }
203
204 static void
lp_add_func_attributes(LLVMValueRef function,unsigned attrib_mask)205 lp_add_func_attributes(LLVMValueRef function, unsigned attrib_mask)
206 {
207 /* NoUnwind indicates that the intrinsic never raises a C++ exception.
208 * Set it for all intrinsics.
209 */
210 attrib_mask |= LP_FUNC_ATTR_NOUNWIND;
211 attrib_mask &= ~LP_FUNC_ATTR_LEGACY;
212
213 while (attrib_mask) {
214 enum lp_func_attr attr = 1u << u_bit_scan(&attrib_mask);
215 lp_add_function_attr(function, -1, attr);
216 }
217 }
218
219 LLVMValueRef
lp_build_intrinsic(LLVMBuilderRef builder,const char * name,LLVMTypeRef ret_type,LLVMValueRef * args,unsigned num_args,unsigned attr_mask)220 lp_build_intrinsic(LLVMBuilderRef builder,
221 const char *name,
222 LLVMTypeRef ret_type,
223 LLVMValueRef *args,
224 unsigned num_args,
225 unsigned attr_mask)
226 {
227 LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
228 LLVMValueRef function, call;
229 bool set_callsite_attrs = HAVE_LLVM >= 0x0400 &&
230 !(attr_mask & LP_FUNC_ATTR_LEGACY);
231
232 function = LLVMGetNamedFunction(module, name);
233 if(!function) {
234 LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS];
235 unsigned i;
236
237 assert(num_args <= LP_MAX_FUNC_ARGS);
238
239 for(i = 0; i < num_args; ++i) {
240 assert(args[i]);
241 arg_types[i] = LLVMTypeOf(args[i]);
242 }
243
244 function = lp_declare_intrinsic(module, name, ret_type, arg_types, num_args);
245
246 if (!set_callsite_attrs)
247 lp_add_func_attributes(function, attr_mask);
248
249 if (gallivm_debug & GALLIVM_DEBUG_IR) {
250 lp_debug_dump_value(function);
251 }
252 }
253
254 call = LLVMBuildCall(builder, function, args, num_args, "");
255 if (set_callsite_attrs)
256 lp_add_func_attributes(call, attr_mask);
257 return call;
258 }
259
260
261 LLVMValueRef
lp_build_intrinsic_unary(LLVMBuilderRef builder,const char * name,LLVMTypeRef ret_type,LLVMValueRef a)262 lp_build_intrinsic_unary(LLVMBuilderRef builder,
263 const char *name,
264 LLVMTypeRef ret_type,
265 LLVMValueRef a)
266 {
267 return lp_build_intrinsic(builder, name, ret_type, &a, 1, 0);
268 }
269
270
271 LLVMValueRef
lp_build_intrinsic_binary(LLVMBuilderRef builder,const char * name,LLVMTypeRef ret_type,LLVMValueRef a,LLVMValueRef b)272 lp_build_intrinsic_binary(LLVMBuilderRef builder,
273 const char *name,
274 LLVMTypeRef ret_type,
275 LLVMValueRef a,
276 LLVMValueRef b)
277 {
278 LLVMValueRef args[2];
279
280 args[0] = a;
281 args[1] = b;
282
283 return lp_build_intrinsic(builder, name, ret_type, args, 2, 0);
284 }
285
286
287 /**
288 * Call intrinsic with arguments adapted to intrinsic vector length.
289 *
290 * Split vectors which are too large for the hw, or expand them if they
291 * are too small, so a caller calling a function which might use intrinsics
292 * doesn't need to do splitting/expansion on its own.
293 * This only supports intrinsics where src and dst types match.
294 */
295 LLVMValueRef
lp_build_intrinsic_binary_anylength(struct gallivm_state * gallivm,const char * name,struct lp_type src_type,unsigned intr_size,LLVMValueRef a,LLVMValueRef b)296 lp_build_intrinsic_binary_anylength(struct gallivm_state *gallivm,
297 const char *name,
298 struct lp_type src_type,
299 unsigned intr_size,
300 LLVMValueRef a,
301 LLVMValueRef b)
302 {
303 unsigned i;
304 struct lp_type intrin_type = src_type;
305 LLVMBuilderRef builder = gallivm->builder;
306 LLVMValueRef i32undef = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
307 LLVMValueRef anative, bnative;
308 unsigned intrin_length = intr_size / src_type.width;
309
310 intrin_type.length = intrin_length;
311
312 if (intrin_length > src_type.length) {
313 LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
314 LLVMValueRef constvec, tmp;
315
316 for (i = 0; i < src_type.length; i++) {
317 elems[i] = lp_build_const_int32(gallivm, i);
318 }
319 for (; i < intrin_length; i++) {
320 elems[i] = i32undef;
321 }
322 if (src_type.length == 1) {
323 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, intrin_type);
324 a = LLVMBuildBitCast(builder, a, LLVMVectorType(elem_type, 1), "");
325 b = LLVMBuildBitCast(builder, b, LLVMVectorType(elem_type, 1), "");
326 }
327 constvec = LLVMConstVector(elems, intrin_length);
328 anative = LLVMBuildShuffleVector(builder, a, a, constvec, "");
329 bnative = LLVMBuildShuffleVector(builder, b, b, constvec, "");
330 tmp = lp_build_intrinsic_binary(builder, name,
331 lp_build_vec_type(gallivm, intrin_type),
332 anative, bnative);
333 if (src_type.length > 1) {
334 constvec = LLVMConstVector(elems, src_type.length);
335 return LLVMBuildShuffleVector(builder, tmp, tmp, constvec, "");
336 }
337 else {
338 return LLVMBuildExtractElement(builder, tmp, elems[0], "");
339 }
340 }
341 else if (intrin_length < src_type.length) {
342 unsigned num_vec = src_type.length / intrin_length;
343 LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
344
345 /* don't support arbitrary size here as this is so yuck */
346 if (src_type.length % intrin_length) {
347 /* FIXME: This is something which should be supported
348 * but there doesn't seem to be any need for it currently
349 * so crash and burn.
350 */
351 debug_printf("%s: should handle arbitrary vector size\n",
352 __FUNCTION__);
353 assert(0);
354 return NULL;
355 }
356
357 for (i = 0; i < num_vec; i++) {
358 anative = lp_build_extract_range(gallivm, a, i*intrin_length,
359 intrin_length);
360 bnative = lp_build_extract_range(gallivm, b, i*intrin_length,
361 intrin_length);
362 tmp[i] = lp_build_intrinsic_binary(builder, name,
363 lp_build_vec_type(gallivm, intrin_type),
364 anative, bnative);
365 }
366 return lp_build_concat(gallivm, tmp, intrin_type, num_vec);
367 }
368 else {
369 return lp_build_intrinsic_binary(builder, name,
370 lp_build_vec_type(gallivm, src_type),
371 a, b);
372 }
373 }
374
375
376 LLVMValueRef
lp_build_intrinsic_map(struct gallivm_state * gallivm,const char * name,LLVMTypeRef ret_type,LLVMValueRef * args,unsigned num_args)377 lp_build_intrinsic_map(struct gallivm_state *gallivm,
378 const char *name,
379 LLVMTypeRef ret_type,
380 LLVMValueRef *args,
381 unsigned num_args)
382 {
383 LLVMBuilderRef builder = gallivm->builder;
384 LLVMTypeRef ret_elem_type = LLVMGetElementType(ret_type);
385 unsigned n = LLVMGetVectorSize(ret_type);
386 unsigned i, j;
387 LLVMValueRef res;
388
389 assert(num_args <= LP_MAX_FUNC_ARGS);
390
391 res = LLVMGetUndef(ret_type);
392 for(i = 0; i < n; ++i) {
393 LLVMValueRef index = lp_build_const_int32(gallivm, i);
394 LLVMValueRef arg_elems[LP_MAX_FUNC_ARGS];
395 LLVMValueRef res_elem;
396 for(j = 0; j < num_args; ++j)
397 arg_elems[j] = LLVMBuildExtractElement(builder, args[j], index, "");
398 res_elem = lp_build_intrinsic(builder, name, ret_elem_type, arg_elems, num_args, 0);
399 res = LLVMBuildInsertElement(builder, res, res_elem, index, "");
400 }
401
402 return res;
403 }
404
405
406 LLVMValueRef
lp_build_intrinsic_map_unary(struct gallivm_state * gallivm,const char * name,LLVMTypeRef ret_type,LLVMValueRef a)407 lp_build_intrinsic_map_unary(struct gallivm_state *gallivm,
408 const char *name,
409 LLVMTypeRef ret_type,
410 LLVMValueRef a)
411 {
412 return lp_build_intrinsic_map(gallivm, name, ret_type, &a, 1);
413 }
414
415
416 LLVMValueRef
lp_build_intrinsic_map_binary(struct gallivm_state * gallivm,const char * name,LLVMTypeRef ret_type,LLVMValueRef a,LLVMValueRef b)417 lp_build_intrinsic_map_binary(struct gallivm_state *gallivm,
418 const char *name,
419 LLVMTypeRef ret_type,
420 LLVMValueRef a,
421 LLVMValueRef b)
422 {
423 LLVMValueRef args[2];
424
425 args[0] = a;
426 args[1] = b;
427
428 return lp_build_intrinsic_map(gallivm, name, ret_type, args, 2);
429 }
430
431
432