1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28
29 /**
30 * @file
31 * Helpers for emiting intrinsic calls.
32 *
33 * LLVM vanilla IR doesn't represent all basic arithmetic operations we care
34 * about, and it is often necessary to resort target-specific intrinsics for
35 * performance, convenience.
36 *
37 * Ideally we would like to stay away from target specific intrinsics and
38 * move all the instruction selection logic into upstream LLVM where it belongs.
39 *
40 * These functions are also used for calling C functions provided by us from
41 * generated LLVM code.
42 *
43 * @author Jose Fonseca <jfonseca@vmware.com>
44 */
45
46
47 #include "util/u_debug.h"
48
49 #include "lp_bld_const.h"
50 #include "lp_bld_intr.h"
51 #include "lp_bld_type.h"
52 #include "lp_bld_pack.h"
53
54
55 LLVMValueRef
lp_declare_intrinsic(LLVMModuleRef module,const char * name,LLVMTypeRef ret_type,LLVMTypeRef * arg_types,unsigned num_args)56 lp_declare_intrinsic(LLVMModuleRef module,
57 const char *name,
58 LLVMTypeRef ret_type,
59 LLVMTypeRef *arg_types,
60 unsigned num_args)
61 {
62 LLVMTypeRef function_type;
63 LLVMValueRef function;
64
65 assert(!LLVMGetNamedFunction(module, name));
66
67 function_type = LLVMFunctionType(ret_type, arg_types, num_args, 0);
68 function = LLVMAddFunction(module, name, function_type);
69
70 LLVMSetFunctionCallConv(function, LLVMCCallConv);
71 LLVMSetLinkage(function, LLVMExternalLinkage);
72
73 assert(LLVMIsDeclaration(function));
74
75 return function;
76 }
77
78
79 LLVMValueRef
lp_build_intrinsic(LLVMBuilderRef builder,const char * name,LLVMTypeRef ret_type,LLVMValueRef * args,unsigned num_args)80 lp_build_intrinsic(LLVMBuilderRef builder,
81 const char *name,
82 LLVMTypeRef ret_type,
83 LLVMValueRef *args,
84 unsigned num_args)
85 {
86 LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
87 LLVMValueRef function;
88
89 function = LLVMGetNamedFunction(module, name);
90 if(!function) {
91 LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS];
92 unsigned i;
93
94 assert(num_args <= LP_MAX_FUNC_ARGS);
95
96 for(i = 0; i < num_args; ++i) {
97 assert(args[i]);
98 arg_types[i] = LLVMTypeOf(args[i]);
99 }
100
101 function = lp_declare_intrinsic(module, name, ret_type, arg_types, num_args);
102 }
103
104 return LLVMBuildCall(builder, function, args, num_args, "");
105 }
106
107
108 LLVMValueRef
lp_build_intrinsic_unary(LLVMBuilderRef builder,const char * name,LLVMTypeRef ret_type,LLVMValueRef a)109 lp_build_intrinsic_unary(LLVMBuilderRef builder,
110 const char *name,
111 LLVMTypeRef ret_type,
112 LLVMValueRef a)
113 {
114 return lp_build_intrinsic(builder, name, ret_type, &a, 1);
115 }
116
117
118 LLVMValueRef
lp_build_intrinsic_binary(LLVMBuilderRef builder,const char * name,LLVMTypeRef ret_type,LLVMValueRef a,LLVMValueRef b)119 lp_build_intrinsic_binary(LLVMBuilderRef builder,
120 const char *name,
121 LLVMTypeRef ret_type,
122 LLVMValueRef a,
123 LLVMValueRef b)
124 {
125 LLVMValueRef args[2];
126
127 args[0] = a;
128 args[1] = b;
129
130 return lp_build_intrinsic(builder, name, ret_type, args, 2);
131 }
132
133
134 /**
135 * Call intrinsic with arguments adapted to intrinsic vector length.
136 *
137 * Split vectors which are too large for the hw, or expand them if they
138 * are too small, so a caller calling a function which might use intrinsics
139 * doesn't need to do splitting/expansion on its own.
140 * This only supports intrinsics where src and dst types match.
141 */
142 LLVMValueRef
lp_build_intrinsic_binary_anylength(struct gallivm_state * gallivm,const char * name,struct lp_type src_type,unsigned intr_size,LLVMValueRef a,LLVMValueRef b)143 lp_build_intrinsic_binary_anylength(struct gallivm_state *gallivm,
144 const char *name,
145 struct lp_type src_type,
146 unsigned intr_size,
147 LLVMValueRef a,
148 LLVMValueRef b)
149 {
150 unsigned i;
151 struct lp_type intrin_type = src_type;
152 LLVMBuilderRef builder = gallivm->builder;
153 LLVMValueRef i32undef = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
154 LLVMValueRef anative, bnative;
155 unsigned intrin_length = intr_size / src_type.width;
156
157 intrin_type.length = intrin_length;
158
159 if (intrin_length > src_type.length) {
160 LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
161 LLVMValueRef constvec, tmp;
162
163 for (i = 0; i < src_type.length; i++) {
164 elems[i] = lp_build_const_int32(gallivm, i);
165 }
166 for (; i < intrin_length; i++) {
167 elems[i] = i32undef;
168 }
169 if (src_type.length == 1) {
170 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, intrin_type);
171 a = LLVMBuildBitCast(builder, a, LLVMVectorType(elem_type, 1), "");
172 b = LLVMBuildBitCast(builder, b, LLVMVectorType(elem_type, 1), "");
173 }
174 constvec = LLVMConstVector(elems, intrin_length);
175 anative = LLVMBuildShuffleVector(builder, a, a, constvec, "");
176 bnative = LLVMBuildShuffleVector(builder, b, b, constvec, "");
177 tmp = lp_build_intrinsic_binary(builder, name,
178 lp_build_vec_type(gallivm, intrin_type),
179 anative, bnative);
180 if (src_type.length > 1) {
181 constvec = LLVMConstVector(elems, src_type.length);
182 return LLVMBuildShuffleVector(builder, tmp, tmp, constvec, "");
183 }
184 else {
185 return LLVMBuildExtractElement(builder, tmp, elems[0], "");
186 }
187 }
188 else if (intrin_length < src_type.length) {
189 unsigned num_vec = src_type.length / intrin_length;
190 LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
191
192 /* don't support arbitrary size here as this is so yuck */
193 if (src_type.length % intrin_length) {
194 /* FIXME: This is something which should be supported
195 * but there doesn't seem to be any need for it currently
196 * so crash and burn.
197 */
198 debug_printf("%s: should handle arbitrary vector size\n",
199 __FUNCTION__);
200 assert(0);
201 return NULL;
202 }
203
204 for (i = 0; i < num_vec; i++) {
205 anative = lp_build_extract_range(gallivm, a, i*intrin_length,
206 intrin_length);
207 bnative = lp_build_extract_range(gallivm, b, i*intrin_length,
208 intrin_length);
209 tmp[i] = lp_build_intrinsic_binary(builder, name,
210 lp_build_vec_type(gallivm, intrin_type),
211 anative, bnative);
212 }
213 return lp_build_concat(gallivm, tmp, intrin_type, num_vec);
214 }
215 else {
216 return lp_build_intrinsic_binary(builder, name,
217 lp_build_vec_type(gallivm, src_type),
218 a, b);
219 }
220 }
221
222
223 LLVMValueRef
lp_build_intrinsic_map(struct gallivm_state * gallivm,const char * name,LLVMTypeRef ret_type,LLVMValueRef * args,unsigned num_args)224 lp_build_intrinsic_map(struct gallivm_state *gallivm,
225 const char *name,
226 LLVMTypeRef ret_type,
227 LLVMValueRef *args,
228 unsigned num_args)
229 {
230 LLVMBuilderRef builder = gallivm->builder;
231 LLVMTypeRef ret_elem_type = LLVMGetElementType(ret_type);
232 unsigned n = LLVMGetVectorSize(ret_type);
233 unsigned i, j;
234 LLVMValueRef res;
235
236 assert(num_args <= LP_MAX_FUNC_ARGS);
237
238 res = LLVMGetUndef(ret_type);
239 for(i = 0; i < n; ++i) {
240 LLVMValueRef index = lp_build_const_int32(gallivm, i);
241 LLVMValueRef arg_elems[LP_MAX_FUNC_ARGS];
242 LLVMValueRef res_elem;
243 for(j = 0; j < num_args; ++j)
244 arg_elems[j] = LLVMBuildExtractElement(builder, args[j], index, "");
245 res_elem = lp_build_intrinsic(builder, name, ret_elem_type, arg_elems, num_args);
246 res = LLVMBuildInsertElement(builder, res, res_elem, index, "");
247 }
248
249 return res;
250 }
251
252
253 LLVMValueRef
lp_build_intrinsic_map_unary(struct gallivm_state * gallivm,const char * name,LLVMTypeRef ret_type,LLVMValueRef a)254 lp_build_intrinsic_map_unary(struct gallivm_state *gallivm,
255 const char *name,
256 LLVMTypeRef ret_type,
257 LLVMValueRef a)
258 {
259 return lp_build_intrinsic_map(gallivm, name, ret_type, &a, 1);
260 }
261
262
263 LLVMValueRef
lp_build_intrinsic_map_binary(struct gallivm_state * gallivm,const char * name,LLVMTypeRef ret_type,LLVMValueRef a,LLVMValueRef b)264 lp_build_intrinsic_map_binary(struct gallivm_state *gallivm,
265 const char *name,
266 LLVMTypeRef ret_type,
267 LLVMValueRef a,
268 LLVMValueRef b)
269 {
270 LLVMValueRef args[2];
271
272 args[0] = a;
273 args[1] = b;
274
275 return lp_build_intrinsic_map(gallivm, name, ret_type, args, 2);
276 }
277
278
279