1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25
26 #include <stdint.h>
27 #include "lp_bld_coro.h"
28 #include "util/os_memory.h"
29 #include "lp_bld_init.h"
30 #include "lp_bld_const.h"
31 #include "lp_bld_intr.h"
32 #include "lp_bld_flow.h"
33
34 #if LLVM_VERSION_MAJOR < 6
35 /* not a wrapper, just lets it compile */
LLVMTokenTypeInContext(LLVMContextRef C)36 static LLVMTypeRef LLVMTokenTypeInContext(LLVMContextRef C)
37 {
38 assert(0);
39 return LLVMVoidTypeInContext(C);
40 }
41 #endif
42
lp_build_coro_id(struct gallivm_state * gallivm)43 LLVMValueRef lp_build_coro_id(struct gallivm_state *gallivm)
44 {
45 LLVMValueRef coro_id_args[4];
46 coro_id_args[0] = lp_build_const_int32(gallivm, 0);
47 coro_id_args[1] = LLVMConstPointerNull(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0));
48 coro_id_args[2] = coro_id_args[1];
49 coro_id_args[3] = coro_id_args[1];
50 LLVMValueRef coro_id = lp_build_intrinsic(gallivm->builder,
51 "llvm.coro.id",
52 LLVMTokenTypeInContext(gallivm->context),
53 coro_id_args, 4, 0);
54 return coro_id;
55 }
56
lp_build_coro_size(struct gallivm_state * gallivm)57 LLVMValueRef lp_build_coro_size(struct gallivm_state *gallivm)
58 {
59 return lp_build_intrinsic(gallivm->builder,
60 "llvm.coro.size.i32",
61 LLVMInt32TypeInContext(gallivm->context),
62 NULL, 0, 0);
63 }
64
lp_build_coro_begin(struct gallivm_state * gallivm,LLVMValueRef coro_id,LLVMValueRef mem_ptr)65 LLVMValueRef lp_build_coro_begin(struct gallivm_state *gallivm,
66 LLVMValueRef coro_id, LLVMValueRef mem_ptr)
67 {
68 LLVMValueRef coro_begin_args[2];
69 coro_begin_args[0] = coro_id;
70 coro_begin_args[1] = mem_ptr;
71 LLVMValueRef coro_hdl = lp_build_intrinsic(gallivm->builder,
72 "llvm.coro.begin",
73 LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
74 coro_begin_args, 2, 0);
75 return coro_hdl;
76 }
77
lp_build_coro_free(struct gallivm_state * gallivm,LLVMValueRef coro_id,LLVMValueRef coro_hdl)78 LLVMValueRef lp_build_coro_free(struct gallivm_state *gallivm,
79 LLVMValueRef coro_id, LLVMValueRef coro_hdl)
80 {
81 LLVMValueRef coro_free_args[2];
82 coro_free_args[0] = coro_id;
83 coro_free_args[1] = coro_hdl;
84 return lp_build_intrinsic(gallivm->builder,
85 "llvm.coro.free",
86 LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
87 coro_free_args, 2, 0);
88 }
89
lp_build_coro_end(struct gallivm_state * gallivm,LLVMValueRef coro_hdl)90 void lp_build_coro_end(struct gallivm_state *gallivm, LLVMValueRef coro_hdl)
91 {
92 LLVMValueRef coro_end_args[2];
93 coro_end_args[0] = coro_hdl;
94 coro_end_args[1] = LLVMConstInt(LLVMInt1TypeInContext(gallivm->context), 0, 0);
95 lp_build_intrinsic(gallivm->builder,
96 "llvm.coro.end",
97 LLVMInt1TypeInContext(gallivm->context),
98 coro_end_args, 2, 0);
99 }
100
lp_build_coro_resume(struct gallivm_state * gallivm,LLVMValueRef coro_hdl)101 void lp_build_coro_resume(struct gallivm_state *gallivm, LLVMValueRef coro_hdl)
102 {
103 lp_build_intrinsic(gallivm->builder,
104 "llvm.coro.resume",
105 LLVMVoidTypeInContext(gallivm->context),
106 &coro_hdl, 1, 0);
107 }
108
lp_build_coro_destroy(struct gallivm_state * gallivm,LLVMValueRef coro_hdl)109 void lp_build_coro_destroy(struct gallivm_state *gallivm, LLVMValueRef coro_hdl)
110 {
111 lp_build_intrinsic(gallivm->builder,
112 "llvm.coro.destroy",
113 LLVMVoidTypeInContext(gallivm->context),
114 &coro_hdl, 1, 0);
115 }
116
lp_build_coro_done(struct gallivm_state * gallivm,LLVMValueRef coro_hdl)117 LLVMValueRef lp_build_coro_done(struct gallivm_state *gallivm, LLVMValueRef coro_hdl)
118 {
119 return lp_build_intrinsic(gallivm->builder,
120 "llvm.coro.done",
121 LLVMInt1TypeInContext(gallivm->context),
122 &coro_hdl, 1, 0);
123 }
124
lp_build_coro_suspend(struct gallivm_state * gallivm,bool last)125 LLVMValueRef lp_build_coro_suspend(struct gallivm_state *gallivm, bool last)
126 {
127 LLVMValueRef coro_susp_args[2];
128 coro_susp_args[0] = LLVMConstNull(LLVMTokenTypeInContext(gallivm->context));
129 coro_susp_args[1] = LLVMConstInt(LLVMInt1TypeInContext(gallivm->context), last, 0);
130 LLVMValueRef coro_suspend = lp_build_intrinsic(gallivm->builder,
131 "llvm.coro.suspend",
132 LLVMInt8TypeInContext(gallivm->context),
133 coro_susp_args, 2, 0);
134 return coro_suspend;
135 }
136
lp_build_coro_alloc(struct gallivm_state * gallivm,LLVMValueRef id)137 LLVMValueRef lp_build_coro_alloc(struct gallivm_state *gallivm, LLVMValueRef id)
138 {
139 return lp_build_intrinsic(gallivm->builder,
140 "llvm.coro.alloc",
141 LLVMInt1TypeInContext(gallivm->context),
142 &id, 1, 0);
143 }
144
145 static char *
coro_malloc(int size)146 coro_malloc(int size)
147 {
148 return os_malloc_aligned(size, 4096);
149 }
150
151 static void
coro_free(char * ptr)152 coro_free(char *ptr)
153 {
154 os_free_aligned(ptr);
155 }
156
lp_build_coro_add_malloc_hooks(struct gallivm_state * gallivm)157 void lp_build_coro_add_malloc_hooks(struct gallivm_state *gallivm)
158 {
159 assert(gallivm->engine);
160
161 assert(gallivm->coro_malloc_hook);
162 assert(gallivm->coro_free_hook);
163 LLVMAddGlobalMapping(gallivm->engine, gallivm->coro_malloc_hook, coro_malloc);
164 LLVMAddGlobalMapping(gallivm->engine, gallivm->coro_free_hook, coro_free);
165 }
166
lp_build_coro_declare_malloc_hooks(struct gallivm_state * gallivm)167 void lp_build_coro_declare_malloc_hooks(struct gallivm_state *gallivm)
168 {
169 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
170 LLVMTypeRef mem_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
171 LLVMTypeRef malloc_type = LLVMFunctionType(mem_ptr_type, &int32_type, 1, 0);
172 gallivm->coro_malloc_hook_type = malloc_type;
173 gallivm->coro_malloc_hook = LLVMAddFunction(gallivm->module, "coro_malloc", malloc_type);
174 LLVMTypeRef free_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), &mem_ptr_type, 1, 0);
175 gallivm->coro_free_hook_type = free_type;
176 gallivm->coro_free_hook = LLVMAddFunction(gallivm->module, "coro_free", free_type);
177 }
178
lp_build_coro_begin_alloc_mem(struct gallivm_state * gallivm,LLVMValueRef coro_id)179 LLVMValueRef lp_build_coro_begin_alloc_mem(struct gallivm_state *gallivm, LLVMValueRef coro_id)
180 {
181 LLVMTypeRef mem_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
182 LLVMValueRef do_alloc = lp_build_coro_alloc(gallivm, coro_id);
183 struct lp_build_if_state if_state_coro;
184 lp_build_if(&if_state_coro, gallivm, do_alloc);
185 LLVMValueRef coro_size = lp_build_coro_size(gallivm);
186 LLVMValueRef alloc_mem;
187
188 assert(gallivm->coro_malloc_hook);
189 LLVMTypeRef malloc_type =
190 LLVMFunctionType(mem_ptr_type,
191 (LLVMTypeRef[]){LLVMInt32TypeInContext(gallivm->context)}, 1, 0);
192 alloc_mem = LLVMBuildCall2(gallivm->builder, malloc_type, gallivm->coro_malloc_hook, &coro_size, 1, "");
193 lp_build_endif(&if_state_coro);
194
195 LLVMValueRef phi = LLVMBuildPhi(gallivm->builder, mem_ptr_type, "");
196 LLVMValueRef zero_bool = LLVMConstNull(mem_ptr_type);
197 LLVMAddIncoming(phi, &alloc_mem, &if_state_coro.true_block, 1);
198 LLVMAddIncoming(phi, &zero_bool, &if_state_coro.entry_block, 1);
199
200 LLVMValueRef coro_hdl = lp_build_coro_begin(gallivm, coro_id, phi);
201 return coro_hdl;
202 }
203
lp_build_coro_alloc_mem_array(struct gallivm_state * gallivm,LLVMValueRef coro_hdl_ptr,LLVMValueRef coro_idx,LLVMValueRef coro_num_hdls)204 LLVMValueRef lp_build_coro_alloc_mem_array(struct gallivm_state *gallivm,
205 LLVMValueRef coro_hdl_ptr, LLVMValueRef coro_idx,
206 LLVMValueRef coro_num_hdls)
207 {
208 LLVMTypeRef mem_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
209 LLVMValueRef alloced_ptr = LLVMBuildLoad2(gallivm->builder, mem_ptr_type, coro_hdl_ptr, "");
210
211 LLVMValueRef not_alloced = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, alloced_ptr, LLVMConstNull(mem_ptr_type), "");
212 LLVMValueRef coro_size = lp_build_coro_size(gallivm);
213
214 struct lp_build_if_state if_state_coro;
215 lp_build_if(&if_state_coro, gallivm, not_alloced);
216
217 LLVMValueRef alloc_mem;
218 LLVMValueRef alloc_size = LLVMBuildMul(gallivm->builder, coro_num_hdls, coro_size, "");
219 assert(gallivm->coro_malloc_hook);
220 assert(gallivm->coro_malloc_hook_type);
221 alloc_mem = LLVMBuildCall2(gallivm->builder, gallivm->coro_malloc_hook_type, gallivm->coro_malloc_hook, &alloc_size, 1, "");
222 LLVMBuildStore(gallivm->builder, alloc_mem, coro_hdl_ptr);
223 lp_build_endif(&if_state_coro);
224
225 return LLVMBuildMul(gallivm->builder, coro_size, coro_idx, "");
226 }
227
lp_build_coro_free_mem(struct gallivm_state * gallivm,LLVMValueRef coro_id,LLVMValueRef coro_hdl)228 void lp_build_coro_free_mem(struct gallivm_state *gallivm, LLVMValueRef coro_id, LLVMValueRef coro_hdl)
229 {
230 LLVMValueRef alloc_mem = lp_build_coro_free(gallivm, coro_id, coro_hdl);
231
232 assert(gallivm->coro_free_hook);
233 assert(gallivm->coro_free_hook_type);
234 alloc_mem = LLVMBuildCall2(gallivm->builder, gallivm->coro_free_hook_type, gallivm->coro_free_hook, &alloc_mem, 1, "");
235 }
236
lp_build_coro_suspend_switch(struct gallivm_state * gallivm,const struct lp_build_coro_suspend_info * sus_info,LLVMBasicBlockRef resume_block,bool final_suspend)237 void lp_build_coro_suspend_switch(struct gallivm_state *gallivm, const struct lp_build_coro_suspend_info *sus_info,
238 LLVMBasicBlockRef resume_block, bool final_suspend)
239 {
240 LLVMValueRef coro_suspend = lp_build_coro_suspend(gallivm, final_suspend);
241 LLVMValueRef myswitch = LLVMBuildSwitch(gallivm->builder, coro_suspend,
242 sus_info->suspend, resume_block ? 2 : 1);
243 LLVMAddCase(myswitch, LLVMConstInt(LLVMInt8TypeInContext(gallivm->context), 1, 0), sus_info->cleanup);
244 if (resume_block)
245 LLVMAddCase(myswitch, LLVMConstInt(LLVMInt8TypeInContext(gallivm->context), 0, 0), resume_block);
246 }
247