1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25
26 #include <stdint.h>
27 #include "lp_bld_coro.h"
28 #include "util/os_memory.h"
29 #include "lp_bld_init.h"
30 #include "lp_bld_const.h"
31 #include "lp_bld_intr.h"
32 #include "lp_bld_flow.h"
33
34 #if LLVM_VERSION_MAJOR < 6
35 /* not a wrapper, just lets it compile */
LLVMTokenTypeInContext(LLVMContextRef C)36 static LLVMTypeRef LLVMTokenTypeInContext(LLVMContextRef C)
37 {
38 assert(0);
39 return LLVMVoidTypeInContext(C);
40 }
41 #endif
42
lp_build_coro_id(struct gallivm_state * gallivm)43 LLVMValueRef lp_build_coro_id(struct gallivm_state *gallivm)
44 {
45 LLVMValueRef coro_id_args[4];
46 coro_id_args[0] = lp_build_const_int32(gallivm, 0);
47 coro_id_args[1] = LLVMConstPointerNull(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0));
48 coro_id_args[2] = coro_id_args[1];
49 coro_id_args[3] = coro_id_args[1];
50 LLVMValueRef coro_id = lp_build_intrinsic(gallivm->builder,
51 "llvm.coro.id",
52 LLVMTokenTypeInContext(gallivm->context),
53 coro_id_args, 4, 0);
54 return coro_id;
55 }
56
lp_build_coro_size(struct gallivm_state * gallivm)57 LLVMValueRef lp_build_coro_size(struct gallivm_state *gallivm)
58 {
59 return lp_build_intrinsic(gallivm->builder,
60 "llvm.coro.size.i32",
61 LLVMInt32TypeInContext(gallivm->context),
62 NULL, 0, 0);
63 }
64
lp_build_coro_begin(struct gallivm_state * gallivm,LLVMValueRef coro_id,LLVMValueRef mem_ptr)65 LLVMValueRef lp_build_coro_begin(struct gallivm_state *gallivm,
66 LLVMValueRef coro_id, LLVMValueRef mem_ptr)
67 {
68 LLVMValueRef coro_begin_args[2];
69 coro_begin_args[0] = coro_id;
70 coro_begin_args[1] = mem_ptr;
71 LLVMValueRef coro_hdl = lp_build_intrinsic(gallivm->builder,
72 "llvm.coro.begin",
73 LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
74 coro_begin_args, 2, 0);
75 return coro_hdl;
76 }
77
lp_build_coro_free(struct gallivm_state * gallivm,LLVMValueRef coro_id,LLVMValueRef coro_hdl)78 LLVMValueRef lp_build_coro_free(struct gallivm_state *gallivm,
79 LLVMValueRef coro_id, LLVMValueRef coro_hdl)
80 {
81 LLVMValueRef coro_free_args[2];
82 coro_free_args[0] = coro_id;
83 coro_free_args[1] = coro_hdl;
84 return lp_build_intrinsic(gallivm->builder,
85 "llvm.coro.free",
86 LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
87 coro_free_args, 2, 0);
88 }
89
lp_build_coro_end(struct gallivm_state * gallivm,LLVMValueRef coro_hdl)90 void lp_build_coro_end(struct gallivm_state *gallivm, LLVMValueRef coro_hdl)
91 {
92 LLVMValueRef coro_end_args[3];
93 int num_args = 2;
94 coro_end_args[0] = coro_hdl;
95 coro_end_args[1] = LLVMConstInt(LLVMInt1TypeInContext(gallivm->context), 0, 0);
96 #if LLVM_VERSION_MAJOR >= 18
97 coro_end_args[2] = LLVMConstNull(LLVMTokenTypeInContext(gallivm->context));
98 num_args++;
99 #endif
100 lp_build_intrinsic(gallivm->builder,
101 "llvm.coro.end",
102 LLVMInt1TypeInContext(gallivm->context),
103 coro_end_args, num_args, 0);
104 }
105
lp_build_coro_resume(struct gallivm_state * gallivm,LLVMValueRef coro_hdl)106 void lp_build_coro_resume(struct gallivm_state *gallivm, LLVMValueRef coro_hdl)
107 {
108 lp_build_intrinsic(gallivm->builder,
109 "llvm.coro.resume",
110 LLVMVoidTypeInContext(gallivm->context),
111 &coro_hdl, 1, 0);
112 }
113
lp_build_coro_destroy(struct gallivm_state * gallivm,LLVMValueRef coro_hdl)114 void lp_build_coro_destroy(struct gallivm_state *gallivm, LLVMValueRef coro_hdl)
115 {
116 lp_build_intrinsic(gallivm->builder,
117 "llvm.coro.destroy",
118 LLVMVoidTypeInContext(gallivm->context),
119 &coro_hdl, 1, 0);
120 }
121
lp_build_coro_done(struct gallivm_state * gallivm,LLVMValueRef coro_hdl)122 LLVMValueRef lp_build_coro_done(struct gallivm_state *gallivm, LLVMValueRef coro_hdl)
123 {
124 return lp_build_intrinsic(gallivm->builder,
125 "llvm.coro.done",
126 LLVMInt1TypeInContext(gallivm->context),
127 &coro_hdl, 1, 0);
128 }
129
lp_build_coro_suspend(struct gallivm_state * gallivm,bool last)130 LLVMValueRef lp_build_coro_suspend(struct gallivm_state *gallivm, bool last)
131 {
132 LLVMValueRef coro_susp_args[2];
133 coro_susp_args[0] = LLVMConstNull(LLVMTokenTypeInContext(gallivm->context));
134 coro_susp_args[1] = LLVMConstInt(LLVMInt1TypeInContext(gallivm->context), last, 0);
135 LLVMValueRef coro_suspend = lp_build_intrinsic(gallivm->builder,
136 "llvm.coro.suspend",
137 LLVMInt8TypeInContext(gallivm->context),
138 coro_susp_args, 2, 0);
139 return coro_suspend;
140 }
141
lp_build_coro_alloc(struct gallivm_state * gallivm,LLVMValueRef id)142 LLVMValueRef lp_build_coro_alloc(struct gallivm_state *gallivm, LLVMValueRef id)
143 {
144 return lp_build_intrinsic(gallivm->builder,
145 "llvm.coro.alloc",
146 LLVMInt1TypeInContext(gallivm->context),
147 &id, 1, 0);
148 }
149
150 static char *
coro_malloc(int size)151 coro_malloc(int size)
152 {
153 return os_malloc_aligned(size, 4096);
154 }
155
156 static void
coro_free(char * ptr)157 coro_free(char *ptr)
158 {
159 os_free_aligned(ptr);
160 }
161
lp_build_coro_add_malloc_hooks(struct gallivm_state * gallivm)162 void lp_build_coro_add_malloc_hooks(struct gallivm_state *gallivm)
163 {
164 assert(gallivm->engine);
165
166 assert(gallivm->coro_malloc_hook);
167 assert(gallivm->coro_free_hook);
168 LLVMAddGlobalMapping(gallivm->engine, gallivm->coro_malloc_hook, coro_malloc);
169 LLVMAddGlobalMapping(gallivm->engine, gallivm->coro_free_hook, coro_free);
170 }
171
lp_build_coro_declare_malloc_hooks(struct gallivm_state * gallivm)172 void lp_build_coro_declare_malloc_hooks(struct gallivm_state *gallivm)
173 {
174 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
175 LLVMTypeRef mem_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
176 LLVMTypeRef malloc_type = LLVMFunctionType(mem_ptr_type, &int32_type, 1, 0);
177 gallivm->coro_malloc_hook_type = malloc_type;
178 gallivm->coro_malloc_hook = LLVMAddFunction(gallivm->module, "coro_malloc", malloc_type);
179 LLVMTypeRef free_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), &mem_ptr_type, 1, 0);
180 gallivm->coro_free_hook_type = free_type;
181 gallivm->coro_free_hook = LLVMAddFunction(gallivm->module, "coro_free", free_type);
182 }
183
lp_build_coro_begin_alloc_mem(struct gallivm_state * gallivm,LLVMValueRef coro_id)184 LLVMValueRef lp_build_coro_begin_alloc_mem(struct gallivm_state *gallivm, LLVMValueRef coro_id)
185 {
186 LLVMTypeRef mem_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
187 LLVMValueRef do_alloc = lp_build_coro_alloc(gallivm, coro_id);
188 struct lp_build_if_state if_state_coro;
189 lp_build_if(&if_state_coro, gallivm, do_alloc);
190 LLVMValueRef coro_size = lp_build_coro_size(gallivm);
191 LLVMValueRef alloc_mem;
192
193 assert(gallivm->coro_malloc_hook);
194 LLVMTypeRef malloc_type =
195 LLVMFunctionType(mem_ptr_type,
196 (LLVMTypeRef[]){LLVMInt32TypeInContext(gallivm->context)}, 1, 0);
197 alloc_mem = LLVMBuildCall2(gallivm->builder, malloc_type, gallivm->coro_malloc_hook, &coro_size, 1, "");
198 lp_build_endif(&if_state_coro);
199
200 LLVMValueRef phi = LLVMBuildPhi(gallivm->builder, mem_ptr_type, "");
201 LLVMValueRef zero_bool = LLVMConstNull(mem_ptr_type);
202 LLVMAddIncoming(phi, &alloc_mem, &if_state_coro.true_block, 1);
203 LLVMAddIncoming(phi, &zero_bool, &if_state_coro.entry_block, 1);
204
205 LLVMValueRef coro_hdl = lp_build_coro_begin(gallivm, coro_id, phi);
206 return coro_hdl;
207 }
208
lp_build_coro_alloc_mem_array(struct gallivm_state * gallivm,LLVMValueRef coro_hdl_ptr,LLVMValueRef coro_idx,LLVMValueRef coro_num_hdls)209 LLVMValueRef lp_build_coro_alloc_mem_array(struct gallivm_state *gallivm,
210 LLVMValueRef coro_hdl_ptr, LLVMValueRef coro_idx,
211 LLVMValueRef coro_num_hdls)
212 {
213 LLVMTypeRef mem_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
214 LLVMValueRef alloced_ptr = LLVMBuildLoad2(gallivm->builder, mem_ptr_type, coro_hdl_ptr, "");
215
216 LLVMValueRef not_alloced = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, alloced_ptr, LLVMConstNull(mem_ptr_type), "");
217 LLVMValueRef coro_size = lp_build_coro_size(gallivm);
218
219 struct lp_build_if_state if_state_coro;
220 lp_build_if(&if_state_coro, gallivm, not_alloced);
221
222 LLVMValueRef alloc_mem;
223 LLVMValueRef alloc_size = LLVMBuildMul(gallivm->builder, coro_num_hdls, coro_size, "");
224 assert(gallivm->coro_malloc_hook);
225 assert(gallivm->coro_malloc_hook_type);
226 alloc_mem = LLVMBuildCall2(gallivm->builder, gallivm->coro_malloc_hook_type, gallivm->coro_malloc_hook, &alloc_size, 1, "");
227 LLVMBuildStore(gallivm->builder, alloc_mem, coro_hdl_ptr);
228 lp_build_endif(&if_state_coro);
229
230 return LLVMBuildMul(gallivm->builder, coro_size, coro_idx, "");
231 }
232
lp_build_coro_free_mem(struct gallivm_state * gallivm,LLVMValueRef coro_id,LLVMValueRef coro_hdl)233 void lp_build_coro_free_mem(struct gallivm_state *gallivm, LLVMValueRef coro_id, LLVMValueRef coro_hdl)
234 {
235 LLVMValueRef alloc_mem = lp_build_coro_free(gallivm, coro_id, coro_hdl);
236
237 assert(gallivm->coro_free_hook);
238 assert(gallivm->coro_free_hook_type);
239 alloc_mem = LLVMBuildCall2(gallivm->builder, gallivm->coro_free_hook_type, gallivm->coro_free_hook, &alloc_mem, 1, "");
240 }
241
lp_build_coro_suspend_switch(struct gallivm_state * gallivm,const struct lp_build_coro_suspend_info * sus_info,LLVMBasicBlockRef resume_block,bool final_suspend)242 void lp_build_coro_suspend_switch(struct gallivm_state *gallivm, const struct lp_build_coro_suspend_info *sus_info,
243 LLVMBasicBlockRef resume_block, bool final_suspend)
244 {
245 LLVMValueRef coro_suspend = lp_build_coro_suspend(gallivm, final_suspend);
246 LLVMValueRef myswitch = LLVMBuildSwitch(gallivm->builder, coro_suspend,
247 sus_info->suspend, resume_block ? 2 : 1);
248 LLVMAddCase(myswitch, LLVMConstInt(LLVMInt8TypeInContext(gallivm->context), 1, 0), sus_info->cleanup);
249 if (resume_block)
250 LLVMAddCase(myswitch, LLVMConstInt(LLVMInt8TypeInContext(gallivm->context), 0, 0), resume_block);
251 }
252