1 /****************************************************************************
2 * Copyright (C) 2015 Intel Corporation. All Rights Reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 ***************************************************************************/
23
24 // llvm redefines DEBUG
25 #pragma push_macro("DEBUG")
26 #undef DEBUG
27 #include "JitManager.h"
28 #include "llvm-c/Core.h"
29 #include "llvm/Support/CBindingWrapping.h"
30 #pragma pop_macro("DEBUG")
31
32 #include "state.h"
33 #include "gen_state_llvm.h"
34 #include "builder.h"
35
36 #include "tgsi/tgsi_strings.h"
37 #include "util/u_format.h"
38 #include "util/u_prim.h"
39 #include "gallivm/lp_bld_init.h"
40 #include "gallivm/lp_bld_flow.h"
41 #include "gallivm/lp_bld_struct.h"
42 #include "gallivm/lp_bld_tgsi.h"
43
44 #include "swr_context.h"
45 #include "gen_swr_context_llvm.h"
46 #include "swr_resource.h"
47 #include "swr_state.h"
48 #include "swr_screen.h"
49
50 using namespace SwrJit;
51 using namespace llvm;
52
53 static unsigned
54 locate_linkage(ubyte name, ubyte index, struct tgsi_shader_info *info);
55
operator ==(const swr_jit_fs_key & lhs,const swr_jit_fs_key & rhs)56 bool operator==(const swr_jit_fs_key &lhs, const swr_jit_fs_key &rhs)
57 {
58 return !memcmp(&lhs, &rhs, sizeof(lhs));
59 }
60
operator ==(const swr_jit_vs_key & lhs,const swr_jit_vs_key & rhs)61 bool operator==(const swr_jit_vs_key &lhs, const swr_jit_vs_key &rhs)
62 {
63 return !memcmp(&lhs, &rhs, sizeof(lhs));
64 }
65
operator ==(const swr_jit_fetch_key & lhs,const swr_jit_fetch_key & rhs)66 bool operator==(const swr_jit_fetch_key &lhs, const swr_jit_fetch_key &rhs)
67 {
68 return !memcmp(&lhs, &rhs, sizeof(lhs));
69 }
70
operator ==(const swr_jit_gs_key & lhs,const swr_jit_gs_key & rhs)71 bool operator==(const swr_jit_gs_key &lhs, const swr_jit_gs_key &rhs)
72 {
73 return !memcmp(&lhs, &rhs, sizeof(lhs));
74 }
75
76 static void
swr_generate_sampler_key(const struct lp_tgsi_info & info,struct swr_context * ctx,enum pipe_shader_type shader_type,struct swr_jit_sampler_key & key)77 swr_generate_sampler_key(const struct lp_tgsi_info &info,
78 struct swr_context *ctx,
79 enum pipe_shader_type shader_type,
80 struct swr_jit_sampler_key &key)
81 {
82 key.nr_samplers = info.base.file_max[TGSI_FILE_SAMPLER] + 1;
83
84 for (unsigned i = 0; i < key.nr_samplers; i++) {
85 if (info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
86 lp_sampler_static_sampler_state(
87 &key.sampler[i].sampler_state,
88 ctx->samplers[shader_type][i]);
89 }
90 }
91
92 /*
93 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
94 * are dx10-style? Can't really have mixed opcodes, at least not
95 * if we want to skip the holes here (without rescanning tgsi).
96 */
97 if (info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
98 key.nr_sampler_views =
99 info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
100 for (unsigned i = 0; i < key.nr_sampler_views; i++) {
101 if (info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1 << i)) {
102 const struct pipe_sampler_view *view =
103 ctx->sampler_views[shader_type][i];
104 lp_sampler_static_texture_state(
105 &key.sampler[i].texture_state, view);
106 if (view) {
107 struct swr_resource *swr_res = swr_resource(view->texture);
108 const struct util_format_description *desc =
109 util_format_description(view->format);
110 if (swr_res->has_depth && swr_res->has_stencil &&
111 !util_format_has_depth(desc))
112 key.sampler[i].texture_state.format = PIPE_FORMAT_S8_UINT;
113 }
114 }
115 }
116 } else {
117 key.nr_sampler_views = key.nr_samplers;
118 for (unsigned i = 0; i < key.nr_sampler_views; i++) {
119 if (info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
120 const struct pipe_sampler_view *view =
121 ctx->sampler_views[shader_type][i];
122 lp_sampler_static_texture_state(
123 &key.sampler[i].texture_state, view);
124 if (view) {
125 struct swr_resource *swr_res = swr_resource(view->texture);
126 const struct util_format_description *desc =
127 util_format_description(view->format);
128 if (swr_res->has_depth && swr_res->has_stencil &&
129 !util_format_has_depth(desc))
130 key.sampler[i].texture_state.format = PIPE_FORMAT_S8_UINT;
131 }
132 }
133 }
134 }
135 }
136
137 void
swr_generate_fs_key(struct swr_jit_fs_key & key,struct swr_context * ctx,swr_fragment_shader * swr_fs)138 swr_generate_fs_key(struct swr_jit_fs_key &key,
139 struct swr_context *ctx,
140 swr_fragment_shader *swr_fs)
141 {
142 memset(&key, 0, sizeof(key));
143
144 key.nr_cbufs = ctx->framebuffer.nr_cbufs;
145 key.light_twoside = ctx->rasterizer->light_twoside;
146 key.sprite_coord_enable = ctx->rasterizer->sprite_coord_enable;
147
148 struct tgsi_shader_info *pPrevShader;
149 if (ctx->gs)
150 pPrevShader = &ctx->gs->info.base;
151 else
152 pPrevShader = &ctx->vs->info.base;
153
154 memcpy(&key.vs_output_semantic_name,
155 &pPrevShader->output_semantic_name,
156 sizeof(key.vs_output_semantic_name));
157 memcpy(&key.vs_output_semantic_idx,
158 &pPrevShader->output_semantic_index,
159 sizeof(key.vs_output_semantic_idx));
160
161 swr_generate_sampler_key(swr_fs->info, ctx, PIPE_SHADER_FRAGMENT, key);
162
163 key.poly_stipple_enable = ctx->rasterizer->poly_stipple_enable &&
164 ctx->poly_stipple.prim_is_poly;
165 }
166
167 void
swr_generate_vs_key(struct swr_jit_vs_key & key,struct swr_context * ctx,swr_vertex_shader * swr_vs)168 swr_generate_vs_key(struct swr_jit_vs_key &key,
169 struct swr_context *ctx,
170 swr_vertex_shader *swr_vs)
171 {
172 memset(&key, 0, sizeof(key));
173
174 key.clip_plane_mask =
175 swr_vs->info.base.clipdist_writemask ?
176 swr_vs->info.base.clipdist_writemask & ctx->rasterizer->clip_plane_enable :
177 ctx->rasterizer->clip_plane_enable;
178
179 swr_generate_sampler_key(swr_vs->info, ctx, PIPE_SHADER_VERTEX, key);
180 }
181
182 void
swr_generate_fetch_key(struct swr_jit_fetch_key & key,struct swr_vertex_element_state * velems)183 swr_generate_fetch_key(struct swr_jit_fetch_key &key,
184 struct swr_vertex_element_state *velems)
185 {
186 memset(&key, 0, sizeof(key));
187
188 key.fsState = velems->fsState;
189 }
190
191 void
swr_generate_gs_key(struct swr_jit_gs_key & key,struct swr_context * ctx,swr_geometry_shader * swr_gs)192 swr_generate_gs_key(struct swr_jit_gs_key &key,
193 struct swr_context *ctx,
194 swr_geometry_shader *swr_gs)
195 {
196 memset(&key, 0, sizeof(key));
197
198 struct tgsi_shader_info *pPrevShader = &ctx->vs->info.base;
199
200 memcpy(&key.vs_output_semantic_name,
201 &pPrevShader->output_semantic_name,
202 sizeof(key.vs_output_semantic_name));
203 memcpy(&key.vs_output_semantic_idx,
204 &pPrevShader->output_semantic_index,
205 sizeof(key.vs_output_semantic_idx));
206
207 swr_generate_sampler_key(swr_gs->info, ctx, PIPE_SHADER_GEOMETRY, key);
208 }
209
210 struct BuilderSWR : public Builder {
BuilderSWRBuilderSWR211 BuilderSWR(JitManager *pJitMgr, const char *pName)
212 : Builder(pJitMgr)
213 {
214 pJitMgr->SetupNewModule();
215 gallivm = gallivm_create(pName, wrap(&JM()->mContext));
216 pJitMgr->mpCurrentModule = unwrap(gallivm->module);
217 }
218
~BuilderSWRBuilderSWR219 ~BuilderSWR() {
220 gallivm_free_ir(gallivm);
221 }
222
223 void WriteVS(Value *pVal, Value *pVsContext, Value *pVtxOutput,
224 unsigned slot, unsigned channel);
225
226 struct gallivm_state *gallivm;
227 PFN_VERTEX_FUNC CompileVS(struct swr_context *ctx, swr_jit_vs_key &key);
228 PFN_PIXEL_KERNEL CompileFS(struct swr_context *ctx, swr_jit_fs_key &key);
229 PFN_GS_FUNC CompileGS(struct swr_context *ctx, swr_jit_gs_key &key);
230
231 LLVMValueRef
232 swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
233 struct lp_build_tgsi_context * bld_base,
234 boolean is_vindex_indirect,
235 LLVMValueRef vertex_index,
236 boolean is_aindex_indirect,
237 LLVMValueRef attrib_index,
238 LLVMValueRef swizzle_index);
239 void
240 swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
241 struct lp_build_tgsi_context * bld_base,
242 LLVMValueRef (*outputs)[4],
243 LLVMValueRef emitted_vertices_vec);
244
245 void
246 swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
247 struct lp_build_tgsi_context * bld_base,
248 LLVMValueRef verts_per_prim_vec,
249 LLVMValueRef emitted_prims_vec);
250
251 void
252 swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
253 struct lp_build_tgsi_context * bld_base,
254 LLVMValueRef total_emitted_vertices_vec,
255 LLVMValueRef emitted_prims_vec);
256
257 };
258
259 struct swr_gs_llvm_iface {
260 struct lp_build_tgsi_gs_iface base;
261 struct tgsi_shader_info *info;
262
263 BuilderSWR *pBuilder;
264
265 Value *pGsCtx;
266 SWR_GS_STATE *pGsState;
267 uint32_t num_outputs;
268 uint32_t num_verts_per_prim;
269
270 Value *pVtxAttribMap;
271 };
272
273 // trampoline functions so we can use the builder llvm construction methods
274 static LLVMValueRef
swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface * gs_iface,struct lp_build_tgsi_context * bld_base,boolean is_vindex_indirect,LLVMValueRef vertex_index,boolean is_aindex_indirect,LLVMValueRef attrib_index,LLVMValueRef swizzle_index)275 swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
276 struct lp_build_tgsi_context * bld_base,
277 boolean is_vindex_indirect,
278 LLVMValueRef vertex_index,
279 boolean is_aindex_indirect,
280 LLVMValueRef attrib_index,
281 LLVMValueRef swizzle_index)
282 {
283 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_iface;
284
285 return iface->pBuilder->swr_gs_llvm_fetch_input(gs_iface, bld_base,
286 is_vindex_indirect,
287 vertex_index,
288 is_aindex_indirect,
289 attrib_index,
290 swizzle_index);
291 }
292
293 static void
swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface * gs_base,struct lp_build_tgsi_context * bld_base,LLVMValueRef (* outputs)[4],LLVMValueRef emitted_vertices_vec)294 swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
295 struct lp_build_tgsi_context * bld_base,
296 LLVMValueRef (*outputs)[4],
297 LLVMValueRef emitted_vertices_vec)
298 {
299 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
300
301 iface->pBuilder->swr_gs_llvm_emit_vertex(gs_base, bld_base,
302 outputs,
303 emitted_vertices_vec);
304 }
305
306 static void
swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface * gs_base,struct lp_build_tgsi_context * bld_base,LLVMValueRef verts_per_prim_vec,LLVMValueRef emitted_prims_vec)307 swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
308 struct lp_build_tgsi_context * bld_base,
309 LLVMValueRef verts_per_prim_vec,
310 LLVMValueRef emitted_prims_vec)
311 {
312 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
313
314 iface->pBuilder->swr_gs_llvm_end_primitive(gs_base, bld_base,
315 verts_per_prim_vec,
316 emitted_prims_vec);
317 }
318
319 static void
swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface * gs_base,struct lp_build_tgsi_context * bld_base,LLVMValueRef total_emitted_vertices_vec,LLVMValueRef emitted_prims_vec)320 swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
321 struct lp_build_tgsi_context * bld_base,
322 LLVMValueRef total_emitted_vertices_vec,
323 LLVMValueRef emitted_prims_vec)
324 {
325 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
326
327 iface->pBuilder->swr_gs_llvm_epilogue(gs_base, bld_base,
328 total_emitted_vertices_vec,
329 emitted_prims_vec);
330 }
331
332 LLVMValueRef
swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface * gs_iface,struct lp_build_tgsi_context * bld_base,boolean is_vindex_indirect,LLVMValueRef vertex_index,boolean is_aindex_indirect,LLVMValueRef attrib_index,LLVMValueRef swizzle_index)333 BuilderSWR::swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
334 struct lp_build_tgsi_context * bld_base,
335 boolean is_vindex_indirect,
336 LLVMValueRef vertex_index,
337 boolean is_aindex_indirect,
338 LLVMValueRef attrib_index,
339 LLVMValueRef swizzle_index)
340 {
341 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_iface;
342 Value *vert_index = unwrap(vertex_index);
343 Value *attr_index = unwrap(attrib_index);
344
345 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
346
347 if (is_vindex_indirect || is_aindex_indirect) {
348 int i;
349 Value *res = unwrap(bld_base->base.zero);
350 struct lp_type type = bld_base->base.type;
351
352 for (i = 0; i < type.length; i++) {
353 Value *vert_chan_index = vert_index;
354 Value *attr_chan_index = attr_index;
355
356 if (is_vindex_indirect) {
357 vert_chan_index = VEXTRACT(vert_index, C(i));
358 }
359 if (is_aindex_indirect) {
360 attr_chan_index = VEXTRACT(attr_index, C(i));
361 }
362
363 Value *attrib =
364 LOAD(GEP(iface->pVtxAttribMap, {C(0), attr_chan_index}));
365
366 Value *pVertex = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pVerts});
367 Value *pInputVertStride = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_inputVertStride});
368
369 Value *pVector = ADD(MUL(vert_chan_index, pInputVertStride), attrib);
370 Value *pInput = LOAD(GEP(pVertex, {pVector, unwrap(swizzle_index)}));
371
372 Value *value = VEXTRACT(pInput, C(i));
373 res = VINSERT(res, value, C(i));
374 }
375
376 return wrap(res);
377 } else {
378 Value *attrib = LOAD(GEP(iface->pVtxAttribMap, {C(0), attr_index}));
379
380 Value *pVertex = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pVerts});
381 Value *pInputVertStride = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_inputVertStride});
382
383 Value *pVector = ADD(MUL(vert_index, pInputVertStride), attrib);
384
385 Value *pInput = LOAD(GEP(pVertex, {pVector, unwrap(swizzle_index)}));
386
387 return wrap(pInput);
388 }
389 }
390
391 // GS output stream layout
392 #define VERTEX_COUNT_SIZE 32
393 #define CONTROL_HEADER_SIZE (8*32)
394
395 void
swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface * gs_base,struct lp_build_tgsi_context * bld_base,LLVMValueRef (* outputs)[4],LLVMValueRef emitted_vertices_vec)396 BuilderSWR::swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
397 struct lp_build_tgsi_context * bld_base,
398 LLVMValueRef (*outputs)[4],
399 LLVMValueRef emitted_vertices_vec)
400 {
401 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
402
403 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
404
405 const uint32_t headerSize = VERTEX_COUNT_SIZE + CONTROL_HEADER_SIZE;
406 const uint32_t attribSize = 4 * sizeof(float);
407 const uint32_t vertSize = attribSize * SWR_VTX_NUM_SLOTS;
408 Value *pVertexOffset = MUL(unwrap(emitted_vertices_vec), VIMMED1(vertSize));
409
410 Value *vMask = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_mask});
411 Value *vMask1 = TRUNC(vMask, VectorType::get(mInt1Ty, mVWidth));
412
413 Value *pStack = STACKSAVE();
414 Value *pTmpPtr = ALLOCA(mFP32Ty, C(4)); // used for dummy write for lane masking
415
416 for (uint32_t attrib = 0; attrib < iface->num_outputs; ++attrib) {
417 uint32_t attribSlot = attrib;
418 uint32_t sgvChannel = 0;
419 if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_PSIZE) {
420 attribSlot = VERTEX_SGV_SLOT;
421 sgvChannel = VERTEX_SGV_POINT_SIZE_COMP;
422 } else if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_LAYER) {
423 attribSlot = VERTEX_SGV_SLOT;
424 sgvChannel = VERTEX_SGV_RTAI_COMP;
425 } else if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_POSITION) {
426 attribSlot = VERTEX_POSITION_SLOT;
427 } else {
428 attribSlot = VERTEX_ATTRIB_START_SLOT + attrib;
429 if (iface->info->writes_position) {
430 attribSlot--;
431 }
432 }
433
434 Value *pOutputOffset = ADD(pVertexOffset, VIMMED1(headerSize + attribSize * attribSlot)); // + sgvChannel ?
435
436 for (uint32_t lane = 0; lane < mVWidth; ++lane) {
437 Value *pLaneOffset = VEXTRACT(pOutputOffset, C(lane));
438 Value *pStream = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
439 Value *pStreamOffset = GEP(pStream, pLaneOffset);
440 pStreamOffset = BITCAST(pStreamOffset, mFP32PtrTy);
441
442 Value *pLaneMask = VEXTRACT(vMask1, C(lane));
443 pStreamOffset = SELECT(pLaneMask, pStreamOffset, pTmpPtr);
444
445 for (uint32_t channel = 0; channel < 4; ++channel) {
446 Value *vData;
447
448 if (attribSlot == VERTEX_SGV_SLOT)
449 vData = LOAD(unwrap(outputs[attrib][0]));
450 else
451 vData = LOAD(unwrap(outputs[attrib][channel]));
452
453 if (attribSlot != VERTEX_SGV_SLOT ||
454 sgvChannel == channel) {
455 vData = VEXTRACT(vData, C(lane));
456 STORE(vData, pStreamOffset);
457 }
458 pStreamOffset = GEP(pStreamOffset, C(1));
459 }
460 }
461 }
462
463 STACKRESTORE(pStack);
464 }
465
466 void
swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface * gs_base,struct lp_build_tgsi_context * bld_base,LLVMValueRef verts_per_prim_vec,LLVMValueRef emitted_prims_vec)467 BuilderSWR::swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
468 struct lp_build_tgsi_context * bld_base,
469 LLVMValueRef verts_per_prim_vec,
470 LLVMValueRef emitted_prims_vec)
471 {
472 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
473
474 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
475
476 Value *vMask = LOAD(iface->pGsCtx, { 0, SWR_GS_CONTEXT_mask });
477 Value *vMask1 = TRUNC(vMask, VectorType::get(mInt1Ty, 8));
478
479 uint32_t vertsPerPrim = iface->num_verts_per_prim;
480
481 Value *vCount =
482 ADD(MUL(unwrap(emitted_prims_vec), VIMMED1(vertsPerPrim)),
483 unwrap(verts_per_prim_vec));
484
485 struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
486 vCount = LOAD(unwrap(bld->total_emitted_vertices_vec_ptr));
487
488 struct lp_exec_mask *exec_mask = &bld->exec_mask;
489 Value *mask = unwrap(lp_build_mask_value(bld->mask));
490 if (exec_mask->has_mask)
491 mask = AND(mask, unwrap(exec_mask->exec_mask));
492
493 Value *cmpMask = VMASK(ICMP_NE(unwrap(verts_per_prim_vec), VIMMED1(0)));
494 mask = AND(mask, cmpMask);
495 vMask1 = TRUNC(mask, VectorType::get(mInt1Ty, 8));
496
497 vCount = SUB(vCount, VIMMED1(1));
498 Value *vOffset = ADD(UDIV(vCount, VIMMED1(8)), VIMMED1(VERTEX_COUNT_SIZE));
499 Value *vValue = SHL(VIMMED1(1), UREM(vCount, VIMMED1(8)));
500
501 vValue = TRUNC(vValue, VectorType::get(mInt8Ty, 8));
502
503 Value *pStack = STACKSAVE();
504 Value *pTmpPtr = ALLOCA(mInt8Ty, C(4)); // used for dummy read/write for lane masking
505
506 for (uint32_t lane = 0; lane < mVWidth; ++lane) {
507 Value *vLaneOffset = VEXTRACT(vOffset, C(lane));
508 Value *pStream = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
509 Value *pStreamOffset = GEP(pStream, vLaneOffset);
510
511 Value *pLaneMask = VEXTRACT(vMask1, C(lane));
512 pStreamOffset = SELECT(pLaneMask, pStreamOffset, pTmpPtr);
513
514 Value *vVal = LOAD(pStreamOffset);
515 vVal = OR(vVal, VEXTRACT(vValue, C(lane)));
516 STORE(vVal, pStreamOffset);
517 }
518
519 STACKRESTORE(pStack);
520 }
521
522 void
swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface * gs_base,struct lp_build_tgsi_context * bld_base,LLVMValueRef total_emitted_vertices_vec,LLVMValueRef emitted_prims_vec)523 BuilderSWR::swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
524 struct lp_build_tgsi_context * bld_base,
525 LLVMValueRef total_emitted_vertices_vec,
526 LLVMValueRef emitted_prims_vec)
527 {
528 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
529
530 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
531
532 // Store emit count to each output stream in the first DWORD
533 for (uint32_t lane = 0; lane < mVWidth; ++lane)
534 {
535 Value* pStream = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
536 pStream = BITCAST(pStream, mInt32PtrTy);
537 Value* pLaneCount = VEXTRACT(unwrap(total_emitted_vertices_vec), C(lane));
538 STORE(pLaneCount, pStream);
539 }
540 }
541
542 PFN_GS_FUNC
CompileGS(struct swr_context * ctx,swr_jit_gs_key & key)543 BuilderSWR::CompileGS(struct swr_context *ctx, swr_jit_gs_key &key)
544 {
545 SWR_GS_STATE *pGS = &ctx->gs->gsState;
546 struct tgsi_shader_info *info = &ctx->gs->info.base;
547
548 memset(pGS, 0, sizeof(*pGS));
549
550 pGS->gsEnable = true;
551
552 pGS->numInputAttribs = info->num_inputs;
553 pGS->outputTopology =
554 swr_convert_prim_topology(info->properties[TGSI_PROPERTY_GS_OUTPUT_PRIM]);
555 pGS->maxNumVerts = info->properties[TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES];
556 pGS->instanceCount = info->properties[TGSI_PROPERTY_GS_INVOCATIONS];
557
558 // XXX: single stream for now...
559 pGS->isSingleStream = true;
560 pGS->singleStreamID = 0;
561
562 pGS->vertexAttribOffset = VERTEX_ATTRIB_START_SLOT; // TODO: optimize
563 pGS->srcVertexAttribOffset = VERTEX_ATTRIB_START_SLOT; // TODO: optimize
564 pGS->inputVertStride = pGS->numInputAttribs + pGS->vertexAttribOffset;
565 pGS->outputVertexSize = SWR_VTX_NUM_SLOTS;
566 pGS->controlDataSize = 8; // GS ouputs max of 8 32B units
567 pGS->controlDataOffset = VERTEX_COUNT_SIZE;
568 pGS->outputVertexOffset = pGS->controlDataOffset + CONTROL_HEADER_SIZE;
569
570 pGS->allocationSize =
571 VERTEX_COUNT_SIZE + // vertex count
572 CONTROL_HEADER_SIZE + // control header
573 (SWR_VTX_NUM_SLOTS * 16) * // sizeof vertex
574 pGS->maxNumVerts; // num verts
575
576 struct swr_geometry_shader *gs = ctx->gs;
577
578 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
579 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
580
581 memset(outputs, 0, sizeof(outputs));
582
583 AttrBuilder attrBuilder;
584 attrBuilder.addStackAlignmentAttr(JM()->mVWidth * sizeof(float));
585
586 std::vector<Type *> gsArgs{PointerType::get(Gen_swr_draw_context(JM()), 0),
587 PointerType::get(Gen_SWR_GS_CONTEXT(JM()), 0)};
588 FunctionType *vsFuncType =
589 FunctionType::get(Type::getVoidTy(JM()->mContext), gsArgs, false);
590
591 // create new vertex shader function
592 auto pFunction = Function::Create(vsFuncType,
593 GlobalValue::ExternalLinkage,
594 "GS",
595 JM()->mpCurrentModule);
596 #if HAVE_LLVM < 0x0500
597 AttributeSet attrSet = AttributeSet::get(
598 JM()->mContext, AttributeSet::FunctionIndex, attrBuilder);
599 pFunction->addAttributes(AttributeSet::FunctionIndex, attrSet);
600 #else
601 pFunction->addAttributes(AttributeList::FunctionIndex, attrBuilder);
602 #endif
603
604 BasicBlock *block = BasicBlock::Create(JM()->mContext, "entry", pFunction);
605 IRB()->SetInsertPoint(block);
606 LLVMPositionBuilderAtEnd(gallivm->builder, wrap(block));
607
608 auto argitr = pFunction->arg_begin();
609 Value *hPrivateData = &*argitr++;
610 hPrivateData->setName("hPrivateData");
611 Value *pGsCtx = &*argitr++;
612 pGsCtx->setName("gsCtx");
613
614 Value *consts_ptr =
615 GEP(hPrivateData, {C(0), C(swr_draw_context_constantGS)});
616 consts_ptr->setName("gs_constants");
617 Value *const_sizes_ptr =
618 GEP(hPrivateData, {0, swr_draw_context_num_constantsGS});
619 const_sizes_ptr->setName("num_gs_constants");
620
621 struct lp_build_sampler_soa *sampler =
622 swr_sampler_soa_create(key.sampler, PIPE_SHADER_GEOMETRY);
623
624 struct lp_bld_tgsi_system_values system_values;
625 memset(&system_values, 0, sizeof(system_values));
626 system_values.prim_id = wrap(LOAD(pGsCtx, {0, SWR_GS_CONTEXT_PrimitiveID}));
627 system_values.instance_id = wrap(LOAD(pGsCtx, {0, SWR_GS_CONTEXT_InstanceID}));
628
629 std::vector<Constant*> mapConstants;
630 Value *vtxAttribMap = ALLOCA(ArrayType::get(mInt32Ty, PIPE_MAX_SHADER_INPUTS));
631 for (unsigned slot = 0; slot < info->num_inputs; slot++) {
632 ubyte semantic_name = info->input_semantic_name[slot];
633 ubyte semantic_idx = info->input_semantic_index[slot];
634
635 unsigned vs_slot = locate_linkage(semantic_name, semantic_idx, &ctx->vs->info.base);
636
637 vs_slot += VERTEX_ATTRIB_START_SLOT;
638
639 if (ctx->vs->info.base.output_semantic_name[0] == TGSI_SEMANTIC_POSITION)
640 vs_slot--;
641
642 if (semantic_name == TGSI_SEMANTIC_POSITION)
643 vs_slot = VERTEX_POSITION_SLOT;
644
645 STORE(C(vs_slot), vtxAttribMap, {0, slot});
646 mapConstants.push_back(C(vs_slot));
647 }
648
649 struct lp_build_mask_context mask;
650 Value *mask_val = LOAD(pGsCtx, {0, SWR_GS_CONTEXT_mask}, "gsMask");
651 lp_build_mask_begin(&mask, gallivm,
652 lp_type_float_vec(32, 32 * 8), wrap(mask_val));
653
654 // zero out cut buffer so we can load/modify/store bits
655 for (uint32_t lane = 0; lane < mVWidth; ++lane)
656 {
657 Value* pStream = LOAD(pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
658 MEMSET(pStream, C((char)0), VERTEX_COUNT_SIZE + CONTROL_HEADER_SIZE, sizeof(float) * KNOB_SIMD_WIDTH);
659 }
660
661 struct swr_gs_llvm_iface gs_iface;
662 gs_iface.base.fetch_input = ::swr_gs_llvm_fetch_input;
663 gs_iface.base.emit_vertex = ::swr_gs_llvm_emit_vertex;
664 gs_iface.base.end_primitive = ::swr_gs_llvm_end_primitive;
665 gs_iface.base.gs_epilogue = ::swr_gs_llvm_epilogue;
666 gs_iface.pBuilder = this;
667 gs_iface.pGsCtx = pGsCtx;
668 gs_iface.pGsState = pGS;
669 gs_iface.num_outputs = gs->info.base.num_outputs;
670 gs_iface.num_verts_per_prim =
671 u_vertices_per_prim((pipe_prim_type)info->properties[TGSI_PROPERTY_GS_OUTPUT_PRIM]);
672 gs_iface.info = info;
673 gs_iface.pVtxAttribMap = vtxAttribMap;
674
675 lp_build_tgsi_soa(gallivm,
676 gs->pipe.tokens,
677 lp_type_float_vec(32, 32 * 8),
678 &mask,
679 wrap(consts_ptr),
680 wrap(const_sizes_ptr),
681 &system_values,
682 inputs,
683 outputs,
684 wrap(hPrivateData), // (sampler context)
685 NULL, // thread data
686 sampler,
687 &gs->info.base,
688 &gs_iface.base);
689
690 lp_build_mask_end(&mask);
691
692 sampler->destroy(sampler);
693
694 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
695
696 RET_VOID();
697
698 gallivm_verify_function(gallivm, wrap(pFunction));
699 gallivm_compile_module(gallivm);
700
701 PFN_GS_FUNC pFunc =
702 (PFN_GS_FUNC)gallivm_jit_function(gallivm, wrap(pFunction));
703
704 debug_printf("geom shader %p\n", pFunc);
705 assert(pFunc && "Error: GeomShader = NULL");
706
707 JM()->mIsModuleFinalized = true;
708
709 return pFunc;
710 }
711
712 PFN_GS_FUNC
swr_compile_gs(struct swr_context * ctx,swr_jit_gs_key & key)713 swr_compile_gs(struct swr_context *ctx, swr_jit_gs_key &key)
714 {
715 BuilderSWR builder(
716 reinterpret_cast<JitManager *>(swr_screen(ctx->pipe.screen)->hJitMgr),
717 "GS");
718 PFN_GS_FUNC func = builder.CompileGS(ctx, key);
719
720 ctx->gs->map.insert(std::make_pair(key, make_unique<VariantGS>(builder.gallivm, func)));
721 return func;
722 }
723
724 void
WriteVS(Value * pVal,Value * pVsContext,Value * pVtxOutput,unsigned slot,unsigned channel)725 BuilderSWR::WriteVS(Value *pVal, Value *pVsContext, Value *pVtxOutput, unsigned slot, unsigned channel)
726 {
727 #if USE_SIMD16_FRONTEND && !USE_SIMD16_VS
728 // interleave the simdvertex components into the dest simd16vertex
729 // slot16offset = slot8offset * 2
730 // comp16offset = comp8offset * 2 + alternateOffset
731
732 Value *offset = LOAD(pVsContext, { 0, SWR_VS_CONTEXT_AlternateOffset });
733 Value *pOut = GEP(pVtxOutput, { C(0), C(0), C(slot * 2), offset } );
734 STORE(pVal, pOut, {channel * 2});
735 #else
736 Value *pOut = GEP(pVtxOutput, {0, 0, slot});
737 STORE(pVal, pOut, {0, channel});
738 #endif
739 }
740
741 PFN_VERTEX_FUNC
CompileVS(struct swr_context * ctx,swr_jit_vs_key & key)742 BuilderSWR::CompileVS(struct swr_context *ctx, swr_jit_vs_key &key)
743 {
744 struct swr_vertex_shader *swr_vs = ctx->vs;
745
746 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
747 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
748
749 memset(outputs, 0, sizeof(outputs));
750
751 AttrBuilder attrBuilder;
752 attrBuilder.addStackAlignmentAttr(JM()->mVWidth * sizeof(float));
753
754 std::vector<Type *> vsArgs{PointerType::get(Gen_swr_draw_context(JM()), 0),
755 PointerType::get(Gen_SWR_VS_CONTEXT(JM()), 0)};
756 FunctionType *vsFuncType =
757 FunctionType::get(Type::getVoidTy(JM()->mContext), vsArgs, false);
758
759 // create new vertex shader function
760 auto pFunction = Function::Create(vsFuncType,
761 GlobalValue::ExternalLinkage,
762 "VS",
763 JM()->mpCurrentModule);
764 #if HAVE_LLVM < 0x0500
765 AttributeSet attrSet = AttributeSet::get(
766 JM()->mContext, AttributeSet::FunctionIndex, attrBuilder);
767 pFunction->addAttributes(AttributeSet::FunctionIndex, attrSet);
768 #else
769 pFunction->addAttributes(AttributeList::FunctionIndex, attrBuilder);
770 #endif
771
772 BasicBlock *block = BasicBlock::Create(JM()->mContext, "entry", pFunction);
773 IRB()->SetInsertPoint(block);
774 LLVMPositionBuilderAtEnd(gallivm->builder, wrap(block));
775
776 auto argitr = pFunction->arg_begin();
777 Value *hPrivateData = &*argitr++;
778 hPrivateData->setName("hPrivateData");
779 Value *pVsCtx = &*argitr++;
780 pVsCtx->setName("vsCtx");
781
782 Value *consts_ptr = GEP(hPrivateData, {C(0), C(swr_draw_context_constantVS)});
783
784 consts_ptr->setName("vs_constants");
785 Value *const_sizes_ptr =
786 GEP(hPrivateData, {0, swr_draw_context_num_constantsVS});
787 const_sizes_ptr->setName("num_vs_constants");
788
789 Value *vtxInput = LOAD(pVsCtx, {0, SWR_VS_CONTEXT_pVin});
790 #if USE_SIMD16_VS
791 vtxInput = BITCAST(vtxInput, PointerType::get(Gen_simd16vertex(JM()), 0));
792 #endif
793
794 for (uint32_t attrib = 0; attrib < PIPE_MAX_SHADER_INPUTS; attrib++) {
795 const unsigned mask = swr_vs->info.base.input_usage_mask[attrib];
796 for (uint32_t channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
797 if (mask & (1 << channel)) {
798 inputs[attrib][channel] =
799 wrap(LOAD(vtxInput, {0, 0, attrib, channel}));
800 }
801 }
802 }
803
804 struct lp_build_sampler_soa *sampler =
805 swr_sampler_soa_create(key.sampler, PIPE_SHADER_VERTEX);
806
807 struct lp_bld_tgsi_system_values system_values;
808 memset(&system_values, 0, sizeof(system_values));
809 system_values.instance_id = wrap(LOAD(pVsCtx, {0, SWR_VS_CONTEXT_InstanceID}));
810
811 #if USE_SIMD16_VS
812 system_values.vertex_id = wrap(LOAD(pVsCtx, {0, SWR_VS_CONTEXT_VertexID16}));
813 #else
814 system_values.vertex_id = wrap(LOAD(pVsCtx, {0, SWR_VS_CONTEXT_VertexID}));
815 #endif
816
817 #if USE_SIMD16_VS
818 uint32_t vectorWidth = mVWidth16;
819 #else
820 uint32_t vectorWidth = mVWidth;
821 #endif
822
823 lp_build_tgsi_soa(gallivm,
824 swr_vs->pipe.tokens,
825 lp_type_float_vec(32, 32 * vectorWidth),
826 NULL, // mask
827 wrap(consts_ptr),
828 wrap(const_sizes_ptr),
829 &system_values,
830 inputs,
831 outputs,
832 wrap(hPrivateData), // (sampler context)
833 NULL, // thread data
834 sampler, // sampler
835 &swr_vs->info.base,
836 NULL); // geometry shader face
837
838 sampler->destroy(sampler);
839
840 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
841
842 Value *vtxOutput = LOAD(pVsCtx, {0, SWR_VS_CONTEXT_pVout});
843 #if USE_SIMD16_VS
844 vtxOutput = BITCAST(vtxOutput, PointerType::get(Gen_simd16vertex(JM()), 0));
845 #endif
846
847 for (uint32_t channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
848 for (uint32_t attrib = 0; attrib < PIPE_MAX_SHADER_OUTPUTS; attrib++) {
849 if (!outputs[attrib][channel])
850 continue;
851
852 Value *val;
853 uint32_t outSlot;
854
855 if (swr_vs->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_PSIZE) {
856 if (channel != VERTEX_SGV_POINT_SIZE_COMP)
857 continue;
858 val = LOAD(unwrap(outputs[attrib][0]));
859 outSlot = VERTEX_SGV_SLOT;
860 } else if (swr_vs->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_POSITION) {
861 val = LOAD(unwrap(outputs[attrib][channel]));
862 outSlot = VERTEX_POSITION_SLOT;
863 } else {
864 val = LOAD(unwrap(outputs[attrib][channel]));
865 outSlot = VERTEX_ATTRIB_START_SLOT + attrib;
866 if (swr_vs->info.base.output_semantic_name[0] == TGSI_SEMANTIC_POSITION)
867 outSlot--;
868 }
869
870 WriteVS(val, pVsCtx, vtxOutput, outSlot, channel);
871 }
872 }
873
874 if (ctx->rasterizer->clip_plane_enable ||
875 swr_vs->info.base.culldist_writemask) {
876 unsigned clip_mask = ctx->rasterizer->clip_plane_enable;
877
878 unsigned cv = 0;
879 if (swr_vs->info.base.writes_clipvertex) {
880 cv = locate_linkage(TGSI_SEMANTIC_CLIPVERTEX, 0,
881 &swr_vs->info.base);
882 } else {
883 for (int i = 0; i < PIPE_MAX_SHADER_OUTPUTS; i++) {
884 if (swr_vs->info.base.output_semantic_name[i] == TGSI_SEMANTIC_POSITION &&
885 swr_vs->info.base.output_semantic_index[i] == 0) {
886 cv = i;
887 break;
888 }
889 }
890 }
891 LLVMValueRef cx = LLVMBuildLoad(gallivm->builder, outputs[cv][0], "");
892 LLVMValueRef cy = LLVMBuildLoad(gallivm->builder, outputs[cv][1], "");
893 LLVMValueRef cz = LLVMBuildLoad(gallivm->builder, outputs[cv][2], "");
894 LLVMValueRef cw = LLVMBuildLoad(gallivm->builder, outputs[cv][3], "");
895
896 for (unsigned val = 0; val < PIPE_MAX_CLIP_PLANES; val++) {
897 // clip distance overrides user clip planes
898 if ((swr_vs->info.base.clipdist_writemask & clip_mask & (1 << val)) ||
899 ((swr_vs->info.base.culldist_writemask << swr_vs->info.base.num_written_clipdistance) & (1 << val))) {
900 unsigned cv = locate_linkage(TGSI_SEMANTIC_CLIPDIST, val < 4 ? 0 : 1,
901 &swr_vs->info.base);
902 if (val < 4) {
903 LLVMValueRef dist = LLVMBuildLoad(gallivm->builder, outputs[cv][val], "");
904 WriteVS(unwrap(dist), pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_LO_SLOT, val);
905 } else {
906 LLVMValueRef dist = LLVMBuildLoad(gallivm->builder, outputs[cv][val - 4], "");
907 WriteVS(unwrap(dist), pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_HI_SLOT, val - 4);
908 }
909 continue;
910 }
911
912 if (!(clip_mask & (1 << val)))
913 continue;
914
915 Value *px = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 0}));
916 Value *py = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 1}));
917 Value *pz = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 2}));
918 Value *pw = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 3}));
919 #if USE_SIMD16_VS
920 Value *bpx = VBROADCAST_16(px);
921 Value *bpy = VBROADCAST_16(py);
922 Value *bpz = VBROADCAST_16(pz);
923 Value *bpw = VBROADCAST_16(pw);
924 #else
925 Value *bpx = VBROADCAST(px);
926 Value *bpy = VBROADCAST(py);
927 Value *bpz = VBROADCAST(pz);
928 Value *bpw = VBROADCAST(pw);
929 #endif
930 Value *dist = FADD(FMUL(unwrap(cx), bpx),
931 FADD(FMUL(unwrap(cy), bpy),
932 FADD(FMUL(unwrap(cz), bpz),
933 FMUL(unwrap(cw), bpw))));
934
935 if (val < 4)
936 WriteVS(dist, pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_LO_SLOT, val);
937 else
938 WriteVS(dist, pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_HI_SLOT, val - 4);
939 }
940 }
941
942 RET_VOID();
943
944 gallivm_verify_function(gallivm, wrap(pFunction));
945 gallivm_compile_module(gallivm);
946
947 // lp_debug_dump_value(func);
948
949 PFN_VERTEX_FUNC pFunc =
950 (PFN_VERTEX_FUNC)gallivm_jit_function(gallivm, wrap(pFunction));
951
952 debug_printf("vert shader %p\n", pFunc);
953 assert(pFunc && "Error: VertShader = NULL");
954
955 JM()->mIsModuleFinalized = true;
956
957 return pFunc;
958 }
959
960 PFN_VERTEX_FUNC
swr_compile_vs(struct swr_context * ctx,swr_jit_vs_key & key)961 swr_compile_vs(struct swr_context *ctx, swr_jit_vs_key &key)
962 {
963 if (!ctx->vs->pipe.tokens)
964 return NULL;
965
966 BuilderSWR builder(
967 reinterpret_cast<JitManager *>(swr_screen(ctx->pipe.screen)->hJitMgr),
968 "VS");
969 PFN_VERTEX_FUNC func = builder.CompileVS(ctx, key);
970
971 ctx->vs->map.insert(std::make_pair(key, make_unique<VariantVS>(builder.gallivm, func)));
972 return func;
973 }
974
975 unsigned
swr_so_adjust_attrib(unsigned in_attrib,swr_vertex_shader * swr_vs)976 swr_so_adjust_attrib(unsigned in_attrib,
977 swr_vertex_shader *swr_vs)
978 {
979 ubyte semantic_name;
980 unsigned attrib;
981
982 attrib = in_attrib + VERTEX_ATTRIB_START_SLOT;
983
984 if (swr_vs) {
985 semantic_name = swr_vs->info.base.output_semantic_name[in_attrib];
986 if (semantic_name == TGSI_SEMANTIC_POSITION) {
987 attrib = VERTEX_POSITION_SLOT;
988 } else if (semantic_name == TGSI_SEMANTIC_PSIZE) {
989 attrib = VERTEX_SGV_SLOT;
990 } else if (semantic_name == TGSI_SEMANTIC_LAYER) {
991 attrib = VERTEX_SGV_SLOT;
992 } else {
993 if (swr_vs->info.base.writes_position) {
994 attrib--;
995 }
996 }
997 }
998
999 return attrib;
1000 }
1001
1002 static unsigned
locate_linkage(ubyte name,ubyte index,struct tgsi_shader_info * info)1003 locate_linkage(ubyte name, ubyte index, struct tgsi_shader_info *info)
1004 {
1005 for (int i = 0; i < PIPE_MAX_SHADER_OUTPUTS; i++) {
1006 if ((info->output_semantic_name[i] == name)
1007 && (info->output_semantic_index[i] == index)) {
1008 return i;
1009 }
1010 }
1011
1012 return 0xFFFFFFFF;
1013 }
1014
1015 PFN_PIXEL_KERNEL
CompileFS(struct swr_context * ctx,swr_jit_fs_key & key)1016 BuilderSWR::CompileFS(struct swr_context *ctx, swr_jit_fs_key &key)
1017 {
1018 struct swr_fragment_shader *swr_fs = ctx->fs;
1019
1020 struct tgsi_shader_info *pPrevShader;
1021 if (ctx->gs)
1022 pPrevShader = &ctx->gs->info.base;
1023 else
1024 pPrevShader = &ctx->vs->info.base;
1025
1026 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
1027 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
1028
1029 memset(inputs, 0, sizeof(inputs));
1030 memset(outputs, 0, sizeof(outputs));
1031
1032 struct lp_build_sampler_soa *sampler = NULL;
1033
1034 AttrBuilder attrBuilder;
1035 attrBuilder.addStackAlignmentAttr(JM()->mVWidth * sizeof(float));
1036
1037 std::vector<Type *> fsArgs{PointerType::get(Gen_swr_draw_context(JM()), 0),
1038 PointerType::get(Gen_SWR_PS_CONTEXT(JM()), 0)};
1039 FunctionType *funcType =
1040 FunctionType::get(Type::getVoidTy(JM()->mContext), fsArgs, false);
1041
1042 auto pFunction = Function::Create(funcType,
1043 GlobalValue::ExternalLinkage,
1044 "FS",
1045 JM()->mpCurrentModule);
1046 #if HAVE_LLVM < 0x0500
1047 AttributeSet attrSet = AttributeSet::get(
1048 JM()->mContext, AttributeSet::FunctionIndex, attrBuilder);
1049 pFunction->addAttributes(AttributeSet::FunctionIndex, attrSet);
1050 #else
1051 pFunction->addAttributes(AttributeList::FunctionIndex, attrBuilder);
1052 #endif
1053
1054 BasicBlock *block = BasicBlock::Create(JM()->mContext, "entry", pFunction);
1055 IRB()->SetInsertPoint(block);
1056 LLVMPositionBuilderAtEnd(gallivm->builder, wrap(block));
1057
1058 auto args = pFunction->arg_begin();
1059 Value *hPrivateData = &*args++;
1060 hPrivateData->setName("hPrivateData");
1061 Value *pPS = &*args++;
1062 pPS->setName("psCtx");
1063
1064 Value *consts_ptr = GEP(hPrivateData, {0, swr_draw_context_constantFS});
1065 consts_ptr->setName("fs_constants");
1066 Value *const_sizes_ptr =
1067 GEP(hPrivateData, {0, swr_draw_context_num_constantsFS});
1068 const_sizes_ptr->setName("num_fs_constants");
1069
1070 // load *pAttribs, *pPerspAttribs
1071 Value *pRawAttribs = LOAD(pPS, {0, SWR_PS_CONTEXT_pAttribs}, "pRawAttribs");
1072 Value *pPerspAttribs =
1073 LOAD(pPS, {0, SWR_PS_CONTEXT_pPerspAttribs}, "pPerspAttribs");
1074
1075 swr_fs->constantMask = 0;
1076 swr_fs->flatConstantMask = 0;
1077 swr_fs->pointSpriteMask = 0;
1078
1079 for (int attrib = 0; attrib < PIPE_MAX_SHADER_INPUTS; attrib++) {
1080 const unsigned mask = swr_fs->info.base.input_usage_mask[attrib];
1081 const unsigned interpMode = swr_fs->info.base.input_interpolate[attrib];
1082 const unsigned interpLoc = swr_fs->info.base.input_interpolate_loc[attrib];
1083
1084 if (!mask)
1085 continue;
1086
1087 // load i,j
1088 Value *vi = nullptr, *vj = nullptr;
1089 switch (interpLoc) {
1090 case TGSI_INTERPOLATE_LOC_CENTER:
1091 vi = LOAD(pPS, {0, SWR_PS_CONTEXT_vI, PixelPositions_center}, "i");
1092 vj = LOAD(pPS, {0, SWR_PS_CONTEXT_vJ, PixelPositions_center}, "j");
1093 break;
1094 case TGSI_INTERPOLATE_LOC_CENTROID:
1095 vi = LOAD(pPS, {0, SWR_PS_CONTEXT_vI, PixelPositions_centroid}, "i");
1096 vj = LOAD(pPS, {0, SWR_PS_CONTEXT_vJ, PixelPositions_centroid}, "j");
1097 break;
1098 case TGSI_INTERPOLATE_LOC_SAMPLE:
1099 vi = LOAD(pPS, {0, SWR_PS_CONTEXT_vI, PixelPositions_sample}, "i");
1100 vj = LOAD(pPS, {0, SWR_PS_CONTEXT_vJ, PixelPositions_sample}, "j");
1101 break;
1102 }
1103
1104 // load/compute w
1105 Value *vw = nullptr, *pAttribs;
1106 if (interpMode == TGSI_INTERPOLATE_PERSPECTIVE ||
1107 interpMode == TGSI_INTERPOLATE_COLOR) {
1108 pAttribs = pPerspAttribs;
1109 switch (interpLoc) {
1110 case TGSI_INTERPOLATE_LOC_CENTER:
1111 vw = VRCP(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_center}));
1112 break;
1113 case TGSI_INTERPOLATE_LOC_CENTROID:
1114 vw = VRCP(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_centroid}));
1115 break;
1116 case TGSI_INTERPOLATE_LOC_SAMPLE:
1117 vw = VRCP(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_sample}));
1118 break;
1119 }
1120 } else {
1121 pAttribs = pRawAttribs;
1122 vw = VIMMED1(1.f);
1123 }
1124
1125 vw->setName("w");
1126
1127 ubyte semantic_name = swr_fs->info.base.input_semantic_name[attrib];
1128 ubyte semantic_idx = swr_fs->info.base.input_semantic_index[attrib];
1129
1130 if (semantic_name == TGSI_SEMANTIC_FACE) {
1131 Value *ff =
1132 UI_TO_FP(LOAD(pPS, {0, SWR_PS_CONTEXT_frontFace}), mFP32Ty);
1133 ff = FSUB(FMUL(ff, C(2.0f)), C(1.0f));
1134 ff = VECTOR_SPLAT(JM()->mVWidth, ff, "vFrontFace");
1135
1136 inputs[attrib][0] = wrap(ff);
1137 inputs[attrib][1] = wrap(VIMMED1(0.0f));
1138 inputs[attrib][2] = wrap(VIMMED1(0.0f));
1139 inputs[attrib][3] = wrap(VIMMED1(1.0f));
1140 continue;
1141 } else if (semantic_name == TGSI_SEMANTIC_POSITION) { // gl_FragCoord
1142 if (swr_fs->info.base.properties[TGSI_PROPERTY_FS_COORD_PIXEL_CENTER] ==
1143 TGSI_FS_COORD_PIXEL_CENTER_HALF_INTEGER) {
1144 inputs[attrib][0] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vX, PixelPositions_center}, "vX"));
1145 inputs[attrib][1] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vY, PixelPositions_center}, "vY"));
1146 } else {
1147 inputs[attrib][0] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vX, PixelPositions_UL}, "vX"));
1148 inputs[attrib][1] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vY, PixelPositions_UL}, "vY"));
1149 }
1150 inputs[attrib][2] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vZ}, "vZ"));
1151 inputs[attrib][3] =
1152 wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_center}, "vOneOverW"));
1153 continue;
1154 }
1155
1156 unsigned linkedAttrib =
1157 locate_linkage(semantic_name, semantic_idx, pPrevShader) - 1;
1158
1159 uint32_t extraAttribs = 0;
1160 if (semantic_name == TGSI_SEMANTIC_PRIMID && !ctx->gs) {
1161 /* non-gs generated primID - need to grab from swizzleMap override */
1162 linkedAttrib = pPrevShader->num_outputs - 1;
1163 swr_fs->constantMask |= 1 << linkedAttrib;
1164 extraAttribs++;
1165 } else if (semantic_name == TGSI_SEMANTIC_GENERIC &&
1166 key.sprite_coord_enable & (1 << semantic_idx)) {
1167 /* we add an extra attrib to the backendState in swr_update_derived. */
1168 linkedAttrib = pPrevShader->num_outputs + extraAttribs - 1;
1169 swr_fs->pointSpriteMask |= (1 << linkedAttrib);
1170 extraAttribs++;
1171 } else if (linkedAttrib == 0xFFFFFFFF) {
1172 inputs[attrib][0] = wrap(VIMMED1(0.0f));
1173 inputs[attrib][1] = wrap(VIMMED1(0.0f));
1174 inputs[attrib][2] = wrap(VIMMED1(0.0f));
1175 inputs[attrib][3] = wrap(VIMMED1(1.0f));
1176 /* If we're reading in color and 2-sided lighting is enabled, we have
1177 * to keep going.
1178 */
1179 if (semantic_name != TGSI_SEMANTIC_COLOR || !key.light_twoside)
1180 continue;
1181 } else {
1182 if (interpMode == TGSI_INTERPOLATE_CONSTANT) {
1183 swr_fs->constantMask |= 1 << linkedAttrib;
1184 } else if (interpMode == TGSI_INTERPOLATE_COLOR) {
1185 swr_fs->flatConstantMask |= 1 << linkedAttrib;
1186 }
1187 }
1188
1189 unsigned bcolorAttrib = 0xFFFFFFFF;
1190 Value *offset = NULL;
1191 if (semantic_name == TGSI_SEMANTIC_COLOR && key.light_twoside) {
1192 bcolorAttrib = locate_linkage(
1193 TGSI_SEMANTIC_BCOLOR, semantic_idx, pPrevShader) - 1;
1194 /* Neither front nor back colors were available. Nothing to load. */
1195 if (bcolorAttrib == 0xFFFFFFFF && linkedAttrib == 0xFFFFFFFF)
1196 continue;
1197 /* If there is no front color, just always use the back color. */
1198 if (linkedAttrib == 0xFFFFFFFF)
1199 linkedAttrib = bcolorAttrib;
1200
1201 if (bcolorAttrib != 0xFFFFFFFF) {
1202 if (interpMode == TGSI_INTERPOLATE_CONSTANT) {
1203 swr_fs->constantMask |= 1 << bcolorAttrib;
1204 } else if (interpMode == TGSI_INTERPOLATE_COLOR) {
1205 swr_fs->flatConstantMask |= 1 << bcolorAttrib;
1206 }
1207
1208 unsigned diff = 12 * (bcolorAttrib - linkedAttrib);
1209
1210 if (diff) {
1211 Value *back =
1212 XOR(C(1), LOAD(pPS, {0, SWR_PS_CONTEXT_frontFace}), "backFace");
1213
1214 offset = MUL(back, C(diff));
1215 offset->setName("offset");
1216 }
1217 }
1218 }
1219
1220 for (int channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
1221 if (mask & (1 << channel)) {
1222 Value *indexA = C(linkedAttrib * 12 + channel);
1223 Value *indexB = C(linkedAttrib * 12 + channel + 4);
1224 Value *indexC = C(linkedAttrib * 12 + channel + 8);
1225
1226 if (offset) {
1227 indexA = ADD(indexA, offset);
1228 indexB = ADD(indexB, offset);
1229 indexC = ADD(indexC, offset);
1230 }
1231
1232 Value *va = VBROADCAST(LOAD(GEP(pAttribs, indexA)));
1233 Value *vb = VBROADCAST(LOAD(GEP(pAttribs, indexB)));
1234 Value *vc = VBROADCAST(LOAD(GEP(pAttribs, indexC)));
1235
1236 if (interpMode == TGSI_INTERPOLATE_CONSTANT) {
1237 inputs[attrib][channel] = wrap(va);
1238 } else {
1239 Value *vk = FSUB(FSUB(VIMMED1(1.0f), vi), vj);
1240
1241 vc = FMUL(vk, vc);
1242
1243 Value *interp = FMUL(va, vi);
1244 Value *interp1 = FMUL(vb, vj);
1245 interp = FADD(interp, interp1);
1246 interp = FADD(interp, vc);
1247 if (interpMode == TGSI_INTERPOLATE_PERSPECTIVE ||
1248 interpMode == TGSI_INTERPOLATE_COLOR)
1249 interp = FMUL(interp, vw);
1250 inputs[attrib][channel] = wrap(interp);
1251 }
1252 }
1253 }
1254 }
1255
1256 sampler = swr_sampler_soa_create(key.sampler, PIPE_SHADER_FRAGMENT);
1257
1258 struct lp_bld_tgsi_system_values system_values;
1259 memset(&system_values, 0, sizeof(system_values));
1260
1261 struct lp_build_mask_context mask;
1262 bool uses_mask = false;
1263
1264 if (swr_fs->info.base.uses_kill ||
1265 key.poly_stipple_enable) {
1266 Value *vActiveMask = NULL;
1267 if (swr_fs->info.base.uses_kill) {
1268 vActiveMask = LOAD(pPS, {0, SWR_PS_CONTEXT_activeMask}, "activeMask");
1269 }
1270 if (key.poly_stipple_enable) {
1271 // first get fragment xy coords and clip to stipple bounds
1272 Value *vXf = LOAD(pPS, {0, SWR_PS_CONTEXT_vX, PixelPositions_UL});
1273 Value *vYf = LOAD(pPS, {0, SWR_PS_CONTEXT_vY, PixelPositions_UL});
1274 Value *vXu = FP_TO_UI(vXf, mSimdInt32Ty);
1275 Value *vYu = FP_TO_UI(vYf, mSimdInt32Ty);
1276
1277 // stipple pattern is 32x32, which means that one line of stipple
1278 // is stored in one word:
1279 // vXstipple is bit offset inside 32-bit stipple word
1280 // vYstipple is word index is stipple array
1281 Value *vXstipple = AND(vXu, VIMMED1(0x1f)); // & (32-1)
1282 Value *vYstipple = AND(vYu, VIMMED1(0x1f)); // & (32-1)
1283
1284 // grab stipple pattern base address
1285 Value *stipplePtr = GEP(hPrivateData, {0, swr_draw_context_polyStipple, 0});
1286 stipplePtr = BITCAST(stipplePtr, mInt8PtrTy);
1287
1288 // peform a gather to grab stipple words for each lane
1289 Value *vStipple = GATHERDD(VUNDEF_I(), stipplePtr, vYstipple,
1290 VIMMED1(0xffffffff), 4);
1291
1292 // create a mask with one bit corresponding to the x stipple
1293 // and AND it with the pattern, to see if we have a bit
1294 Value *vBitMask = LSHR(VIMMED1(0x80000000), vXstipple);
1295 Value *vStippleMask = AND(vStipple, vBitMask);
1296 vStippleMask = ICMP_NE(vStippleMask, VIMMED1(0));
1297 vStippleMask = VMASK(vStippleMask);
1298
1299 if (swr_fs->info.base.uses_kill) {
1300 vActiveMask = AND(vActiveMask, vStippleMask);
1301 } else {
1302 vActiveMask = vStippleMask;
1303 }
1304 }
1305 lp_build_mask_begin(
1306 &mask, gallivm, lp_type_float_vec(32, 32 * 8), wrap(vActiveMask));
1307 uses_mask = true;
1308 }
1309
1310 lp_build_tgsi_soa(gallivm,
1311 swr_fs->pipe.tokens,
1312 lp_type_float_vec(32, 32 * 8),
1313 uses_mask ? &mask : NULL, // mask
1314 wrap(consts_ptr),
1315 wrap(const_sizes_ptr),
1316 &system_values,
1317 inputs,
1318 outputs,
1319 wrap(hPrivateData),
1320 NULL, // thread data
1321 sampler, // sampler
1322 &swr_fs->info.base,
1323 NULL); // geometry shader face
1324
1325 sampler->destroy(sampler);
1326
1327 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
1328
1329 for (uint32_t attrib = 0; attrib < swr_fs->info.base.num_outputs;
1330 attrib++) {
1331 switch (swr_fs->info.base.output_semantic_name[attrib]) {
1332 case TGSI_SEMANTIC_POSITION: {
1333 // write z
1334 LLVMValueRef outZ =
1335 LLVMBuildLoad(gallivm->builder, outputs[attrib][2], "");
1336 STORE(unwrap(outZ), pPS, {0, SWR_PS_CONTEXT_vZ});
1337 break;
1338 }
1339 case TGSI_SEMANTIC_COLOR: {
1340 for (uint32_t channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
1341 if (!outputs[attrib][channel])
1342 continue;
1343
1344 LLVMValueRef out =
1345 LLVMBuildLoad(gallivm->builder, outputs[attrib][channel], "");
1346 if (swr_fs->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS] &&
1347 swr_fs->info.base.output_semantic_index[attrib] == 0) {
1348 for (uint32_t rt = 0; rt < key.nr_cbufs; rt++) {
1349 STORE(unwrap(out),
1350 pPS,
1351 {0, SWR_PS_CONTEXT_shaded, rt, channel});
1352 }
1353 } else {
1354 STORE(unwrap(out),
1355 pPS,
1356 {0,
1357 SWR_PS_CONTEXT_shaded,
1358 swr_fs->info.base.output_semantic_index[attrib],
1359 channel});
1360 }
1361 }
1362 break;
1363 }
1364 default: {
1365 fprintf(stderr,
1366 "unknown output from FS %s[%d]\n",
1367 tgsi_semantic_names[swr_fs->info.base
1368 .output_semantic_name[attrib]],
1369 swr_fs->info.base.output_semantic_index[attrib]);
1370 break;
1371 }
1372 }
1373 }
1374
1375 LLVMValueRef mask_result = 0;
1376 if (uses_mask) {
1377 mask_result = lp_build_mask_end(&mask);
1378 }
1379
1380 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
1381
1382 if (uses_mask) {
1383 STORE(unwrap(mask_result), pPS, {0, SWR_PS_CONTEXT_activeMask});
1384 }
1385
1386 RET_VOID();
1387
1388 gallivm_verify_function(gallivm, wrap(pFunction));
1389
1390 gallivm_compile_module(gallivm);
1391
1392 PFN_PIXEL_KERNEL kernel =
1393 (PFN_PIXEL_KERNEL)gallivm_jit_function(gallivm, wrap(pFunction));
1394 debug_printf("frag shader %p\n", kernel);
1395 assert(kernel && "Error: FragShader = NULL");
1396
1397 JM()->mIsModuleFinalized = true;
1398
1399 return kernel;
1400 }
1401
1402 PFN_PIXEL_KERNEL
swr_compile_fs(struct swr_context * ctx,swr_jit_fs_key & key)1403 swr_compile_fs(struct swr_context *ctx, swr_jit_fs_key &key)
1404 {
1405 if (!ctx->fs->pipe.tokens)
1406 return NULL;
1407
1408 BuilderSWR builder(
1409 reinterpret_cast<JitManager *>(swr_screen(ctx->pipe.screen)->hJitMgr),
1410 "FS");
1411 PFN_PIXEL_KERNEL func = builder.CompileFS(ctx, key);
1412
1413 ctx->fs->map.insert(std::make_pair(key, make_unique<VariantFS>(builder.gallivm, func)));
1414 return func;
1415 }
1416