1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /**
25 * \file lower_instructions.cpp
26 *
27 * Many GPUs lack native instructions for certain expression operations, and
28 * must replace them with some other expression tree. This pass lowers some
29 * of the most common cases, allowing the lowering code to be implemented once
30 * rather than in each driver backend.
31 *
32 * Currently supported transformations:
33 * - SUB_TO_ADD_NEG
34 * - DIV_TO_MUL_RCP
35 * - INT_DIV_TO_MUL_RCP
36 * - EXP_TO_EXP2
37 * - POW_TO_EXP2
38 * - LOG_TO_LOG2
39 * - MOD_TO_FLOOR
40 * - LDEXP_TO_ARITH
41 * - DFREXP_TO_ARITH
42 * - CARRY_TO_ARITH
43 * - BORROW_TO_ARITH
44 * - SAT_TO_CLAMP
45 * - DOPS_TO_DFRAC
46 *
47 * SUB_TO_ADD_NEG:
48 * ---------------
49 * Breaks an ir_binop_sub expression down to add(op0, neg(op1))
50 *
51 * This simplifies expression reassociation, and for many backends
52 * there is no subtract operation separate from adding the negation.
53 * For backends with native subtract operations, they will probably
54 * want to recognize add(op0, neg(op1)) or the other way around to
55 * produce a subtract anyway.
56 *
57 * FDIV_TO_MUL_RCP, DDIV_TO_MUL_RCP, and INT_DIV_TO_MUL_RCP:
58 * ---------------------------------------------------------
59 * Breaks an ir_binop_div expression down to op0 * (rcp(op1)).
60 *
61 * Many GPUs don't have a divide instruction (945 and 965 included),
62 * but they do have an RCP instruction to compute an approximate
63 * reciprocal. By breaking the operation down, constant reciprocals
64 * can get constant folded.
65 *
66 * FDIV_TO_MUL_RCP only lowers single-precision floating point division;
67 * DDIV_TO_MUL_RCP only lowers double-precision floating point division.
68 * DIV_TO_MUL_RCP is a convenience macro that sets both flags.
69 * INT_DIV_TO_MUL_RCP handles the integer case, converting to and from floating
70 * point so that RCP is possible.
71 *
72 * EXP_TO_EXP2 and LOG_TO_LOG2:
73 * ----------------------------
74 * Many GPUs don't have a base e log or exponent instruction, but they
75 * do have base 2 versions, so this pass converts exp and log to exp2
76 * and log2 operations.
77 *
78 * POW_TO_EXP2:
79 * -----------
80 * Many older GPUs don't have an x**y instruction. For these GPUs, convert
81 * x**y to 2**(y * log2(x)).
82 *
83 * MOD_TO_FLOOR:
84 * -------------
85 * Breaks an ir_binop_mod expression down to (op0 - op1 * floor(op0 / op1))
86 *
87 * Many GPUs don't have a MOD instruction (945 and 965 included), and
88 * if we have to break it down like this anyway, it gives an
89 * opportunity to do things like constant fold the (1.0 / op1) easily.
90 *
91 * Note: before we used to implement this as op1 * fract(op / op1) but this
92 * implementation had significant precision errors.
93 *
94 * LDEXP_TO_ARITH:
95 * -------------
96 * Converts ir_binop_ldexp to arithmetic and bit operations for float sources.
97 *
98 * DFREXP_DLDEXP_TO_ARITH:
99 * ---------------
100 * Converts ir_binop_ldexp, ir_unop_frexp_sig, and ir_unop_frexp_exp to
101 * arithmetic and bit ops for double arguments.
102 *
103 * CARRY_TO_ARITH:
104 * ---------------
105 * Converts ir_carry into (x + y) < x.
106 *
107 * BORROW_TO_ARITH:
108 * ----------------
109 * Converts ir_borrow into (x < y).
110 *
111 * SAT_TO_CLAMP:
112 * -------------
113 * Converts ir_unop_saturate into min(max(x, 0.0), 1.0)
114 *
115 * DOPS_TO_DFRAC:
116 * --------------
117 * Converts double trunc, ceil, floor, round to fract
118 */
119
120 #include "c99_math.h"
121 #include "program/prog_instruction.h" /* for swizzle */
122 #include "compiler/glsl_types.h"
123 #include "ir.h"
124 #include "ir_builder.h"
125 #include "ir_optimization.h"
126
127 using namespace ir_builder;
128
129 namespace {
130
131 class lower_instructions_visitor : public ir_hierarchical_visitor {
132 public:
lower_instructions_visitor(unsigned lower)133 lower_instructions_visitor(unsigned lower)
134 : progress(false), lower(lower) { }
135
136 ir_visitor_status visit_leave(ir_expression *);
137
138 bool progress;
139
140 private:
141 unsigned lower; /** Bitfield of which operations to lower */
142
143 void sub_to_add_neg(ir_expression *);
144 void div_to_mul_rcp(ir_expression *);
145 void int_div_to_mul_rcp(ir_expression *);
146 void mod_to_floor(ir_expression *);
147 void exp_to_exp2(ir_expression *);
148 void pow_to_exp2(ir_expression *);
149 void log_to_log2(ir_expression *);
150 void ldexp_to_arith(ir_expression *);
151 void dldexp_to_arith(ir_expression *);
152 void dfrexp_sig_to_arith(ir_expression *);
153 void dfrexp_exp_to_arith(ir_expression *);
154 void carry_to_arith(ir_expression *);
155 void borrow_to_arith(ir_expression *);
156 void sat_to_clamp(ir_expression *);
157 void double_dot_to_fma(ir_expression *);
158 void double_lrp(ir_expression *);
159 void dceil_to_dfrac(ir_expression *);
160 void dfloor_to_dfrac(ir_expression *);
161 void dround_even_to_dfrac(ir_expression *);
162 void dtrunc_to_dfrac(ir_expression *);
163 void dsign_to_csel(ir_expression *);
164 void bit_count_to_math(ir_expression *);
165 void extract_to_shifts(ir_expression *);
166 void insert_to_shifts(ir_expression *);
167 void reverse_to_shifts(ir_expression *ir);
168 void find_lsb_to_float_cast(ir_expression *ir);
169 void find_msb_to_float_cast(ir_expression *ir);
170 void imul_high_to_mul(ir_expression *ir);
171 void sqrt_to_abs_sqrt(ir_expression *ir);
172
173 ir_expression *_carry(operand a, operand b);
174 };
175
176 } /* anonymous namespace */
177
178 /**
179 * Determine if a particular type of lowering should occur
180 */
181 #define lowering(x) (this->lower & x)
182
183 bool
lower_instructions(exec_list * instructions,unsigned what_to_lower)184 lower_instructions(exec_list *instructions, unsigned what_to_lower)
185 {
186 lower_instructions_visitor v(what_to_lower);
187
188 visit_list_elements(&v, instructions);
189 return v.progress;
190 }
191
192 void
sub_to_add_neg(ir_expression * ir)193 lower_instructions_visitor::sub_to_add_neg(ir_expression *ir)
194 {
195 ir->operation = ir_binop_add;
196 ir->init_num_operands();
197 ir->operands[1] = new(ir) ir_expression(ir_unop_neg, ir->operands[1]->type,
198 ir->operands[1], NULL);
199 this->progress = true;
200 }
201
202 void
div_to_mul_rcp(ir_expression * ir)203 lower_instructions_visitor::div_to_mul_rcp(ir_expression *ir)
204 {
205 assert(ir->operands[1]->type->is_float() || ir->operands[1]->type->is_double());
206
207 /* New expression for the 1.0 / op1 */
208 ir_rvalue *expr;
209 expr = new(ir) ir_expression(ir_unop_rcp,
210 ir->operands[1]->type,
211 ir->operands[1]);
212
213 /* op0 / op1 -> op0 * (1.0 / op1) */
214 ir->operation = ir_binop_mul;
215 ir->init_num_operands();
216 ir->operands[1] = expr;
217
218 this->progress = true;
219 }
220
221 void
int_div_to_mul_rcp(ir_expression * ir)222 lower_instructions_visitor::int_div_to_mul_rcp(ir_expression *ir)
223 {
224 assert(ir->operands[1]->type->is_integer());
225
226 /* Be careful with integer division -- we need to do it as a
227 * float and re-truncate, since rcp(n > 1) of an integer would
228 * just be 0.
229 */
230 ir_rvalue *op0, *op1;
231 const struct glsl_type *vec_type;
232
233 vec_type = glsl_type::get_instance(GLSL_TYPE_FLOAT,
234 ir->operands[1]->type->vector_elements,
235 ir->operands[1]->type->matrix_columns);
236
237 if (ir->operands[1]->type->base_type == GLSL_TYPE_INT)
238 op1 = new(ir) ir_expression(ir_unop_i2f, vec_type, ir->operands[1], NULL);
239 else
240 op1 = new(ir) ir_expression(ir_unop_u2f, vec_type, ir->operands[1], NULL);
241
242 op1 = new(ir) ir_expression(ir_unop_rcp, op1->type, op1, NULL);
243
244 vec_type = glsl_type::get_instance(GLSL_TYPE_FLOAT,
245 ir->operands[0]->type->vector_elements,
246 ir->operands[0]->type->matrix_columns);
247
248 if (ir->operands[0]->type->base_type == GLSL_TYPE_INT)
249 op0 = new(ir) ir_expression(ir_unop_i2f, vec_type, ir->operands[0], NULL);
250 else
251 op0 = new(ir) ir_expression(ir_unop_u2f, vec_type, ir->operands[0], NULL);
252
253 vec_type = glsl_type::get_instance(GLSL_TYPE_FLOAT,
254 ir->type->vector_elements,
255 ir->type->matrix_columns);
256
257 op0 = new(ir) ir_expression(ir_binop_mul, vec_type, op0, op1);
258
259 if (ir->operands[1]->type->base_type == GLSL_TYPE_INT) {
260 ir->operation = ir_unop_f2i;
261 ir->operands[0] = op0;
262 } else {
263 ir->operation = ir_unop_i2u;
264 ir->operands[0] = new(ir) ir_expression(ir_unop_f2i, op0);
265 }
266 ir->init_num_operands();
267 ir->operands[1] = NULL;
268
269 this->progress = true;
270 }
271
272 void
exp_to_exp2(ir_expression * ir)273 lower_instructions_visitor::exp_to_exp2(ir_expression *ir)
274 {
275 ir_constant *log2_e = new(ir) ir_constant(float(M_LOG2E));
276
277 ir->operation = ir_unop_exp2;
278 ir->init_num_operands();
279 ir->operands[0] = new(ir) ir_expression(ir_binop_mul, ir->operands[0]->type,
280 ir->operands[0], log2_e);
281 this->progress = true;
282 }
283
284 void
pow_to_exp2(ir_expression * ir)285 lower_instructions_visitor::pow_to_exp2(ir_expression *ir)
286 {
287 ir_expression *const log2_x =
288 new(ir) ir_expression(ir_unop_log2, ir->operands[0]->type,
289 ir->operands[0]);
290
291 ir->operation = ir_unop_exp2;
292 ir->init_num_operands();
293 ir->operands[0] = new(ir) ir_expression(ir_binop_mul, ir->operands[1]->type,
294 ir->operands[1], log2_x);
295 ir->operands[1] = NULL;
296 this->progress = true;
297 }
298
299 void
log_to_log2(ir_expression * ir)300 lower_instructions_visitor::log_to_log2(ir_expression *ir)
301 {
302 ir->operation = ir_binop_mul;
303 ir->init_num_operands();
304 ir->operands[0] = new(ir) ir_expression(ir_unop_log2, ir->operands[0]->type,
305 ir->operands[0], NULL);
306 ir->operands[1] = new(ir) ir_constant(float(1.0 / M_LOG2E));
307 this->progress = true;
308 }
309
310 void
mod_to_floor(ir_expression * ir)311 lower_instructions_visitor::mod_to_floor(ir_expression *ir)
312 {
313 ir_variable *x = new(ir) ir_variable(ir->operands[0]->type, "mod_x",
314 ir_var_temporary);
315 ir_variable *y = new(ir) ir_variable(ir->operands[1]->type, "mod_y",
316 ir_var_temporary);
317 this->base_ir->insert_before(x);
318 this->base_ir->insert_before(y);
319
320 ir_assignment *const assign_x =
321 new(ir) ir_assignment(new(ir) ir_dereference_variable(x),
322 ir->operands[0]);
323 ir_assignment *const assign_y =
324 new(ir) ir_assignment(new(ir) ir_dereference_variable(y),
325 ir->operands[1]);
326
327 this->base_ir->insert_before(assign_x);
328 this->base_ir->insert_before(assign_y);
329
330 ir_expression *const div_expr =
331 new(ir) ir_expression(ir_binop_div, x->type,
332 new(ir) ir_dereference_variable(x),
333 new(ir) ir_dereference_variable(y));
334
335 /* Don't generate new IR that would need to be lowered in an additional
336 * pass.
337 */
338 if ((lowering(FDIV_TO_MUL_RCP) && ir->type->is_float()) ||
339 (lowering(DDIV_TO_MUL_RCP) && ir->type->is_double()))
340 div_to_mul_rcp(div_expr);
341
342 ir_expression *const floor_expr =
343 new(ir) ir_expression(ir_unop_floor, x->type, div_expr);
344
345 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
346 dfloor_to_dfrac(floor_expr);
347
348 ir_expression *const mul_expr =
349 new(ir) ir_expression(ir_binop_mul,
350 new(ir) ir_dereference_variable(y),
351 floor_expr);
352
353 ir->operation = ir_binop_sub;
354 ir->init_num_operands();
355 ir->operands[0] = new(ir) ir_dereference_variable(x);
356 ir->operands[1] = mul_expr;
357 this->progress = true;
358 }
359
360 void
ldexp_to_arith(ir_expression * ir)361 lower_instructions_visitor::ldexp_to_arith(ir_expression *ir)
362 {
363 /* Translates
364 * ir_binop_ldexp x exp
365 * into
366 *
367 * extracted_biased_exp = rshift(bitcast_f2i(abs(x)), exp_shift);
368 * resulting_biased_exp = min(extracted_biased_exp + exp, 255);
369 *
370 * if (extracted_biased_exp >= 255)
371 * return x; // +/-inf, NaN
372 *
373 * sign_mantissa = bitcast_f2u(x) & sign_mantissa_mask;
374 *
375 * if (min(resulting_biased_exp, extracted_biased_exp) < 1)
376 * resulting_biased_exp = 0;
377 * if (resulting_biased_exp >= 255 ||
378 * min(resulting_biased_exp, extracted_biased_exp) < 1) {
379 * sign_mantissa &= sign_mask;
380 * }
381 *
382 * return bitcast_u2f(sign_mantissa |
383 * lshift(i2u(resulting_biased_exp), exp_shift));
384 *
385 * which we can't actually implement as such, since the GLSL IR doesn't
386 * have vectorized if-statements. We actually implement it without branches
387 * using conditional-select:
388 *
389 * extracted_biased_exp = rshift(bitcast_f2i(abs(x)), exp_shift);
390 * resulting_biased_exp = min(extracted_biased_exp + exp, 255);
391 *
392 * sign_mantissa = bitcast_f2u(x) & sign_mantissa_mask;
393 *
394 * flush_to_zero = lequal(min(resulting_biased_exp, extracted_biased_exp), 0);
395 * resulting_biased_exp = csel(flush_to_zero, 0, resulting_biased_exp)
396 * zero_mantissa = logic_or(flush_to_zero,
397 * gequal(resulting_biased_exp, 255));
398 * sign_mantissa = csel(zero_mantissa, sign_mantissa & sign_mask, sign_mantissa);
399 *
400 * result = sign_mantissa |
401 * lshift(i2u(resulting_biased_exp), exp_shift));
402 *
403 * return csel(extracted_biased_exp >= 255, x, bitcast_u2f(result));
404 *
405 * The definition of ldexp in the GLSL spec says:
406 *
407 * "If this product is too large to be represented in the
408 * floating-point type, the result is undefined."
409 *
410 * However, the definition of ldexp in the GLSL ES spec does not contain
411 * this sentence, so we do need to handle overflow correctly.
412 *
413 * There is additional language limiting the defined range of exp, but this
414 * is merely to allow implementations that store 2^exp in a temporary
415 * variable.
416 */
417
418 const unsigned vec_elem = ir->type->vector_elements;
419
420 /* Types */
421 const glsl_type *ivec = glsl_type::get_instance(GLSL_TYPE_INT, vec_elem, 1);
422 const glsl_type *uvec = glsl_type::get_instance(GLSL_TYPE_UINT, vec_elem, 1);
423 const glsl_type *bvec = glsl_type::get_instance(GLSL_TYPE_BOOL, vec_elem, 1);
424
425 /* Temporary variables */
426 ir_variable *x = new(ir) ir_variable(ir->type, "x", ir_var_temporary);
427 ir_variable *exp = new(ir) ir_variable(ivec, "exp", ir_var_temporary);
428 ir_variable *result = new(ir) ir_variable(uvec, "result", ir_var_temporary);
429
430 ir_variable *extracted_biased_exp =
431 new(ir) ir_variable(ivec, "extracted_biased_exp", ir_var_temporary);
432 ir_variable *resulting_biased_exp =
433 new(ir) ir_variable(ivec, "resulting_biased_exp", ir_var_temporary);
434
435 ir_variable *sign_mantissa =
436 new(ir) ir_variable(uvec, "sign_mantissa", ir_var_temporary);
437
438 ir_variable *flush_to_zero =
439 new(ir) ir_variable(bvec, "flush_to_zero", ir_var_temporary);
440 ir_variable *zero_mantissa =
441 new(ir) ir_variable(bvec, "zero_mantissa", ir_var_temporary);
442
443 ir_instruction &i = *base_ir;
444
445 /* Copy <x> and <exp> arguments. */
446 i.insert_before(x);
447 i.insert_before(assign(x, ir->operands[0]));
448 i.insert_before(exp);
449 i.insert_before(assign(exp, ir->operands[1]));
450
451 /* Extract the biased exponent from <x>. */
452 i.insert_before(extracted_biased_exp);
453 i.insert_before(assign(extracted_biased_exp,
454 rshift(bitcast_f2i(abs(x)),
455 new(ir) ir_constant(23, vec_elem))));
456
457 /* The definition of ldexp in the GLSL 4.60 spec says:
458 *
459 * "If exp is greater than +128 (single-precision) or +1024
460 * (double-precision), the value returned is undefined. If exp is less
461 * than -126 (single-precision) or -1022 (double-precision), the value
462 * returned may be flushed to zero."
463 *
464 * So we do not have to guard against the possibility of addition overflow,
465 * which could happen when exp is close to INT_MAX. Addition underflow
466 * cannot happen (the worst case is 0 + (-INT_MAX)).
467 */
468 i.insert_before(resulting_biased_exp);
469 i.insert_before(assign(resulting_biased_exp,
470 min2(add(extracted_biased_exp, exp),
471 new(ir) ir_constant(255, vec_elem))));
472
473 i.insert_before(sign_mantissa);
474 i.insert_before(assign(sign_mantissa,
475 bit_and(bitcast_f2u(x),
476 new(ir) ir_constant(0x807fffffu, vec_elem))));
477
478 /* We flush to zero if the original or resulting biased exponent is 0,
479 * indicating a +/-0.0 or subnormal input or output.
480 *
481 * The mantissa is set to 0 if the resulting biased exponent is 255, since
482 * an overflow should produce a +/-inf result.
483 *
484 * Note that NaN inputs are handled separately.
485 */
486 i.insert_before(flush_to_zero);
487 i.insert_before(assign(flush_to_zero,
488 lequal(min2(resulting_biased_exp,
489 extracted_biased_exp),
490 ir_constant::zero(ir, ivec))));
491 i.insert_before(assign(resulting_biased_exp,
492 csel(flush_to_zero,
493 ir_constant::zero(ir, ivec),
494 resulting_biased_exp)));
495
496 i.insert_before(zero_mantissa);
497 i.insert_before(assign(zero_mantissa,
498 logic_or(flush_to_zero,
499 equal(resulting_biased_exp,
500 new(ir) ir_constant(255, vec_elem)))));
501 i.insert_before(assign(sign_mantissa,
502 csel(zero_mantissa,
503 bit_and(sign_mantissa,
504 new(ir) ir_constant(0x80000000u, vec_elem)),
505 sign_mantissa)));
506
507 /* Don't generate new IR that would need to be lowered in an additional
508 * pass.
509 */
510 i.insert_before(result);
511 if (!lowering(INSERT_TO_SHIFTS)) {
512 i.insert_before(assign(result,
513 bitfield_insert(sign_mantissa,
514 i2u(resulting_biased_exp),
515 new(ir) ir_constant(23u, vec_elem),
516 new(ir) ir_constant(8u, vec_elem))));
517 } else {
518 i.insert_before(assign(result,
519 bit_or(sign_mantissa,
520 lshift(i2u(resulting_biased_exp),
521 new(ir) ir_constant(23, vec_elem)))));
522 }
523
524 ir->operation = ir_triop_csel;
525 ir->init_num_operands();
526 ir->operands[0] = gequal(extracted_biased_exp,
527 new(ir) ir_constant(255, vec_elem));
528 ir->operands[1] = new(ir) ir_dereference_variable(x);
529 ir->operands[2] = bitcast_u2f(result);
530
531 this->progress = true;
532 }
533
534 void
dldexp_to_arith(ir_expression * ir)535 lower_instructions_visitor::dldexp_to_arith(ir_expression *ir)
536 {
537 /* See ldexp_to_arith for structure. Uses frexp_exp to extract the exponent
538 * from the significand.
539 */
540
541 const unsigned vec_elem = ir->type->vector_elements;
542
543 /* Types */
544 const glsl_type *ivec = glsl_type::get_instance(GLSL_TYPE_INT, vec_elem, 1);
545 const glsl_type *bvec = glsl_type::get_instance(GLSL_TYPE_BOOL, vec_elem, 1);
546
547 /* Constants */
548 ir_constant *zeroi = ir_constant::zero(ir, ivec);
549
550 ir_constant *sign_mask = new(ir) ir_constant(0x80000000u);
551
552 ir_constant *exp_shift = new(ir) ir_constant(20u);
553 ir_constant *exp_width = new(ir) ir_constant(11u);
554 ir_constant *exp_bias = new(ir) ir_constant(1022, vec_elem);
555
556 /* Temporary variables */
557 ir_variable *x = new(ir) ir_variable(ir->type, "x", ir_var_temporary);
558 ir_variable *exp = new(ir) ir_variable(ivec, "exp", ir_var_temporary);
559
560 ir_variable *zero_sign_x = new(ir) ir_variable(ir->type, "zero_sign_x",
561 ir_var_temporary);
562
563 ir_variable *extracted_biased_exp =
564 new(ir) ir_variable(ivec, "extracted_biased_exp", ir_var_temporary);
565 ir_variable *resulting_biased_exp =
566 new(ir) ir_variable(ivec, "resulting_biased_exp", ir_var_temporary);
567
568 ir_variable *is_not_zero_or_underflow =
569 new(ir) ir_variable(bvec, "is_not_zero_or_underflow", ir_var_temporary);
570
571 ir_instruction &i = *base_ir;
572
573 /* Copy <x> and <exp> arguments. */
574 i.insert_before(x);
575 i.insert_before(assign(x, ir->operands[0]));
576 i.insert_before(exp);
577 i.insert_before(assign(exp, ir->operands[1]));
578
579 ir_expression *frexp_exp = expr(ir_unop_frexp_exp, x);
580 if (lowering(DFREXP_DLDEXP_TO_ARITH))
581 dfrexp_exp_to_arith(frexp_exp);
582
583 /* Extract the biased exponent from <x>. */
584 i.insert_before(extracted_biased_exp);
585 i.insert_before(assign(extracted_biased_exp, add(frexp_exp, exp_bias)));
586
587 i.insert_before(resulting_biased_exp);
588 i.insert_before(assign(resulting_biased_exp,
589 add(extracted_biased_exp, exp)));
590
591 /* Test if result is ±0.0, subnormal, or underflow by checking if the
592 * resulting biased exponent would be less than 0x1. If so, the result is
593 * 0.0 with the sign of x. (Actually, invert the conditions so that
594 * immediate values are the second arguments, which is better for i965)
595 * TODO: Implement in a vector fashion.
596 */
597 i.insert_before(zero_sign_x);
598 for (unsigned elem = 0; elem < vec_elem; elem++) {
599 ir_variable *unpacked =
600 new(ir) ir_variable(glsl_type::uvec2_type, "unpacked", ir_var_temporary);
601 i.insert_before(unpacked);
602 i.insert_before(
603 assign(unpacked,
604 expr(ir_unop_unpack_double_2x32, swizzle(x, elem, 1))));
605 i.insert_before(assign(unpacked, bit_and(swizzle_y(unpacked), sign_mask->clone(ir, NULL)),
606 WRITEMASK_Y));
607 i.insert_before(assign(unpacked, ir_constant::zero(ir, glsl_type::uint_type), WRITEMASK_X));
608 i.insert_before(assign(zero_sign_x,
609 expr(ir_unop_pack_double_2x32, unpacked),
610 1 << elem));
611 }
612 i.insert_before(is_not_zero_or_underflow);
613 i.insert_before(assign(is_not_zero_or_underflow,
614 gequal(resulting_biased_exp,
615 new(ir) ir_constant(0x1, vec_elem))));
616 i.insert_before(assign(x, csel(is_not_zero_or_underflow,
617 x, zero_sign_x)));
618 i.insert_before(assign(resulting_biased_exp,
619 csel(is_not_zero_or_underflow,
620 resulting_biased_exp, zeroi)));
621
622 /* We could test for overflows by checking if the resulting biased exponent
623 * would be greater than 0xFE. Turns out we don't need to because the GLSL
624 * spec says:
625 *
626 * "If this product is too large to be represented in the
627 * floating-point type, the result is undefined."
628 */
629
630 ir_rvalue *results[4] = {NULL};
631 for (unsigned elem = 0; elem < vec_elem; elem++) {
632 ir_variable *unpacked =
633 new(ir) ir_variable(glsl_type::uvec2_type, "unpacked", ir_var_temporary);
634 i.insert_before(unpacked);
635 i.insert_before(
636 assign(unpacked,
637 expr(ir_unop_unpack_double_2x32, swizzle(x, elem, 1))));
638
639 ir_expression *bfi = bitfield_insert(
640 swizzle_y(unpacked),
641 i2u(swizzle(resulting_biased_exp, elem, 1)),
642 exp_shift->clone(ir, NULL),
643 exp_width->clone(ir, NULL));
644
645 i.insert_before(assign(unpacked, bfi, WRITEMASK_Y));
646
647 results[elem] = expr(ir_unop_pack_double_2x32, unpacked);
648 }
649
650 ir->operation = ir_quadop_vector;
651 ir->init_num_operands();
652 ir->operands[0] = results[0];
653 ir->operands[1] = results[1];
654 ir->operands[2] = results[2];
655 ir->operands[3] = results[3];
656
657 /* Don't generate new IR that would need to be lowered in an additional
658 * pass.
659 */
660
661 this->progress = true;
662 }
663
664 void
dfrexp_sig_to_arith(ir_expression * ir)665 lower_instructions_visitor::dfrexp_sig_to_arith(ir_expression *ir)
666 {
667 const unsigned vec_elem = ir->type->vector_elements;
668 const glsl_type *bvec = glsl_type::get_instance(GLSL_TYPE_BOOL, vec_elem, 1);
669
670 /* Double-precision floating-point values are stored as
671 * 1 sign bit;
672 * 11 exponent bits;
673 * 52 mantissa bits.
674 *
675 * We're just extracting the significand here, so we only need to modify
676 * the upper 32-bit uint. Unfortunately we must extract each double
677 * independently as there is no vector version of unpackDouble.
678 */
679
680 ir_instruction &i = *base_ir;
681
682 ir_variable *is_not_zero =
683 new(ir) ir_variable(bvec, "is_not_zero", ir_var_temporary);
684 ir_rvalue *results[4] = {NULL};
685
686 ir_constant *dzero = new(ir) ir_constant(0.0, vec_elem);
687 i.insert_before(is_not_zero);
688 i.insert_before(
689 assign(is_not_zero,
690 nequal(abs(ir->operands[0]->clone(ir, NULL)), dzero)));
691
692 /* TODO: Remake this as more vector-friendly when int64 support is
693 * available.
694 */
695 for (unsigned elem = 0; elem < vec_elem; elem++) {
696 ir_constant *zero = new(ir) ir_constant(0u, 1);
697 ir_constant *sign_mantissa_mask = new(ir) ir_constant(0x800fffffu, 1);
698
699 /* Exponent of double floating-point values in the range [0.5, 1.0). */
700 ir_constant *exponent_value = new(ir) ir_constant(0x3fe00000u, 1);
701
702 ir_variable *bits =
703 new(ir) ir_variable(glsl_type::uint_type, "bits", ir_var_temporary);
704 ir_variable *unpacked =
705 new(ir) ir_variable(glsl_type::uvec2_type, "unpacked", ir_var_temporary);
706
707 ir_rvalue *x = swizzle(ir->operands[0]->clone(ir, NULL), elem, 1);
708
709 i.insert_before(bits);
710 i.insert_before(unpacked);
711 i.insert_before(assign(unpacked, expr(ir_unop_unpack_double_2x32, x)));
712
713 /* Manipulate the high uint to remove the exponent and replace it with
714 * either the default exponent or zero.
715 */
716 i.insert_before(assign(bits, swizzle_y(unpacked)));
717 i.insert_before(assign(bits, bit_and(bits, sign_mantissa_mask)));
718 i.insert_before(assign(bits, bit_or(bits,
719 csel(swizzle(is_not_zero, elem, 1),
720 exponent_value,
721 zero))));
722 i.insert_before(assign(unpacked, bits, WRITEMASK_Y));
723 results[elem] = expr(ir_unop_pack_double_2x32, unpacked);
724 }
725
726 /* Put the dvec back together */
727 ir->operation = ir_quadop_vector;
728 ir->init_num_operands();
729 ir->operands[0] = results[0];
730 ir->operands[1] = results[1];
731 ir->operands[2] = results[2];
732 ir->operands[3] = results[3];
733
734 this->progress = true;
735 }
736
737 void
dfrexp_exp_to_arith(ir_expression * ir)738 lower_instructions_visitor::dfrexp_exp_to_arith(ir_expression *ir)
739 {
740 const unsigned vec_elem = ir->type->vector_elements;
741 const glsl_type *bvec = glsl_type::get_instance(GLSL_TYPE_BOOL, vec_elem, 1);
742 const glsl_type *uvec = glsl_type::get_instance(GLSL_TYPE_UINT, vec_elem, 1);
743
744 /* Double-precision floating-point values are stored as
745 * 1 sign bit;
746 * 11 exponent bits;
747 * 52 mantissa bits.
748 *
749 * We're just extracting the exponent here, so we only care about the upper
750 * 32-bit uint.
751 */
752
753 ir_instruction &i = *base_ir;
754
755 ir_variable *is_not_zero =
756 new(ir) ir_variable(bvec, "is_not_zero", ir_var_temporary);
757 ir_variable *high_words =
758 new(ir) ir_variable(uvec, "high_words", ir_var_temporary);
759 ir_constant *dzero = new(ir) ir_constant(0.0, vec_elem);
760 ir_constant *izero = new(ir) ir_constant(0, vec_elem);
761
762 ir_rvalue *absval = abs(ir->operands[0]);
763
764 i.insert_before(is_not_zero);
765 i.insert_before(high_words);
766 i.insert_before(assign(is_not_zero, nequal(absval->clone(ir, NULL), dzero)));
767
768 /* Extract all of the upper uints. */
769 for (unsigned elem = 0; elem < vec_elem; elem++) {
770 ir_rvalue *x = swizzle(absval->clone(ir, NULL), elem, 1);
771
772 i.insert_before(assign(high_words,
773 swizzle_y(expr(ir_unop_unpack_double_2x32, x)),
774 1 << elem));
775
776 }
777 ir_constant *exponent_shift = new(ir) ir_constant(20, vec_elem);
778 ir_constant *exponent_bias = new(ir) ir_constant(-1022, vec_elem);
779
780 /* For non-zero inputs, shift the exponent down and apply bias. */
781 ir->operation = ir_triop_csel;
782 ir->init_num_operands();
783 ir->operands[0] = new(ir) ir_dereference_variable(is_not_zero);
784 ir->operands[1] = add(exponent_bias, u2i(rshift(high_words, exponent_shift)));
785 ir->operands[2] = izero;
786
787 this->progress = true;
788 }
789
790 void
carry_to_arith(ir_expression * ir)791 lower_instructions_visitor::carry_to_arith(ir_expression *ir)
792 {
793 /* Translates
794 * ir_binop_carry x y
795 * into
796 * sum = ir_binop_add x y
797 * bcarry = ir_binop_less sum x
798 * carry = ir_unop_b2i bcarry
799 */
800
801 ir_rvalue *x_clone = ir->operands[0]->clone(ir, NULL);
802 ir->operation = ir_unop_i2u;
803 ir->init_num_operands();
804 ir->operands[0] = b2i(less(add(ir->operands[0], ir->operands[1]), x_clone));
805 ir->operands[1] = NULL;
806
807 this->progress = true;
808 }
809
810 void
borrow_to_arith(ir_expression * ir)811 lower_instructions_visitor::borrow_to_arith(ir_expression *ir)
812 {
813 /* Translates
814 * ir_binop_borrow x y
815 * into
816 * bcarry = ir_binop_less x y
817 * carry = ir_unop_b2i bcarry
818 */
819
820 ir->operation = ir_unop_i2u;
821 ir->init_num_operands();
822 ir->operands[0] = b2i(less(ir->operands[0], ir->operands[1]));
823 ir->operands[1] = NULL;
824
825 this->progress = true;
826 }
827
828 void
sat_to_clamp(ir_expression * ir)829 lower_instructions_visitor::sat_to_clamp(ir_expression *ir)
830 {
831 /* Translates
832 * ir_unop_saturate x
833 * into
834 * ir_binop_min (ir_binop_max(x, 0.0), 1.0)
835 */
836
837 ir->operation = ir_binop_min;
838 ir->init_num_operands();
839 ir->operands[0] = new(ir) ir_expression(ir_binop_max, ir->operands[0]->type,
840 ir->operands[0],
841 new(ir) ir_constant(0.0f));
842 ir->operands[1] = new(ir) ir_constant(1.0f);
843
844 this->progress = true;
845 }
846
847 void
double_dot_to_fma(ir_expression * ir)848 lower_instructions_visitor::double_dot_to_fma(ir_expression *ir)
849 {
850 ir_variable *temp = new(ir) ir_variable(ir->operands[0]->type->get_base_type(), "dot_res",
851 ir_var_temporary);
852 this->base_ir->insert_before(temp);
853
854 int nc = ir->operands[0]->type->components();
855 for (int i = nc - 1; i >= 1; i--) {
856 ir_assignment *assig;
857 if (i == (nc - 1)) {
858 assig = assign(temp, mul(swizzle(ir->operands[0]->clone(ir, NULL), i, 1),
859 swizzle(ir->operands[1]->clone(ir, NULL), i, 1)));
860 } else {
861 assig = assign(temp, fma(swizzle(ir->operands[0]->clone(ir, NULL), i, 1),
862 swizzle(ir->operands[1]->clone(ir, NULL), i, 1),
863 temp));
864 }
865 this->base_ir->insert_before(assig);
866 }
867
868 ir->operation = ir_triop_fma;
869 ir->init_num_operands();
870 ir->operands[0] = swizzle(ir->operands[0], 0, 1);
871 ir->operands[1] = swizzle(ir->operands[1], 0, 1);
872 ir->operands[2] = new(ir) ir_dereference_variable(temp);
873
874 this->progress = true;
875
876 }
877
878 void
double_lrp(ir_expression * ir)879 lower_instructions_visitor::double_lrp(ir_expression *ir)
880 {
881 int swizval;
882 ir_rvalue *op0 = ir->operands[0], *op2 = ir->operands[2];
883 ir_constant *one = new(ir) ir_constant(1.0, op2->type->vector_elements);
884
885 switch (op2->type->vector_elements) {
886 case 1:
887 swizval = SWIZZLE_XXXX;
888 break;
889 default:
890 assert(op0->type->vector_elements == op2->type->vector_elements);
891 swizval = SWIZZLE_XYZW;
892 break;
893 }
894
895 ir->operation = ir_triop_fma;
896 ir->init_num_operands();
897 ir->operands[0] = swizzle(op2, swizval, op0->type->vector_elements);
898 ir->operands[2] = mul(sub(one, op2->clone(ir, NULL)), op0);
899
900 this->progress = true;
901 }
902
903 void
dceil_to_dfrac(ir_expression * ir)904 lower_instructions_visitor::dceil_to_dfrac(ir_expression *ir)
905 {
906 /*
907 * frtemp = frac(x);
908 * temp = sub(x, frtemp);
909 * result = temp + ((frtemp != 0.0) ? 1.0 : 0.0);
910 */
911 ir_instruction &i = *base_ir;
912 ir_constant *zero = new(ir) ir_constant(0.0, ir->operands[0]->type->vector_elements);
913 ir_constant *one = new(ir) ir_constant(1.0, ir->operands[0]->type->vector_elements);
914 ir_variable *frtemp = new(ir) ir_variable(ir->operands[0]->type, "frtemp",
915 ir_var_temporary);
916
917 i.insert_before(frtemp);
918 i.insert_before(assign(frtemp, fract(ir->operands[0])));
919
920 ir->operation = ir_binop_add;
921 ir->init_num_operands();
922 ir->operands[0] = sub(ir->operands[0]->clone(ir, NULL), frtemp);
923 ir->operands[1] = csel(nequal(frtemp, zero), one, zero->clone(ir, NULL));
924
925 this->progress = true;
926 }
927
928 void
dfloor_to_dfrac(ir_expression * ir)929 lower_instructions_visitor::dfloor_to_dfrac(ir_expression *ir)
930 {
931 /*
932 * frtemp = frac(x);
933 * result = sub(x, frtemp);
934 */
935 ir->operation = ir_binop_sub;
936 ir->init_num_operands();
937 ir->operands[1] = fract(ir->operands[0]->clone(ir, NULL));
938
939 this->progress = true;
940 }
941 void
dround_even_to_dfrac(ir_expression * ir)942 lower_instructions_visitor::dround_even_to_dfrac(ir_expression *ir)
943 {
944 /*
945 * insane but works
946 * temp = x + 0.5;
947 * frtemp = frac(temp);
948 * t2 = sub(temp, frtemp);
949 * if (frac(x) == 0.5)
950 * result = frac(t2 * 0.5) == 0 ? t2 : t2 - 1;
951 * else
952 * result = t2;
953
954 */
955 ir_instruction &i = *base_ir;
956 ir_variable *frtemp = new(ir) ir_variable(ir->operands[0]->type, "frtemp",
957 ir_var_temporary);
958 ir_variable *temp = new(ir) ir_variable(ir->operands[0]->type, "temp",
959 ir_var_temporary);
960 ir_variable *t2 = new(ir) ir_variable(ir->operands[0]->type, "t2",
961 ir_var_temporary);
962 ir_constant *p5 = new(ir) ir_constant(0.5, ir->operands[0]->type->vector_elements);
963 ir_constant *one = new(ir) ir_constant(1.0, ir->operands[0]->type->vector_elements);
964 ir_constant *zero = new(ir) ir_constant(0.0, ir->operands[0]->type->vector_elements);
965
966 i.insert_before(temp);
967 i.insert_before(assign(temp, add(ir->operands[0], p5)));
968
969 i.insert_before(frtemp);
970 i.insert_before(assign(frtemp, fract(temp)));
971
972 i.insert_before(t2);
973 i.insert_before(assign(t2, sub(temp, frtemp)));
974
975 ir->operation = ir_triop_csel;
976 ir->init_num_operands();
977 ir->operands[0] = equal(fract(ir->operands[0]->clone(ir, NULL)),
978 p5->clone(ir, NULL));
979 ir->operands[1] = csel(equal(fract(mul(t2, p5->clone(ir, NULL))),
980 zero),
981 t2,
982 sub(t2, one));
983 ir->operands[2] = new(ir) ir_dereference_variable(t2);
984
985 this->progress = true;
986 }
987
988 void
dtrunc_to_dfrac(ir_expression * ir)989 lower_instructions_visitor::dtrunc_to_dfrac(ir_expression *ir)
990 {
991 /*
992 * frtemp = frac(x);
993 * temp = sub(x, frtemp);
994 * result = x >= 0 ? temp : temp + (frtemp == 0.0) ? 0 : 1;
995 */
996 ir_rvalue *arg = ir->operands[0];
997 ir_instruction &i = *base_ir;
998
999 ir_constant *zero = new(ir) ir_constant(0.0, arg->type->vector_elements);
1000 ir_constant *one = new(ir) ir_constant(1.0, arg->type->vector_elements);
1001 ir_variable *frtemp = new(ir) ir_variable(arg->type, "frtemp",
1002 ir_var_temporary);
1003 ir_variable *temp = new(ir) ir_variable(ir->operands[0]->type, "temp",
1004 ir_var_temporary);
1005
1006 i.insert_before(frtemp);
1007 i.insert_before(assign(frtemp, fract(arg)));
1008 i.insert_before(temp);
1009 i.insert_before(assign(temp, sub(arg->clone(ir, NULL), frtemp)));
1010
1011 ir->operation = ir_triop_csel;
1012 ir->init_num_operands();
1013 ir->operands[0] = gequal(arg->clone(ir, NULL), zero);
1014 ir->operands[1] = new (ir) ir_dereference_variable(temp);
1015 ir->operands[2] = add(temp,
1016 csel(equal(frtemp, zero->clone(ir, NULL)),
1017 zero->clone(ir, NULL),
1018 one));
1019
1020 this->progress = true;
1021 }
1022
1023 void
dsign_to_csel(ir_expression * ir)1024 lower_instructions_visitor::dsign_to_csel(ir_expression *ir)
1025 {
1026 /*
1027 * temp = x > 0.0 ? 1.0 : 0.0;
1028 * result = x < 0.0 ? -1.0 : temp;
1029 */
1030 ir_rvalue *arg = ir->operands[0];
1031 ir_constant *zero = new(ir) ir_constant(0.0, arg->type->vector_elements);
1032 ir_constant *one = new(ir) ir_constant(1.0, arg->type->vector_elements);
1033 ir_constant *neg_one = new(ir) ir_constant(-1.0, arg->type->vector_elements);
1034
1035 ir->operation = ir_triop_csel;
1036 ir->init_num_operands();
1037 ir->operands[0] = less(arg->clone(ir, NULL),
1038 zero->clone(ir, NULL));
1039 ir->operands[1] = neg_one;
1040 ir->operands[2] = csel(greater(arg, zero),
1041 one,
1042 zero->clone(ir, NULL));
1043
1044 this->progress = true;
1045 }
1046
1047 void
bit_count_to_math(ir_expression * ir)1048 lower_instructions_visitor::bit_count_to_math(ir_expression *ir)
1049 {
1050 /* For more details, see:
1051 *
1052 * http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetPaallel
1053 */
1054 const unsigned elements = ir->operands[0]->type->vector_elements;
1055 ir_variable *temp = new(ir) ir_variable(glsl_type::uvec(elements), "temp",
1056 ir_var_temporary);
1057 ir_constant *c55555555 = new(ir) ir_constant(0x55555555u);
1058 ir_constant *c33333333 = new(ir) ir_constant(0x33333333u);
1059 ir_constant *c0F0F0F0F = new(ir) ir_constant(0x0F0F0F0Fu);
1060 ir_constant *c01010101 = new(ir) ir_constant(0x01010101u);
1061 ir_constant *c1 = new(ir) ir_constant(1u);
1062 ir_constant *c2 = new(ir) ir_constant(2u);
1063 ir_constant *c4 = new(ir) ir_constant(4u);
1064 ir_constant *c24 = new(ir) ir_constant(24u);
1065
1066 base_ir->insert_before(temp);
1067
1068 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1069 base_ir->insert_before(assign(temp, ir->operands[0]));
1070 } else {
1071 assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT);
1072 base_ir->insert_before(assign(temp, i2u(ir->operands[0])));
1073 }
1074
1075 /* temp = temp - ((temp >> 1) & 0x55555555u); */
1076 base_ir->insert_before(assign(temp, sub(temp, bit_and(rshift(temp, c1),
1077 c55555555))));
1078
1079 /* temp = (temp & 0x33333333u) + ((temp >> 2) & 0x33333333u); */
1080 base_ir->insert_before(assign(temp, add(bit_and(temp, c33333333),
1081 bit_and(rshift(temp, c2),
1082 c33333333->clone(ir, NULL)))));
1083
1084 /* int(((temp + (temp >> 4) & 0xF0F0F0Fu) * 0x1010101u) >> 24); */
1085 ir->operation = ir_unop_u2i;
1086 ir->init_num_operands();
1087 ir->operands[0] = rshift(mul(bit_and(add(temp, rshift(temp, c4)), c0F0F0F0F),
1088 c01010101),
1089 c24);
1090
1091 this->progress = true;
1092 }
1093
1094 void
extract_to_shifts(ir_expression * ir)1095 lower_instructions_visitor::extract_to_shifts(ir_expression *ir)
1096 {
1097 ir_variable *bits =
1098 new(ir) ir_variable(ir->operands[0]->type, "bits", ir_var_temporary);
1099
1100 base_ir->insert_before(bits);
1101 base_ir->insert_before(assign(bits, ir->operands[2]));
1102
1103 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1104 ir_constant *c1 =
1105 new(ir) ir_constant(1u, ir->operands[0]->type->vector_elements);
1106 ir_constant *c32 =
1107 new(ir) ir_constant(32u, ir->operands[0]->type->vector_elements);
1108 ir_constant *cFFFFFFFF =
1109 new(ir) ir_constant(0xFFFFFFFFu, ir->operands[0]->type->vector_elements);
1110
1111 /* At least some hardware treats (x << y) as (x << (y%32)). This means
1112 * we'd get a mask of 0 when bits is 32. Special case it.
1113 *
1114 * mask = bits == 32 ? 0xffffffff : (1u << bits) - 1u;
1115 */
1116 ir_expression *mask = csel(equal(bits, c32),
1117 cFFFFFFFF,
1118 sub(lshift(c1, bits), c1->clone(ir, NULL)));
1119
1120 /* Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1121 *
1122 * If bits is zero, the result will be zero.
1123 *
1124 * Since (1 << 0) - 1 == 0, we don't need to bother with the conditional
1125 * select as in the signed integer case.
1126 *
1127 * (value >> offset) & mask;
1128 */
1129 ir->operation = ir_binop_bit_and;
1130 ir->init_num_operands();
1131 ir->operands[0] = rshift(ir->operands[0], ir->operands[1]);
1132 ir->operands[1] = mask;
1133 ir->operands[2] = NULL;
1134 } else {
1135 ir_constant *c0 =
1136 new(ir) ir_constant(int(0), ir->operands[0]->type->vector_elements);
1137 ir_constant *c32 =
1138 new(ir) ir_constant(int(32), ir->operands[0]->type->vector_elements);
1139 ir_variable *temp =
1140 new(ir) ir_variable(ir->operands[0]->type, "temp", ir_var_temporary);
1141
1142 /* temp = 32 - bits; */
1143 base_ir->insert_before(temp);
1144 base_ir->insert_before(assign(temp, sub(c32, bits)));
1145
1146 /* expr = value << (temp - offset)) >> temp; */
1147 ir_expression *expr =
1148 rshift(lshift(ir->operands[0], sub(temp, ir->operands[1])), temp);
1149
1150 /* Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1151 *
1152 * If bits is zero, the result will be zero.
1153 *
1154 * Due to the (x << (y%32)) behavior mentioned before, the (value <<
1155 * (32-0)) doesn't "erase" all of the data as we would like, so finish
1156 * up with:
1157 *
1158 * (bits == 0) ? 0 : e;
1159 */
1160 ir->operation = ir_triop_csel;
1161 ir->init_num_operands();
1162 ir->operands[0] = equal(c0, bits);
1163 ir->operands[1] = c0->clone(ir, NULL);
1164 ir->operands[2] = expr;
1165 }
1166
1167 this->progress = true;
1168 }
1169
1170 void
insert_to_shifts(ir_expression * ir)1171 lower_instructions_visitor::insert_to_shifts(ir_expression *ir)
1172 {
1173 ir_constant *c1;
1174 ir_constant *c32;
1175 ir_constant *cFFFFFFFF;
1176 ir_variable *offset =
1177 new(ir) ir_variable(ir->operands[0]->type, "offset", ir_var_temporary);
1178 ir_variable *bits =
1179 new(ir) ir_variable(ir->operands[0]->type, "bits", ir_var_temporary);
1180 ir_variable *mask =
1181 new(ir) ir_variable(ir->operands[0]->type, "mask", ir_var_temporary);
1182
1183 if (ir->operands[0]->type->base_type == GLSL_TYPE_INT) {
1184 c1 = new(ir) ir_constant(int(1), ir->operands[0]->type->vector_elements);
1185 c32 = new(ir) ir_constant(int(32), ir->operands[0]->type->vector_elements);
1186 cFFFFFFFF = new(ir) ir_constant(int(0xFFFFFFFF), ir->operands[0]->type->vector_elements);
1187 } else {
1188 assert(ir->operands[0]->type->base_type == GLSL_TYPE_UINT);
1189
1190 c1 = new(ir) ir_constant(1u, ir->operands[0]->type->vector_elements);
1191 c32 = new(ir) ir_constant(32u, ir->operands[0]->type->vector_elements);
1192 cFFFFFFFF = new(ir) ir_constant(0xFFFFFFFFu, ir->operands[0]->type->vector_elements);
1193 }
1194
1195 base_ir->insert_before(offset);
1196 base_ir->insert_before(assign(offset, ir->operands[2]));
1197
1198 base_ir->insert_before(bits);
1199 base_ir->insert_before(assign(bits, ir->operands[3]));
1200
1201 /* At least some hardware treats (x << y) as (x << (y%32)). This means
1202 * we'd get a mask of 0 when bits is 32. Special case it.
1203 *
1204 * mask = (bits == 32 ? 0xffffffff : (1u << bits) - 1u) << offset;
1205 *
1206 * Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1207 *
1208 * The result will be undefined if offset or bits is negative, or if the
1209 * sum of offset and bits is greater than the number of bits used to
1210 * store the operand.
1211 *
1212 * Since it's undefined, there are a couple other ways this could be
1213 * implemented. The other way that was considered was to put the csel
1214 * around the whole thing:
1215 *
1216 * final_result = bits == 32 ? insert : ... ;
1217 */
1218 base_ir->insert_before(mask);
1219
1220 base_ir->insert_before(assign(mask, csel(equal(bits, c32),
1221 cFFFFFFFF,
1222 lshift(sub(lshift(c1, bits),
1223 c1->clone(ir, NULL)),
1224 offset))));
1225
1226 /* (base & ~mask) | ((insert << offset) & mask) */
1227 ir->operation = ir_binop_bit_or;
1228 ir->init_num_operands();
1229 ir->operands[0] = bit_and(ir->operands[0], bit_not(mask));
1230 ir->operands[1] = bit_and(lshift(ir->operands[1], offset), mask);
1231 ir->operands[2] = NULL;
1232 ir->operands[3] = NULL;
1233
1234 this->progress = true;
1235 }
1236
1237 void
reverse_to_shifts(ir_expression * ir)1238 lower_instructions_visitor::reverse_to_shifts(ir_expression *ir)
1239 {
1240 /* For more details, see:
1241 *
1242 * http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel
1243 */
1244 ir_constant *c1 =
1245 new(ir) ir_constant(1u, ir->operands[0]->type->vector_elements);
1246 ir_constant *c2 =
1247 new(ir) ir_constant(2u, ir->operands[0]->type->vector_elements);
1248 ir_constant *c4 =
1249 new(ir) ir_constant(4u, ir->operands[0]->type->vector_elements);
1250 ir_constant *c8 =
1251 new(ir) ir_constant(8u, ir->operands[0]->type->vector_elements);
1252 ir_constant *c16 =
1253 new(ir) ir_constant(16u, ir->operands[0]->type->vector_elements);
1254 ir_constant *c33333333 =
1255 new(ir) ir_constant(0x33333333u, ir->operands[0]->type->vector_elements);
1256 ir_constant *c55555555 =
1257 new(ir) ir_constant(0x55555555u, ir->operands[0]->type->vector_elements);
1258 ir_constant *c0F0F0F0F =
1259 new(ir) ir_constant(0x0F0F0F0Fu, ir->operands[0]->type->vector_elements);
1260 ir_constant *c00FF00FF =
1261 new(ir) ir_constant(0x00FF00FFu, ir->operands[0]->type->vector_elements);
1262 ir_variable *temp =
1263 new(ir) ir_variable(glsl_type::uvec(ir->operands[0]->type->vector_elements),
1264 "temp", ir_var_temporary);
1265 ir_instruction &i = *base_ir;
1266
1267 i.insert_before(temp);
1268
1269 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1270 i.insert_before(assign(temp, ir->operands[0]));
1271 } else {
1272 assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT);
1273 i.insert_before(assign(temp, i2u(ir->operands[0])));
1274 }
1275
1276 /* Swap odd and even bits.
1277 *
1278 * temp = ((temp >> 1) & 0x55555555u) | ((temp & 0x55555555u) << 1);
1279 */
1280 i.insert_before(assign(temp, bit_or(bit_and(rshift(temp, c1), c55555555),
1281 lshift(bit_and(temp, c55555555->clone(ir, NULL)),
1282 c1->clone(ir, NULL)))));
1283 /* Swap consecutive pairs.
1284 *
1285 * temp = ((temp >> 2) & 0x33333333u) | ((temp & 0x33333333u) << 2);
1286 */
1287 i.insert_before(assign(temp, bit_or(bit_and(rshift(temp, c2), c33333333),
1288 lshift(bit_and(temp, c33333333->clone(ir, NULL)),
1289 c2->clone(ir, NULL)))));
1290
1291 /* Swap nibbles.
1292 *
1293 * temp = ((temp >> 4) & 0x0F0F0F0Fu) | ((temp & 0x0F0F0F0Fu) << 4);
1294 */
1295 i.insert_before(assign(temp, bit_or(bit_and(rshift(temp, c4), c0F0F0F0F),
1296 lshift(bit_and(temp, c0F0F0F0F->clone(ir, NULL)),
1297 c4->clone(ir, NULL)))));
1298
1299 /* The last step is, basically, bswap. Swap the bytes, then swap the
1300 * words. When this code is run through GCC on x86, it does generate a
1301 * bswap instruction.
1302 *
1303 * temp = ((temp >> 8) & 0x00FF00FFu) | ((temp & 0x00FF00FFu) << 8);
1304 * temp = ( temp >> 16 ) | ( temp << 16);
1305 */
1306 i.insert_before(assign(temp, bit_or(bit_and(rshift(temp, c8), c00FF00FF),
1307 lshift(bit_and(temp, c00FF00FF->clone(ir, NULL)),
1308 c8->clone(ir, NULL)))));
1309
1310 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1311 ir->operation = ir_binop_bit_or;
1312 ir->init_num_operands();
1313 ir->operands[0] = rshift(temp, c16);
1314 ir->operands[1] = lshift(temp, c16->clone(ir, NULL));
1315 } else {
1316 ir->operation = ir_unop_u2i;
1317 ir->init_num_operands();
1318 ir->operands[0] = bit_or(rshift(temp, c16),
1319 lshift(temp, c16->clone(ir, NULL)));
1320 }
1321
1322 this->progress = true;
1323 }
1324
1325 void
find_lsb_to_float_cast(ir_expression * ir)1326 lower_instructions_visitor::find_lsb_to_float_cast(ir_expression *ir)
1327 {
1328 /* For more details, see:
1329 *
1330 * http://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightFloatCast
1331 */
1332 const unsigned elements = ir->operands[0]->type->vector_elements;
1333 ir_constant *c0 = new(ir) ir_constant(unsigned(0), elements);
1334 ir_constant *cminus1 = new(ir) ir_constant(int(-1), elements);
1335 ir_constant *c23 = new(ir) ir_constant(int(23), elements);
1336 ir_constant *c7F = new(ir) ir_constant(int(0x7F), elements);
1337 ir_variable *temp =
1338 new(ir) ir_variable(glsl_type::ivec(elements), "temp", ir_var_temporary);
1339 ir_variable *lsb_only =
1340 new(ir) ir_variable(glsl_type::uvec(elements), "lsb_only", ir_var_temporary);
1341 ir_variable *as_float =
1342 new(ir) ir_variable(glsl_type::vec(elements), "as_float", ir_var_temporary);
1343 ir_variable *lsb =
1344 new(ir) ir_variable(glsl_type::ivec(elements), "lsb", ir_var_temporary);
1345
1346 ir_instruction &i = *base_ir;
1347
1348 i.insert_before(temp);
1349
1350 if (ir->operands[0]->type->base_type == GLSL_TYPE_INT) {
1351 i.insert_before(assign(temp, ir->operands[0]));
1352 } else {
1353 assert(ir->operands[0]->type->base_type == GLSL_TYPE_UINT);
1354 i.insert_before(assign(temp, u2i(ir->operands[0])));
1355 }
1356
1357 /* The int-to-float conversion is lossless because (value & -value) is
1358 * either a power of two or zero. We don't use the result in the zero
1359 * case. The uint() cast is necessary so that 0x80000000 does not
1360 * generate a negative value.
1361 *
1362 * uint lsb_only = uint(value & -value);
1363 * float as_float = float(lsb_only);
1364 */
1365 i.insert_before(lsb_only);
1366 i.insert_before(assign(lsb_only, i2u(bit_and(temp, neg(temp)))));
1367
1368 i.insert_before(as_float);
1369 i.insert_before(assign(as_float, u2f(lsb_only)));
1370
1371 /* This is basically an open-coded frexp. Implementations that have a
1372 * native frexp instruction would be better served by that. This is
1373 * optimized versus a full-featured open-coded implementation in two ways:
1374 *
1375 * - We don't care about a correct result from subnormal numbers (including
1376 * 0.0), so the raw exponent can always be safely unbiased.
1377 *
1378 * - The value cannot be negative, so it does not need to be masked off to
1379 * extract the exponent.
1380 *
1381 * int lsb = (floatBitsToInt(as_float) >> 23) - 0x7f;
1382 */
1383 i.insert_before(lsb);
1384 i.insert_before(assign(lsb, sub(rshift(bitcast_f2i(as_float), c23), c7F)));
1385
1386 /* Use lsb_only in the comparison instead of temp so that the & (far above)
1387 * can possibly generate the result without an explicit comparison.
1388 *
1389 * (lsb_only == 0) ? -1 : lsb;
1390 *
1391 * Since our input values are all integers, the unbiased exponent must not
1392 * be negative. It will only be negative (-0x7f, in fact) if lsb_only is
1393 * 0. Instead of using (lsb_only == 0), we could use (lsb >= 0). Which is
1394 * better is likely GPU dependent. Either way, the difference should be
1395 * small.
1396 */
1397 ir->operation = ir_triop_csel;
1398 ir->init_num_operands();
1399 ir->operands[0] = equal(lsb_only, c0);
1400 ir->operands[1] = cminus1;
1401 ir->operands[2] = new(ir) ir_dereference_variable(lsb);
1402
1403 this->progress = true;
1404 }
1405
1406 void
find_msb_to_float_cast(ir_expression * ir)1407 lower_instructions_visitor::find_msb_to_float_cast(ir_expression *ir)
1408 {
1409 /* For more details, see:
1410 *
1411 * http://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightFloatCast
1412 */
1413 const unsigned elements = ir->operands[0]->type->vector_elements;
1414 ir_constant *c0 = new(ir) ir_constant(int(0), elements);
1415 ir_constant *cminus1 = new(ir) ir_constant(int(-1), elements);
1416 ir_constant *c23 = new(ir) ir_constant(int(23), elements);
1417 ir_constant *c7F = new(ir) ir_constant(int(0x7F), elements);
1418 ir_constant *c000000FF = new(ir) ir_constant(0x000000FFu, elements);
1419 ir_constant *cFFFFFF00 = new(ir) ir_constant(0xFFFFFF00u, elements);
1420 ir_variable *temp =
1421 new(ir) ir_variable(glsl_type::uvec(elements), "temp", ir_var_temporary);
1422 ir_variable *as_float =
1423 new(ir) ir_variable(glsl_type::vec(elements), "as_float", ir_var_temporary);
1424 ir_variable *msb =
1425 new(ir) ir_variable(glsl_type::ivec(elements), "msb", ir_var_temporary);
1426
1427 ir_instruction &i = *base_ir;
1428
1429 i.insert_before(temp);
1430
1431 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1432 i.insert_before(assign(temp, ir->operands[0]));
1433 } else {
1434 assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT);
1435
1436 /* findMSB(uint(abs(some_int))) almost always does the right thing.
1437 * There are two problem values:
1438 *
1439 * * 0x80000000. Since abs(0x80000000) == 0x80000000, findMSB returns
1440 * 31. However, findMSB(int(0x80000000)) == 30.
1441 *
1442 * * 0xffffffff. Since abs(0xffffffff) == 1, findMSB returns
1443 * 31. Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1444 *
1445 * For a value of zero or negative one, -1 will be returned.
1446 *
1447 * For all negative number cases, including 0x80000000 and 0xffffffff,
1448 * the correct value is obtained from findMSB if instead of negating the
1449 * (already negative) value the logical-not is used. A conditonal
1450 * logical-not can be achieved in two instructions.
1451 */
1452 ir_variable *as_int =
1453 new(ir) ir_variable(glsl_type::ivec(elements), "as_int", ir_var_temporary);
1454 ir_constant *c31 = new(ir) ir_constant(int(31), elements);
1455
1456 i.insert_before(as_int);
1457 i.insert_before(assign(as_int, ir->operands[0]));
1458 i.insert_before(assign(temp, i2u(expr(ir_binop_bit_xor,
1459 as_int,
1460 rshift(as_int, c31)))));
1461 }
1462
1463 /* The int-to-float conversion is lossless because bits are conditionally
1464 * masked off the bottom of temp to ensure the value has at most 24 bits of
1465 * data or is zero. We don't use the result in the zero case. The uint()
1466 * cast is necessary so that 0x80000000 does not generate a negative value.
1467 *
1468 * float as_float = float(temp > 255 ? temp & ~255 : temp);
1469 */
1470 i.insert_before(as_float);
1471 i.insert_before(assign(as_float, u2f(csel(greater(temp, c000000FF),
1472 bit_and(temp, cFFFFFF00),
1473 temp))));
1474
1475 /* This is basically an open-coded frexp. Implementations that have a
1476 * native frexp instruction would be better served by that. This is
1477 * optimized versus a full-featured open-coded implementation in two ways:
1478 *
1479 * - We don't care about a correct result from subnormal numbers (including
1480 * 0.0), so the raw exponent can always be safely unbiased.
1481 *
1482 * - The value cannot be negative, so it does not need to be masked off to
1483 * extract the exponent.
1484 *
1485 * int msb = (floatBitsToInt(as_float) >> 23) - 0x7f;
1486 */
1487 i.insert_before(msb);
1488 i.insert_before(assign(msb, sub(rshift(bitcast_f2i(as_float), c23), c7F)));
1489
1490 /* Use msb in the comparison instead of temp so that the subtract can
1491 * possibly generate the result without an explicit comparison.
1492 *
1493 * (msb < 0) ? -1 : msb;
1494 *
1495 * Since our input values are all integers, the unbiased exponent must not
1496 * be negative. It will only be negative (-0x7f, in fact) if temp is 0.
1497 */
1498 ir->operation = ir_triop_csel;
1499 ir->init_num_operands();
1500 ir->operands[0] = less(msb, c0);
1501 ir->operands[1] = cminus1;
1502 ir->operands[2] = new(ir) ir_dereference_variable(msb);
1503
1504 this->progress = true;
1505 }
1506
1507 ir_expression *
_carry(operand a,operand b)1508 lower_instructions_visitor::_carry(operand a, operand b)
1509 {
1510 if (lowering(CARRY_TO_ARITH))
1511 return i2u(b2i(less(add(a, b),
1512 a.val->clone(ralloc_parent(a.val), NULL))));
1513 else
1514 return carry(a, b);
1515 }
1516
1517 void
imul_high_to_mul(ir_expression * ir)1518 lower_instructions_visitor::imul_high_to_mul(ir_expression *ir)
1519 {
1520 /* ABCD
1521 * * EFGH
1522 * ======
1523 * (GH * CD) + (GH * AB) << 16 + (EF * CD) << 16 + (EF * AB) << 32
1524 *
1525 * In GLSL, (a * b) becomes
1526 *
1527 * uint m1 = (a & 0x0000ffffu) * (b & 0x0000ffffu);
1528 * uint m2 = (a & 0x0000ffffu) * (b >> 16);
1529 * uint m3 = (a >> 16) * (b & 0x0000ffffu);
1530 * uint m4 = (a >> 16) * (b >> 16);
1531 *
1532 * uint c1;
1533 * uint c2;
1534 * uint lo_result;
1535 * uint hi_result;
1536 *
1537 * lo_result = uaddCarry(m1, m2 << 16, c1);
1538 * hi_result = m4 + c1;
1539 * lo_result = uaddCarry(lo_result, m3 << 16, c2);
1540 * hi_result = hi_result + c2;
1541 * hi_result = hi_result + (m2 >> 16) + (m3 >> 16);
1542 */
1543 const unsigned elements = ir->operands[0]->type->vector_elements;
1544 ir_variable *src1 =
1545 new(ir) ir_variable(glsl_type::uvec(elements), "src1", ir_var_temporary);
1546 ir_variable *src1h =
1547 new(ir) ir_variable(glsl_type::uvec(elements), "src1h", ir_var_temporary);
1548 ir_variable *src1l =
1549 new(ir) ir_variable(glsl_type::uvec(elements), "src1l", ir_var_temporary);
1550 ir_variable *src2 =
1551 new(ir) ir_variable(glsl_type::uvec(elements), "src2", ir_var_temporary);
1552 ir_variable *src2h =
1553 new(ir) ir_variable(glsl_type::uvec(elements), "src2h", ir_var_temporary);
1554 ir_variable *src2l =
1555 new(ir) ir_variable(glsl_type::uvec(elements), "src2l", ir_var_temporary);
1556 ir_variable *t1 =
1557 new(ir) ir_variable(glsl_type::uvec(elements), "t1", ir_var_temporary);
1558 ir_variable *t2 =
1559 new(ir) ir_variable(glsl_type::uvec(elements), "t2", ir_var_temporary);
1560 ir_variable *lo =
1561 new(ir) ir_variable(glsl_type::uvec(elements), "lo", ir_var_temporary);
1562 ir_variable *hi =
1563 new(ir) ir_variable(glsl_type::uvec(elements), "hi", ir_var_temporary);
1564 ir_variable *different_signs = NULL;
1565 ir_constant *c0000FFFF = new(ir) ir_constant(0x0000FFFFu, elements);
1566 ir_constant *c16 = new(ir) ir_constant(16u, elements);
1567
1568 ir_instruction &i = *base_ir;
1569
1570 i.insert_before(src1);
1571 i.insert_before(src2);
1572 i.insert_before(src1h);
1573 i.insert_before(src2h);
1574 i.insert_before(src1l);
1575 i.insert_before(src2l);
1576
1577 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1578 i.insert_before(assign(src1, ir->operands[0]));
1579 i.insert_before(assign(src2, ir->operands[1]));
1580 } else {
1581 assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT);
1582
1583 ir_variable *itmp1 =
1584 new(ir) ir_variable(glsl_type::ivec(elements), "itmp1", ir_var_temporary);
1585 ir_variable *itmp2 =
1586 new(ir) ir_variable(glsl_type::ivec(elements), "itmp2", ir_var_temporary);
1587 ir_constant *c0 = new(ir) ir_constant(int(0), elements);
1588
1589 i.insert_before(itmp1);
1590 i.insert_before(itmp2);
1591 i.insert_before(assign(itmp1, ir->operands[0]));
1592 i.insert_before(assign(itmp2, ir->operands[1]));
1593
1594 different_signs =
1595 new(ir) ir_variable(glsl_type::bvec(elements), "different_signs",
1596 ir_var_temporary);
1597
1598 i.insert_before(different_signs);
1599 i.insert_before(assign(different_signs, expr(ir_binop_logic_xor,
1600 less(itmp1, c0),
1601 less(itmp2, c0->clone(ir, NULL)))));
1602
1603 i.insert_before(assign(src1, i2u(abs(itmp1))));
1604 i.insert_before(assign(src2, i2u(abs(itmp2))));
1605 }
1606
1607 i.insert_before(assign(src1l, bit_and(src1, c0000FFFF)));
1608 i.insert_before(assign(src2l, bit_and(src2, c0000FFFF->clone(ir, NULL))));
1609 i.insert_before(assign(src1h, rshift(src1, c16)));
1610 i.insert_before(assign(src2h, rshift(src2, c16->clone(ir, NULL))));
1611
1612 i.insert_before(lo);
1613 i.insert_before(hi);
1614 i.insert_before(t1);
1615 i.insert_before(t2);
1616
1617 i.insert_before(assign(lo, mul(src1l, src2l)));
1618 i.insert_before(assign(t1, mul(src1l, src2h)));
1619 i.insert_before(assign(t2, mul(src1h, src2l)));
1620 i.insert_before(assign(hi, mul(src1h, src2h)));
1621
1622 i.insert_before(assign(hi, add(hi, _carry(lo, lshift(t1, c16->clone(ir, NULL))))));
1623 i.insert_before(assign(lo, add(lo, lshift(t1, c16->clone(ir, NULL)))));
1624
1625 i.insert_before(assign(hi, add(hi, _carry(lo, lshift(t2, c16->clone(ir, NULL))))));
1626 i.insert_before(assign(lo, add(lo, lshift(t2, c16->clone(ir, NULL)))));
1627
1628 if (different_signs == NULL) {
1629 assert(ir->operands[0]->type->base_type == GLSL_TYPE_UINT);
1630
1631 ir->operation = ir_binop_add;
1632 ir->init_num_operands();
1633 ir->operands[0] = add(hi, rshift(t1, c16->clone(ir, NULL)));
1634 ir->operands[1] = rshift(t2, c16->clone(ir, NULL));
1635 } else {
1636 assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT);
1637
1638 i.insert_before(assign(hi, add(add(hi, rshift(t1, c16->clone(ir, NULL))),
1639 rshift(t2, c16->clone(ir, NULL)))));
1640
1641 /* For channels where different_signs is set we have to perform a 64-bit
1642 * negation. This is *not* the same as just negating the high 32-bits.
1643 * Consider -3 * 2. The high 32-bits is 0, but the desired result is
1644 * -1, not -0! Recall -x == ~x + 1.
1645 */
1646 ir_variable *neg_hi =
1647 new(ir) ir_variable(glsl_type::ivec(elements), "neg_hi", ir_var_temporary);
1648 ir_constant *c1 = new(ir) ir_constant(1u, elements);
1649
1650 i.insert_before(neg_hi);
1651 i.insert_before(assign(neg_hi, add(bit_not(u2i(hi)),
1652 u2i(_carry(bit_not(lo), c1)))));
1653
1654 ir->operation = ir_triop_csel;
1655 ir->init_num_operands();
1656 ir->operands[0] = new(ir) ir_dereference_variable(different_signs);
1657 ir->operands[1] = new(ir) ir_dereference_variable(neg_hi);
1658 ir->operands[2] = u2i(hi);
1659 }
1660 }
1661
1662 void
sqrt_to_abs_sqrt(ir_expression * ir)1663 lower_instructions_visitor::sqrt_to_abs_sqrt(ir_expression *ir)
1664 {
1665 ir->operands[0] = new(ir) ir_expression(ir_unop_abs, ir->operands[0]);
1666 this->progress = true;
1667 }
1668
1669 ir_visitor_status
visit_leave(ir_expression * ir)1670 lower_instructions_visitor::visit_leave(ir_expression *ir)
1671 {
1672 switch (ir->operation) {
1673 case ir_binop_dot:
1674 if (ir->operands[0]->type->is_double())
1675 double_dot_to_fma(ir);
1676 break;
1677 case ir_triop_lrp:
1678 if (ir->operands[0]->type->is_double())
1679 double_lrp(ir);
1680 break;
1681 case ir_binop_sub:
1682 if (lowering(SUB_TO_ADD_NEG))
1683 sub_to_add_neg(ir);
1684 break;
1685
1686 case ir_binop_div:
1687 if (ir->operands[1]->type->is_integer() && lowering(INT_DIV_TO_MUL_RCP))
1688 int_div_to_mul_rcp(ir);
1689 else if ((ir->operands[1]->type->is_float() && lowering(FDIV_TO_MUL_RCP)) ||
1690 (ir->operands[1]->type->is_double() && lowering(DDIV_TO_MUL_RCP)))
1691 div_to_mul_rcp(ir);
1692 break;
1693
1694 case ir_unop_exp:
1695 if (lowering(EXP_TO_EXP2))
1696 exp_to_exp2(ir);
1697 break;
1698
1699 case ir_unop_log:
1700 if (lowering(LOG_TO_LOG2))
1701 log_to_log2(ir);
1702 break;
1703
1704 case ir_binop_mod:
1705 if (lowering(MOD_TO_FLOOR) && (ir->type->is_float() || ir->type->is_double()))
1706 mod_to_floor(ir);
1707 break;
1708
1709 case ir_binop_pow:
1710 if (lowering(POW_TO_EXP2))
1711 pow_to_exp2(ir);
1712 break;
1713
1714 case ir_binop_ldexp:
1715 if (lowering(LDEXP_TO_ARITH) && ir->type->is_float())
1716 ldexp_to_arith(ir);
1717 if (lowering(DFREXP_DLDEXP_TO_ARITH) && ir->type->is_double())
1718 dldexp_to_arith(ir);
1719 break;
1720
1721 case ir_unop_frexp_exp:
1722 if (lowering(DFREXP_DLDEXP_TO_ARITH) && ir->operands[0]->type->is_double())
1723 dfrexp_exp_to_arith(ir);
1724 break;
1725
1726 case ir_unop_frexp_sig:
1727 if (lowering(DFREXP_DLDEXP_TO_ARITH) && ir->operands[0]->type->is_double())
1728 dfrexp_sig_to_arith(ir);
1729 break;
1730
1731 case ir_binop_carry:
1732 if (lowering(CARRY_TO_ARITH))
1733 carry_to_arith(ir);
1734 break;
1735
1736 case ir_binop_borrow:
1737 if (lowering(BORROW_TO_ARITH))
1738 borrow_to_arith(ir);
1739 break;
1740
1741 case ir_unop_saturate:
1742 if (lowering(SAT_TO_CLAMP))
1743 sat_to_clamp(ir);
1744 break;
1745
1746 case ir_unop_trunc:
1747 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
1748 dtrunc_to_dfrac(ir);
1749 break;
1750
1751 case ir_unop_ceil:
1752 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
1753 dceil_to_dfrac(ir);
1754 break;
1755
1756 case ir_unop_floor:
1757 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
1758 dfloor_to_dfrac(ir);
1759 break;
1760
1761 case ir_unop_round_even:
1762 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
1763 dround_even_to_dfrac(ir);
1764 break;
1765
1766 case ir_unop_sign:
1767 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
1768 dsign_to_csel(ir);
1769 break;
1770
1771 case ir_unop_bit_count:
1772 if (lowering(BIT_COUNT_TO_MATH))
1773 bit_count_to_math(ir);
1774 break;
1775
1776 case ir_triop_bitfield_extract:
1777 if (lowering(EXTRACT_TO_SHIFTS))
1778 extract_to_shifts(ir);
1779 break;
1780
1781 case ir_quadop_bitfield_insert:
1782 if (lowering(INSERT_TO_SHIFTS))
1783 insert_to_shifts(ir);
1784 break;
1785
1786 case ir_unop_bitfield_reverse:
1787 if (lowering(REVERSE_TO_SHIFTS))
1788 reverse_to_shifts(ir);
1789 break;
1790
1791 case ir_unop_find_lsb:
1792 if (lowering(FIND_LSB_TO_FLOAT_CAST))
1793 find_lsb_to_float_cast(ir);
1794 break;
1795
1796 case ir_unop_find_msb:
1797 if (lowering(FIND_MSB_TO_FLOAT_CAST))
1798 find_msb_to_float_cast(ir);
1799 break;
1800
1801 case ir_binop_imul_high:
1802 if (lowering(IMUL_HIGH_TO_MUL))
1803 imul_high_to_mul(ir);
1804 break;
1805
1806 case ir_unop_rsq:
1807 case ir_unop_sqrt:
1808 if (lowering(SQRT_TO_ABS_SQRT))
1809 sqrt_to_abs_sqrt(ir);
1810 break;
1811
1812 default:
1813 return visit_continue;
1814 }
1815
1816 return visit_continue;
1817 }
1818