1 //! Various operations on integer and floating-point numbers
2
3 use crate::prelude::*;
4
bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC>5 pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
6 use BinOp::*;
7 use IntCC::*;
8 Some(match bin_op {
9 Eq => Equal,
10 Lt => {
11 if signed {
12 SignedLessThan
13 } else {
14 UnsignedLessThan
15 }
16 }
17 Le => {
18 if signed {
19 SignedLessThanOrEqual
20 } else {
21 UnsignedLessThanOrEqual
22 }
23 }
24 Ne => NotEqual,
25 Ge => {
26 if signed {
27 SignedGreaterThanOrEqual
28 } else {
29 UnsignedGreaterThanOrEqual
30 }
31 }
32 Gt => {
33 if signed {
34 SignedGreaterThan
35 } else {
36 UnsignedGreaterThan
37 }
38 }
39 _ => return None,
40 })
41 }
42
codegen_compare_bin_op<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, signed: bool, lhs: Value, rhs: Value, ) -> CValue<'tcx>43 fn codegen_compare_bin_op<'tcx>(
44 fx: &mut FunctionCx<'_, '_, 'tcx>,
45 bin_op: BinOp,
46 signed: bool,
47 lhs: Value,
48 rhs: Value,
49 ) -> CValue<'tcx> {
50 let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap();
51 let val = fx.bcx.ins().icmp(intcc, lhs, rhs);
52 CValue::by_val(val, fx.layout_of(fx.tcx.types.bool))
53 }
54
codegen_binop<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, in_lhs: CValue<'tcx>, in_rhs: CValue<'tcx>, ) -> CValue<'tcx>55 pub(crate) fn codegen_binop<'tcx>(
56 fx: &mut FunctionCx<'_, '_, 'tcx>,
57 bin_op: BinOp,
58 in_lhs: CValue<'tcx>,
59 in_rhs: CValue<'tcx>,
60 ) -> CValue<'tcx> {
61 match bin_op {
62 BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
63 match in_lhs.layout().ty.kind() {
64 ty::Bool | ty::Uint(_) | ty::Int(_) | ty::Char => {
65 let signed = type_sign(in_lhs.layout().ty);
66 let lhs = in_lhs.load_scalar(fx);
67 let rhs = in_rhs.load_scalar(fx);
68
69 return codegen_compare_bin_op(fx, bin_op, signed, lhs, rhs);
70 }
71 _ => {}
72 }
73 }
74 _ => {}
75 }
76
77 match in_lhs.layout().ty.kind() {
78 ty::Bool => crate::num::codegen_bool_binop(fx, bin_op, in_lhs, in_rhs),
79 ty::Uint(_) | ty::Int(_) => crate::num::codegen_int_binop(fx, bin_op, in_lhs, in_rhs),
80 ty::Float(_) => crate::num::codegen_float_binop(fx, bin_op, in_lhs, in_rhs),
81 ty::RawPtr(..) | ty::FnPtr(..) => crate::num::codegen_ptr_binop(fx, bin_op, in_lhs, in_rhs),
82 _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
83 }
84 }
85
codegen_bool_binop<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, in_lhs: CValue<'tcx>, in_rhs: CValue<'tcx>, ) -> CValue<'tcx>86 pub(crate) fn codegen_bool_binop<'tcx>(
87 fx: &mut FunctionCx<'_, '_, 'tcx>,
88 bin_op: BinOp,
89 in_lhs: CValue<'tcx>,
90 in_rhs: CValue<'tcx>,
91 ) -> CValue<'tcx> {
92 let lhs = in_lhs.load_scalar(fx);
93 let rhs = in_rhs.load_scalar(fx);
94
95 let b = fx.bcx.ins();
96 let res = match bin_op {
97 BinOp::BitXor => b.bxor(lhs, rhs),
98 BinOp::BitAnd => b.band(lhs, rhs),
99 BinOp::BitOr => b.bor(lhs, rhs),
100 // Compare binops handles by `codegen_binop`.
101 _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
102 };
103
104 CValue::by_val(res, fx.layout_of(fx.tcx.types.bool))
105 }
106
codegen_int_binop<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, in_lhs: CValue<'tcx>, in_rhs: CValue<'tcx>, ) -> CValue<'tcx>107 pub(crate) fn codegen_int_binop<'tcx>(
108 fx: &mut FunctionCx<'_, '_, 'tcx>,
109 bin_op: BinOp,
110 in_lhs: CValue<'tcx>,
111 in_rhs: CValue<'tcx>,
112 ) -> CValue<'tcx> {
113 if bin_op != BinOp::Shl && bin_op != BinOp::Shr {
114 assert_eq!(
115 in_lhs.layout().ty,
116 in_rhs.layout().ty,
117 "int binop requires lhs and rhs of same type"
118 );
119 }
120
121 if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, in_lhs, in_rhs) {
122 return res;
123 }
124
125 let signed = type_sign(in_lhs.layout().ty);
126
127 let lhs = in_lhs.load_scalar(fx);
128 let rhs = in_rhs.load_scalar(fx);
129
130 let b = fx.bcx.ins();
131 // FIXME trap on overflow for the Unchecked versions
132 let val = match bin_op {
133 BinOp::Add | BinOp::AddUnchecked => b.iadd(lhs, rhs),
134 BinOp::Sub | BinOp::SubUnchecked => b.isub(lhs, rhs),
135 BinOp::Mul | BinOp::MulUnchecked => b.imul(lhs, rhs),
136 BinOp::Div => {
137 if signed {
138 b.sdiv(lhs, rhs)
139 } else {
140 b.udiv(lhs, rhs)
141 }
142 }
143 BinOp::Rem => {
144 if signed {
145 b.srem(lhs, rhs)
146 } else {
147 b.urem(lhs, rhs)
148 }
149 }
150 BinOp::BitXor => b.bxor(lhs, rhs),
151 BinOp::BitAnd => b.band(lhs, rhs),
152 BinOp::BitOr => b.bor(lhs, rhs),
153 BinOp::Shl | BinOp::ShlUnchecked => b.ishl(lhs, rhs),
154 BinOp::Shr | BinOp::ShrUnchecked => {
155 if signed {
156 b.sshr(lhs, rhs)
157 } else {
158 b.ushr(lhs, rhs)
159 }
160 }
161 BinOp::Offset => unreachable!("Offset is not an integer operation"),
162 // Compare binops handles by `codegen_binop`.
163 BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge => {
164 unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty);
165 }
166 };
167
168 CValue::by_val(val, in_lhs.layout())
169 }
170
codegen_checked_int_binop<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, in_lhs: CValue<'tcx>, in_rhs: CValue<'tcx>, ) -> CValue<'tcx>171 pub(crate) fn codegen_checked_int_binop<'tcx>(
172 fx: &mut FunctionCx<'_, '_, 'tcx>,
173 bin_op: BinOp,
174 in_lhs: CValue<'tcx>,
175 in_rhs: CValue<'tcx>,
176 ) -> CValue<'tcx> {
177 let lhs = in_lhs.load_scalar(fx);
178 let rhs = in_rhs.load_scalar(fx);
179
180 if let Some(res) = crate::codegen_i128::maybe_codegen_checked(fx, bin_op, in_lhs, in_rhs) {
181 return res;
182 }
183
184 let signed = type_sign(in_lhs.layout().ty);
185
186 let (res, has_overflow) = match bin_op {
187 BinOp::Add => {
188 /*let (val, c_out) = fx.bcx.ins().iadd_cout(lhs, rhs);
189 (val, c_out)*/
190 // FIXME(CraneStation/cranelift#849) legalize iadd_cout for i8 and i16
191 let val = fx.bcx.ins().iadd(lhs, rhs);
192 let has_overflow = if !signed {
193 fx.bcx.ins().icmp(IntCC::UnsignedLessThan, val, lhs)
194 } else {
195 let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0);
196 let slt = fx.bcx.ins().icmp(IntCC::SignedLessThan, val, lhs);
197 fx.bcx.ins().bxor(rhs_is_negative, slt)
198 };
199 (val, has_overflow)
200 }
201 BinOp::Sub => {
202 /*let (val, b_out) = fx.bcx.ins().isub_bout(lhs, rhs);
203 (val, b_out)*/
204 // FIXME(CraneStation/cranelift#849) legalize isub_bout for i8 and i16
205 let val = fx.bcx.ins().isub(lhs, rhs);
206 let has_overflow = if !signed {
207 fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, val, lhs)
208 } else {
209 let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0);
210 let sgt = fx.bcx.ins().icmp(IntCC::SignedGreaterThan, val, lhs);
211 fx.bcx.ins().bxor(rhs_is_negative, sgt)
212 };
213 (val, has_overflow)
214 }
215 BinOp::Mul => {
216 let ty = fx.bcx.func.dfg.value_type(lhs);
217 match ty {
218 types::I8 | types::I16 | types::I32 if !signed => {
219 let lhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), lhs);
220 let rhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), rhs);
221 let val = fx.bcx.ins().imul(lhs, rhs);
222 let has_overflow = fx.bcx.ins().icmp_imm(
223 IntCC::UnsignedGreaterThan,
224 val,
225 (1 << ty.bits()) - 1,
226 );
227 let val = fx.bcx.ins().ireduce(ty, val);
228 (val, has_overflow)
229 }
230 types::I8 | types::I16 | types::I32 if signed => {
231 let lhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), lhs);
232 let rhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), rhs);
233 let val = fx.bcx.ins().imul(lhs, rhs);
234 let has_underflow =
235 fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1)));
236 let has_overflow = fx.bcx.ins().icmp_imm(
237 IntCC::SignedGreaterThan,
238 val,
239 (1 << (ty.bits() - 1)) - 1,
240 );
241 let val = fx.bcx.ins().ireduce(ty, val);
242 (val, fx.bcx.ins().bor(has_underflow, has_overflow))
243 }
244 types::I64 => {
245 let val = fx.bcx.ins().imul(lhs, rhs);
246 let has_overflow = if !signed {
247 let val_hi = fx.bcx.ins().umulhi(lhs, rhs);
248 fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0)
249 } else {
250 // Based on LLVM's instruction sequence for compiling
251 // a.checked_mul(b).is_some() to riscv64gc:
252 // mulh a2, a0, a1
253 // mul a0, a0, a1
254 // srai a0, a0, 63
255 // xor a0, a0, a2
256 // snez a0, a0
257 let val_hi = fx.bcx.ins().smulhi(lhs, rhs);
258 let val_sign = fx.bcx.ins().sshr_imm(val, i64::from(ty.bits() - 1));
259 let xor = fx.bcx.ins().bxor(val_hi, val_sign);
260 fx.bcx.ins().icmp_imm(IntCC::NotEqual, xor, 0)
261 };
262 (val, has_overflow)
263 }
264 types::I128 => {
265 unreachable!("i128 should have been handled by codegen_i128::maybe_codegen")
266 }
267 _ => unreachable!("invalid non-integer type {}", ty),
268 }
269 }
270 _ => bug!("binop {:?} on checked int/uint lhs: {:?} rhs: {:?}", bin_op, in_lhs, in_rhs),
271 };
272
273 let out_layout = fx.layout_of(Ty::new_tup(fx.tcx, &[in_lhs.layout().ty, fx.tcx.types.bool]));
274 CValue::by_val_pair(res, has_overflow, out_layout)
275 }
276
codegen_saturating_int_binop<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, lhs: CValue<'tcx>, rhs: CValue<'tcx>, ) -> CValue<'tcx>277 pub(crate) fn codegen_saturating_int_binop<'tcx>(
278 fx: &mut FunctionCx<'_, '_, 'tcx>,
279 bin_op: BinOp,
280 lhs: CValue<'tcx>,
281 rhs: CValue<'tcx>,
282 ) -> CValue<'tcx> {
283 assert_eq!(lhs.layout().ty, rhs.layout().ty);
284
285 let signed = type_sign(lhs.layout().ty);
286 let clif_ty = fx.clif_type(lhs.layout().ty).unwrap();
287 let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
288
289 let checked_res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs);
290 let (val, has_overflow) = checked_res.load_scalar_pair(fx);
291
292 let val = match (bin_op, signed) {
293 (BinOp::Add, false) => fx.bcx.ins().select(has_overflow, max, val),
294 (BinOp::Sub, false) => fx.bcx.ins().select(has_overflow, min, val),
295 (BinOp::Add, true) => {
296 let rhs = rhs.load_scalar(fx);
297 let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
298 let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
299 fx.bcx.ins().select(has_overflow, sat_val, val)
300 }
301 (BinOp::Sub, true) => {
302 let rhs = rhs.load_scalar(fx);
303 let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
304 let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
305 fx.bcx.ins().select(has_overflow, sat_val, val)
306 }
307 _ => unreachable!(),
308 };
309
310 CValue::by_val(val, lhs.layout())
311 }
312
codegen_float_binop<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, in_lhs: CValue<'tcx>, in_rhs: CValue<'tcx>, ) -> CValue<'tcx>313 pub(crate) fn codegen_float_binop<'tcx>(
314 fx: &mut FunctionCx<'_, '_, 'tcx>,
315 bin_op: BinOp,
316 in_lhs: CValue<'tcx>,
317 in_rhs: CValue<'tcx>,
318 ) -> CValue<'tcx> {
319 assert_eq!(in_lhs.layout().ty, in_rhs.layout().ty);
320
321 let lhs = in_lhs.load_scalar(fx);
322 let rhs = in_rhs.load_scalar(fx);
323
324 let b = fx.bcx.ins();
325 let res = match bin_op {
326 BinOp::Add => b.fadd(lhs, rhs),
327 BinOp::Sub => b.fsub(lhs, rhs),
328 BinOp::Mul => b.fmul(lhs, rhs),
329 BinOp::Div => b.fdiv(lhs, rhs),
330 BinOp::Rem => {
331 let (name, ty) = match in_lhs.layout().ty.kind() {
332 ty::Float(FloatTy::F32) => ("fmodf", types::F32),
333 ty::Float(FloatTy::F64) => ("fmod", types::F64),
334 _ => bug!(),
335 };
336
337 let ret_val = fx.lib_call(
338 name,
339 vec![AbiParam::new(ty), AbiParam::new(ty)],
340 vec![AbiParam::new(ty)],
341 &[lhs, rhs],
342 )[0];
343
344 return CValue::by_val(ret_val, in_lhs.layout());
345 }
346 BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
347 let fltcc = match bin_op {
348 BinOp::Eq => FloatCC::Equal,
349 BinOp::Lt => FloatCC::LessThan,
350 BinOp::Le => FloatCC::LessThanOrEqual,
351 BinOp::Ne => FloatCC::NotEqual,
352 BinOp::Ge => FloatCC::GreaterThanOrEqual,
353 BinOp::Gt => FloatCC::GreaterThan,
354 _ => unreachable!(),
355 };
356 let val = fx.bcx.ins().fcmp(fltcc, lhs, rhs);
357 return CValue::by_val(val, fx.layout_of(fx.tcx.types.bool));
358 }
359 _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
360 };
361
362 CValue::by_val(res, in_lhs.layout())
363 }
364
codegen_ptr_binop<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, in_lhs: CValue<'tcx>, in_rhs: CValue<'tcx>, ) -> CValue<'tcx>365 pub(crate) fn codegen_ptr_binop<'tcx>(
366 fx: &mut FunctionCx<'_, '_, 'tcx>,
367 bin_op: BinOp,
368 in_lhs: CValue<'tcx>,
369 in_rhs: CValue<'tcx>,
370 ) -> CValue<'tcx> {
371 let is_thin_ptr = in_lhs
372 .layout()
373 .ty
374 .builtin_deref(true)
375 .map(|TypeAndMut { ty, mutbl: _ }| !has_ptr_meta(fx.tcx, ty))
376 .unwrap_or(true);
377
378 if is_thin_ptr {
379 match bin_op {
380 BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
381 let lhs = in_lhs.load_scalar(fx);
382 let rhs = in_rhs.load_scalar(fx);
383
384 codegen_compare_bin_op(fx, bin_op, false, lhs, rhs)
385 }
386 BinOp::Offset => {
387 let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty;
388 let (base, offset) = (in_lhs, in_rhs.load_scalar(fx));
389 let pointee_size = fx.layout_of(pointee_ty).size.bytes();
390 let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
391 let base_val = base.load_scalar(fx);
392 let res = fx.bcx.ins().iadd(base_val, ptr_diff);
393 CValue::by_val(res, base.layout())
394 }
395 _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
396 }
397 } else {
398 let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx);
399 let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx);
400
401 let res = match bin_op {
402 BinOp::Eq => {
403 let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);
404 let extra_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_extra, rhs_extra);
405 fx.bcx.ins().band(ptr_eq, extra_eq)
406 }
407 BinOp::Ne => {
408 let ptr_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_ptr, rhs_ptr);
409 let extra_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_extra, rhs_extra);
410 fx.bcx.ins().bor(ptr_ne, extra_ne)
411 }
412 BinOp::Lt | BinOp::Le | BinOp::Ge | BinOp::Gt => {
413 let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);
414
415 let ptr_cmp =
416 fx.bcx.ins().icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr);
417 let extra_cmp = fx.bcx.ins().icmp(
418 bin_op_to_intcc(bin_op, false).unwrap(),
419 lhs_extra,
420 rhs_extra,
421 );
422
423 fx.bcx.ins().select(ptr_eq, extra_cmp, ptr_cmp)
424 }
425 _ => panic!("bin_op {:?} on ptr", bin_op),
426 };
427
428 CValue::by_val(res, fx.layout_of(fx.tcx.types.bool))
429 }
430 }
431
432 // In Rust floating point min and max don't propagate NaN. In Cranelift they do however.
433 // For this reason it is necessary to use `a.is_nan() ? b : (a >= b ? b : a)` for `minnumf*`
434 // and `a.is_nan() ? b : (a <= b ? b : a)` for `maxnumf*`. NaN checks are done by comparing
435 // a float against itself. Only in case of NaN is it not equal to itself.
codegen_float_min(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value436 pub(crate) fn codegen_float_min(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
437 let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
438 let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b);
439 let temp = fx.bcx.ins().select(a_ge_b, b, a);
440 fx.bcx.ins().select(a_is_nan, b, temp)
441 }
442
codegen_float_max(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value443 pub(crate) fn codegen_float_max(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
444 let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
445 let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b);
446 let temp = fx.bcx.ins().select(a_le_b, b, a);
447 fx.bcx.ins().select(a_is_nan, b, temp)
448 }
449