Lines Matching refs:__
80 #define __ assembler-> macro
101 __ movd(output.AsRegister<CpuRegister>(), input.AsFpuRegister<XmmRegister>(), is64bit); in MoveFPToInt()
107 __ movd(output.AsFpuRegister<XmmRegister>(), input.AsRegister<CpuRegister>(), is64bit); in MoveIntToFP()
154 __ bswapl(out); in GenReverseBytes()
155 __ sarl(out, Immediate(16)); in GenReverseBytes()
158 __ bswapl(out); in GenReverseBytes()
161 __ bswapq(out); in GenReverseBytes()
220 __ movsd(xmm_temp, codegen->LiteralInt64Address(INT64_C(0x7FFFFFFFFFFFFFFF))); in MathAbsFP()
221 __ andpd(output.AsFpuRegister<XmmRegister>(), xmm_temp); in MathAbsFP()
223 __ movss(xmm_temp, codegen->LiteralInt32Address(INT32_C(0x7FFFFFFF))); in MathAbsFP()
224 __ andps(output.AsFpuRegister<XmmRegister>(), xmm_temp); in MathAbsFP()
260 __ movq(mask, out); in GenAbsInteger()
261 __ sarq(mask, Immediate(63)); in GenAbsInteger()
263 __ addq(out, mask); in GenAbsInteger()
264 __ xorq(out, mask); in GenAbsInteger()
267 __ movl(mask, out); in GenAbsInteger()
268 __ sarl(mask, Immediate(31)); in GenAbsInteger()
270 __ addl(out, mask); in GenAbsInteger()
271 __ xorl(out, mask); in GenAbsInteger()
328 __ ucomisd(out, op2); in GenMinMaxFP()
330 __ ucomiss(out, op2); in GenMinMaxFP()
333 __ j(Condition::kParityEven, &nan); in GenMinMaxFP()
335 __ j(is_min ? Condition::kAbove : Condition::kBelow, &op2_label); in GenMinMaxFP()
336 __ j(is_min ? Condition::kBelow : Condition::kAbove, &done); in GenMinMaxFP()
341 __ orpd(out, op2); in GenMinMaxFP()
343 __ orps(out, op2); in GenMinMaxFP()
347 __ andpd(out, op2); in GenMinMaxFP()
349 __ andps(out, op2); in GenMinMaxFP()
352 __ jmp(&done); in GenMinMaxFP()
355 __ Bind(&nan); in GenMinMaxFP()
357 __ movsd(out, codegen->LiteralInt64Address(INT64_C(0x7FF8000000000000))); in GenMinMaxFP()
359 __ movss(out, codegen->LiteralInt32Address(INT32_C(0x7FC00000))); in GenMinMaxFP()
361 __ jmp(&done); in GenMinMaxFP()
364 __ Bind(&op2_label); in GenMinMaxFP()
366 __ movsd(out, op2); in GenMinMaxFP()
368 __ movss(out, op2); in GenMinMaxFP()
372 __ Bind(&done); in GenMinMaxFP()
446 __ cmpq(out, op2); in GenMinMax()
448 __ cmpl(out, op2); in GenMinMax()
451 __ cmov(is_min ? Condition::kGreater : Condition::kLess, out, op2, is_long); in GenMinMax()
560 __ roundsd(out, in, Immediate(round_mode)); in GenSSE41FPToFPIntrinsic()
634 __ movss(inPlusPointFive, codegen_->LiteralFloatAddress(0.5f)); in VisitMathRoundFloat()
637 __ addss(inPlusPointFive, in); in VisitMathRoundFloat()
640 __ roundss(inPlusPointFive, inPlusPointFive, Immediate(1)); in VisitMathRoundFloat()
646 __ comiss(inPlusPointFive, codegen_->LiteralFloatAddress(static_cast<float>(kPrimIntMax))); in VisitMathRoundFloat()
647 __ j(kAboveEqual, &done); in VisitMathRoundFloat()
650 __ j(kUnordered, &nan); in VisitMathRoundFloat()
653 __ cvttss2si(out, inPlusPointFive); in VisitMathRoundFloat()
654 __ jmp(&done); in VisitMathRoundFloat()
655 __ Bind(&nan); in VisitMathRoundFloat()
658 __ xorl(out, out); in VisitMathRoundFloat()
659 __ Bind(&done); in VisitMathRoundFloat()
684 __ movsd(inPlusPointFive, codegen_->LiteralDoubleAddress(0.5)); in VisitMathRoundDouble()
687 __ addsd(inPlusPointFive, in); in VisitMathRoundDouble()
690 __ roundsd(inPlusPointFive, inPlusPointFive, Immediate(1)); in VisitMathRoundDouble()
696 __ comisd(inPlusPointFive, codegen_->LiteralDoubleAddress(static_cast<double>(kPrimLongMax))); in VisitMathRoundDouble()
697 __ j(kAboveEqual, &done); in VisitMathRoundDouble()
700 __ j(kUnordered, &nan); in VisitMathRoundDouble()
703 __ cvttsd2si(out, inPlusPointFive, /* is64bit */ true); in VisitMathRoundDouble()
704 __ jmp(&done); in VisitMathRoundDouble()
705 __ Bind(&nan); in VisitMathRoundDouble()
708 __ xorl(out, out); in VisitMathRoundDouble()
709 __ Bind(&done); in VisitMathRoundDouble()
736 __ gs()->call(Address::Absolute(GetThreadOffset<kX86_64WordSize>(entry), true)); in GenFPToFPCall()
927 __ cmpl(idx, Address(obj, count_offset)); in VisitStringCharAt()
929 __ j(kAboveEqual, slow_path->GetEntryLabel()); in VisitStringCharAt()
932 __ movzxw(out, Address(out, idx, ScaleFactor::TIMES_2, value_offset)); in VisitStringCharAt()
934 __ Bind(slow_path->GetExitLabel()); in VisitStringCharAt()
993 __ cmpl(Address(input, length_offset), in CheckPosition()
996 __ cmpl(Address(input, length_offset), length.AsRegister<CpuRegister>()); in CheckPosition()
998 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1002 __ movl(input_len, Address(input, length_offset)); in CheckPosition()
1003 __ cmpl(input_len, Immediate(pos_const)); in CheckPosition()
1004 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1007 __ leal(temp, Address(input_len, -pos_const)); in CheckPosition()
1009 __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in CheckPosition()
1011 __ cmpl(temp, length.AsRegister<CpuRegister>()); in CheckPosition()
1013 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1018 __ testl(pos_reg, pos_reg); in CheckPosition()
1019 __ j(kNotEqual, slow_path->GetEntryLabel()); in CheckPosition()
1023 __ testl(pos_reg, pos_reg); in CheckPosition()
1024 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1027 __ cmpl(Address(input, length_offset), pos_reg); in CheckPosition()
1028 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1031 __ movl(temp, Address(input, length_offset)); in CheckPosition()
1032 __ subl(temp, pos_reg); in CheckPosition()
1034 __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in CheckPosition()
1036 __ cmpl(temp, length.AsRegister<CpuRegister>()); in CheckPosition()
1038 __ j(kLess, slow_path->GetEntryLabel()); in CheckPosition()
1064 __ cmpl(src, dest); in VisitSystemArrayCopyChar()
1065 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1068 __ testl(src, src); in VisitSystemArrayCopyChar()
1069 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1072 __ testl(dest, dest); in VisitSystemArrayCopyChar()
1073 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1078 __ testl(length.AsRegister<CpuRegister>(), length.AsRegister<CpuRegister>()); in VisitSystemArrayCopyChar()
1079 __ j(kLess, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
1090 __ movl(count, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); in VisitSystemArrayCopyChar()
1092 __ movl(count, length.AsRegister<CpuRegister>()); in VisitSystemArrayCopyChar()
1104 __ leal(src_base, Address(src, char_size * src_pos_const + data_offset)); in VisitSystemArrayCopyChar()
1106 __ leal(src_base, Address(src, src_pos.AsRegister<CpuRegister>(), in VisitSystemArrayCopyChar()
1111 __ leal(dest_base, Address(dest, char_size * dest_pos_const + data_offset)); in VisitSystemArrayCopyChar()
1113 __ leal(dest_base, Address(dest, dest_pos.AsRegister<CpuRegister>(), in VisitSystemArrayCopyChar()
1118 __ rep_movsw(); in VisitSystemArrayCopyChar()
1120 __ Bind(slow_path->GetExitLabel()); in VisitSystemArrayCopyChar()
1158 __ cmpl(src, dest); in VisitSystemArrayCopy()
1170 __ j(kNotEqual, &conditions_on_positions_validated); in VisitSystemArrayCopy()
1172 __ cmpl(dest_pos.AsRegister<CpuRegister>(), Immediate(src_pos_constant)); in VisitSystemArrayCopy()
1173 __ j(kGreater, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1177 __ j(kNotEqual, &conditions_on_positions_validated); in VisitSystemArrayCopy()
1181 __ cmpl(src_pos.AsRegister<CpuRegister>(), Immediate(dest_pos_constant)); in VisitSystemArrayCopy()
1182 __ j(kLess, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1184 __ cmpl(src_pos.AsRegister<CpuRegister>(), dest_pos.AsRegister<CpuRegister>()); in VisitSystemArrayCopy()
1185 __ j(kLess, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1189 __ Bind(&conditions_on_positions_validated); in VisitSystemArrayCopy()
1193 __ testl(src, src); in VisitSystemArrayCopy()
1194 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1199 __ testl(dest, dest); in VisitSystemArrayCopy()
1200 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1208 __ testl(length.AsRegister<CpuRegister>(), length.AsRegister<CpuRegister>()); in VisitSystemArrayCopy()
1209 __ j(kLess, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1237 __ movl(temp1, Address(dest, class_offset)); in VisitSystemArrayCopy()
1238 __ movl(temp2, Address(src, class_offset)); in VisitSystemArrayCopy()
1244 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1245 __ MaybeUnpoisonHeapReference(temp2); in VisitSystemArrayCopy()
1252 __ movl(CpuRegister(TMP), Address(temp1, component_offset)); in VisitSystemArrayCopy()
1253 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1254 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1255 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in VisitSystemArrayCopy()
1256 __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
1257 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1263 __ movl(CpuRegister(TMP), Address(temp2, component_offset)); in VisitSystemArrayCopy()
1264 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1265 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1266 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in VisitSystemArrayCopy()
1267 __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
1268 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1271 __ cmpl(temp1, temp2); in VisitSystemArrayCopy()
1275 __ j(kEqual, &do_copy); in VisitSystemArrayCopy()
1277 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1280 __ movl(temp1, Address(temp1, component_offset)); in VisitSystemArrayCopy()
1281 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1283 __ movl(temp1, Address(temp1, super_offset)); in VisitSystemArrayCopy()
1285 __ testl(temp1, temp1); in VisitSystemArrayCopy()
1286 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1287 __ Bind(&do_copy); in VisitSystemArrayCopy()
1289 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1295 __ movl(temp1, Address(src, class_offset)); in VisitSystemArrayCopy()
1296 __ MaybeUnpoisonHeapReference(temp1); in VisitSystemArrayCopy()
1298 __ movl(CpuRegister(TMP), Address(temp1, component_offset)); in VisitSystemArrayCopy()
1299 __ testl(CpuRegister(TMP), CpuRegister(TMP)); in VisitSystemArrayCopy()
1300 __ j(kEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1301 __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); in VisitSystemArrayCopy()
1302 __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); in VisitSystemArrayCopy()
1303 __ j(kNotEqual, slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1312 __ leal(temp1, Address(src, element_size * constant + offset)); in VisitSystemArrayCopy()
1314 __ leal(temp1, Address(src, src_pos.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, offset)); in VisitSystemArrayCopy()
1319 __ leal(temp2, Address(dest, element_size * constant + offset)); in VisitSystemArrayCopy()
1321 __ leal(temp2, Address(dest, dest_pos.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, offset)); in VisitSystemArrayCopy()
1326 __ leal(temp3, Address(temp1, element_size * constant)); in VisitSystemArrayCopy()
1328 __ leal(temp3, Address(temp1, length.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, 0)); in VisitSystemArrayCopy()
1335 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
1336 __ j(kEqual, &done); in VisitSystemArrayCopy()
1337 __ Bind(&loop); in VisitSystemArrayCopy()
1338 __ movl(CpuRegister(TMP), Address(temp1, 0)); in VisitSystemArrayCopy()
1339 __ movl(Address(temp2, 0), CpuRegister(TMP)); in VisitSystemArrayCopy()
1340 __ addl(temp1, Immediate(element_size)); in VisitSystemArrayCopy()
1341 __ addl(temp2, Immediate(element_size)); in VisitSystemArrayCopy()
1342 __ cmpl(temp1, temp3); in VisitSystemArrayCopy()
1343 __ j(kNotEqual, &loop); in VisitSystemArrayCopy()
1344 __ Bind(&done); in VisitSystemArrayCopy()
1353 __ Bind(slow_path->GetExitLabel()); in VisitSystemArrayCopy()
1374 __ testl(argument, argument); in VisitStringCompareTo()
1377 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringCompareTo()
1379 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pStringCompareTo), in VisitStringCompareTo()
1381 __ Bind(slow_path->GetExitLabel()); in VisitStringCompareTo()
1420 __ testl(arg, arg); in VisitStringEquals()
1421 __ j(kEqual, &return_false); in VisitStringEquals()
1427 __ movl(rcx, Address(str, class_offset)); in VisitStringEquals()
1428 __ cmpl(rcx, Address(arg, class_offset)); in VisitStringEquals()
1429 __ j(kNotEqual, &return_false); in VisitStringEquals()
1432 __ cmpl(str, arg); in VisitStringEquals()
1433 __ j(kEqual, &return_true); in VisitStringEquals()
1436 __ movl(rcx, Address(str, count_offset)); in VisitStringEquals()
1438 __ cmpl(rcx, Address(arg, count_offset)); in VisitStringEquals()
1439 __ j(kNotEqual, &return_false); in VisitStringEquals()
1441 __ jrcxz(&return_true); in VisitStringEquals()
1444 __ leal(rsi, Address(str, value_offset)); in VisitStringEquals()
1445 __ leal(rdi, Address(arg, value_offset)); in VisitStringEquals()
1448 __ addl(rcx, Immediate(3)); in VisitStringEquals()
1449 __ shrl(rcx, Immediate(2)); in VisitStringEquals()
1456 __ repe_cmpsq(); in VisitStringEquals()
1458 __ j(kNotEqual, &return_false); in VisitStringEquals()
1462 __ Bind(&return_true); in VisitStringEquals()
1463 __ movl(rsi, Immediate(1)); in VisitStringEquals()
1464 __ jmp(&end); in VisitStringEquals()
1467 __ Bind(&return_false); in VisitStringEquals()
1468 __ xorl(rsi, rsi); in VisitStringEquals()
1469 __ Bind(&end); in VisitStringEquals()
1529 __ jmp(slow_path->GetEntryLabel()); in GenerateStringIndexOf()
1530 __ Bind(slow_path->GetExitLabel()); in GenerateStringIndexOf()
1534 __ cmpl(search_value, Immediate(std::numeric_limits<uint16_t>::max())); in GenerateStringIndexOf()
1537 __ j(kAbove, slow_path->GetEntryLabel()); in GenerateStringIndexOf()
1547 __ movl(string_length, Address(string_obj, count_offset)); in GenerateStringIndexOf()
1552 __ testl(string_length, string_length); in GenerateStringIndexOf()
1553 __ j(kEqual, ¬_found_label); in GenerateStringIndexOf()
1557 __ movl(counter, string_length); in GenerateStringIndexOf()
1560 __ addq(string_obj, Immediate(value_offset)); in GenerateStringIndexOf()
1565 __ cmpl(start_index, string_length); in GenerateStringIndexOf()
1566 __ j(kGreaterEqual, ¬_found_label); in GenerateStringIndexOf()
1569 __ xorl(counter, counter); in GenerateStringIndexOf()
1570 __ cmpl(start_index, Immediate(0)); in GenerateStringIndexOf()
1571 __ cmov(kGreater, counter, start_index, /* is64bit */ false); // 32-bit copy is enough. in GenerateStringIndexOf()
1574 __ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset)); in GenerateStringIndexOf()
1577 __ negq(counter); // Needs to be 64-bit negation, as the address computation is 64-bit. in GenerateStringIndexOf()
1578 __ leaq(counter, Address(string_length, counter, ScaleFactor::TIMES_1, 0)); in GenerateStringIndexOf()
1584 __ repne_scasw(); in GenerateStringIndexOf()
1587 __ j(kNotEqual, ¬_found_label); in GenerateStringIndexOf()
1590 __ subl(string_length, counter); in GenerateStringIndexOf()
1591 __ leal(out, Address(string_length, -1)); in GenerateStringIndexOf()
1594 __ jmp(&done); in GenerateStringIndexOf()
1597 __ Bind(¬_found_label); in GenerateStringIndexOf()
1598 __ movl(out, Immediate(-1)); in GenerateStringIndexOf()
1601 __ Bind(&done); in GenerateStringIndexOf()
1603 __ Bind(slow_path->GetExitLabel()); in GenerateStringIndexOf()
1641 __ testl(byte_array, byte_array); in VisitStringNewStringFromBytes()
1644 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringNewStringFromBytes()
1646 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocStringFromBytes), in VisitStringNewStringFromBytes()
1650 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromBytes()
1673 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocStringFromChars), in VisitStringNewStringFromChars()
1693 __ testl(string_to_copy, string_to_copy); in VisitStringNewStringFromString()
1696 __ j(kEqual, slow_path->GetEntryLabel()); in VisitStringNewStringFromString()
1698 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocStringFromString), in VisitStringNewStringFromString()
1702 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromString()
1746 __ leaq(CpuRegister(RDI), Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); in VisitStringGetCharsNoCheck()
1752 __ leaq(CpuRegister(RSI), Address(obj, srcBegin_value * char_size + value_offset)); in VisitStringGetCharsNoCheck()
1754 __ leaq(CpuRegister(RSI), Address(obj, srcBegin.AsRegister<CpuRegister>(), in VisitStringGetCharsNoCheck()
1759 __ movl(CpuRegister(RCX), srcEnd); in VisitStringGetCharsNoCheck()
1762 __ subl(CpuRegister(RCX), Immediate(srcBegin_value)); in VisitStringGetCharsNoCheck()
1766 __ subl(CpuRegister(RCX), srcBegin.AsRegister<CpuRegister>()); in VisitStringGetCharsNoCheck()
1770 __ rep_movsw(); in VisitStringGetCharsNoCheck()
1780 __ movsxb(out, Address(address, 0)); in GenPeek()
1783 __ movsxw(out, Address(address, 0)); in GenPeek()
1786 __ movl(out, Address(address, 0)); in GenPeek()
1789 __ movq(out, Address(address, 0)); in GenPeek()
1845 __ movb(Address(address, 0), in GenPoke()
1848 __ movb(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
1853 __ movw(Address(address, 0), in GenPoke()
1856 __ movw(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
1861 __ movl(Address(address, 0), in GenPoke()
1864 __ movl(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
1872 __ movq(Address(address, 0), Immediate(v_32)); in GenPoke()
1874 __ movq(Address(address, 0), value.AsRegister<CpuRegister>()); in GenPoke()
1943 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
1953 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
1958 __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
1959 __ MaybeUnpoisonHeapReference(output); in GenUnsafeGet()
1965 __ movq(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); in GenUnsafeGet()
2091 __ movq(Address(base, offset, ScaleFactor::TIMES_1, 0), value); in GenUnsafePut()
2094 __ movl(temp, value); in GenUnsafePut()
2095 __ PoisonHeapReference(temp); in GenUnsafePut()
2096 __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), temp); in GenUnsafePut()
2098 __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), value); in GenUnsafePut()
2215 __ movl(CpuRegister(value_reg), base); in GenCAS()
2229 __ PoisonHeapReference(expected); in GenCAS()
2230 __ PoisonHeapReference(CpuRegister(value_reg)); in GenCAS()
2240 __ LockCmpxchgl(Address(base, offset, TIMES_1, 0), CpuRegister(value_reg)); in GenCAS()
2246 __ setcc(kZero, out); in GenCAS()
2247 __ movzxb(out, out); in GenCAS()
2259 __ UnpoisonHeapReference(CpuRegister(value_reg)); in GenCAS()
2264 __ UnpoisonHeapReference(expected); in GenCAS()
2268 __ LockCmpxchgl(Address(base, offset, TIMES_1, 0), value); in GenCAS()
2270 __ LockCmpxchgq(Address(base, offset, TIMES_1, 0), value); in GenCAS()
2279 __ setcc(kZero, out); in GenCAS()
2280 __ movzxb(out, out); in GenCAS()
2309 __ movl(temp, reg); in SwapBits()
2310 __ shrl(reg, imm_shift); in SwapBits()
2311 __ andl(temp, imm_mask); in SwapBits()
2312 __ andl(reg, imm_mask); in SwapBits()
2313 __ shll(temp, imm_shift); in SwapBits()
2314 __ orl(reg, temp); in SwapBits()
2333 __ bswapl(reg); in VisitIntegerReverse()
2352 __ movq(temp_mask, Immediate(mask)); in SwapBits64()
2353 __ movq(temp, reg); in SwapBits64()
2354 __ shrq(reg, imm_shift); in SwapBits64()
2355 __ andq(temp, temp_mask); in SwapBits64()
2356 __ andq(reg, temp_mask); in SwapBits64()
2357 __ shlq(temp, imm_shift); in SwapBits64()
2358 __ orq(reg, temp); in SwapBits64()
2378 __ bswapq(reg); in VisitLongReverse()
2418 __ popcntq(out, src.AsRegister<CpuRegister>()); in GenBitCount()
2420 __ popcntl(out, src.AsRegister<CpuRegister>()); in GenBitCount()
2424 __ popcntq(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenBitCount()
2427 __ popcntl(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenBitCount()
2469 __ xorl(out, out); // Clears upper bits too. in GenOneBit()
2494 __ bsrq(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2496 __ bsrl(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2500 __ bsrq(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2503 __ bsrl(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2507 __ j(kEqual, &is_zero); in GenOneBit()
2508 __ movl(out, Immediate(1)); // Clears upper bits too. in GenOneBit()
2510 __ shlq(out, tmp); in GenOneBit()
2512 __ shll(out, tmp); in GenOneBit()
2514 __ jmp(&done); in GenOneBit()
2515 __ Bind(&is_zero); in GenOneBit()
2516 __ xorl(out, out); // Clears upper bits too. in GenOneBit()
2517 __ Bind(&done); in GenOneBit()
2522 __ movq(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2524 __ movl(tmp, src.AsRegister<CpuRegister>()); in GenOneBit()
2528 __ movq(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2531 __ movl(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); in GenOneBit()
2535 __ movq(out, tmp); in GenOneBit()
2536 __ negq(tmp); in GenOneBit()
2537 __ andq(out, tmp); in GenOneBit()
2539 __ movl(out, tmp); in GenOneBit()
2540 __ negl(tmp); in GenOneBit()
2541 __ andl(out, tmp); in GenOneBit()
2609 __ bsrq(out, src.AsRegister<CpuRegister>()); in GenLeadingZeros()
2611 __ bsrl(out, src.AsRegister<CpuRegister>()); in GenLeadingZeros()
2615 __ bsrq(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenLeadingZeros()
2618 __ bsrl(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenLeadingZeros()
2623 __ j(kEqual, &is_zero); in GenLeadingZeros()
2626 __ xorl(out, Immediate(zero_value_result - 1)); in GenLeadingZeros()
2627 __ jmp(&done); in GenLeadingZeros()
2630 __ Bind(&is_zero); in GenLeadingZeros()
2631 __ movl(out, Immediate(zero_value_result)); in GenLeadingZeros()
2633 __ Bind(&done); in GenLeadingZeros()
2683 __ bsfq(out, src.AsRegister<CpuRegister>()); in GenTrailingZeros()
2685 __ bsfl(out, src.AsRegister<CpuRegister>()); in GenTrailingZeros()
2689 __ bsfq(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenTrailingZeros()
2692 __ bsfl(out, Address(CpuRegister(RSP), src.GetStackIndex())); in GenTrailingZeros()
2697 __ j(kNotEqual, &done); in GenTrailingZeros()
2700 __ movl(out, Immediate(zero_value_result)); in GenTrailingZeros()
2702 __ Bind(&done); in GenTrailingZeros()
2734 #undef __