Lines Matching refs:__
40 #define __ assembler->GetVIXLAssembler()-> macro
90 __ Add(base, array, element_size * constant + data_offset); in GenSystemArrayCopyBaseAddress()
92 __ Add(base, array, Operand(RegisterFrom(pos), vixl32::LSL, element_size_shift)); in GenSystemArrayCopyBaseAddress()
93 __ Add(base, base, data_offset); in GenSystemArrayCopyBaseAddress()
112 __ Add(end, base, element_size * constant); in GenSystemArrayCopyEndAddress()
114 __ Add(end, base, Operand(RegisterFrom(copy_length), vixl32::LSL, element_size_shift)); in GenSystemArrayCopyEndAddress()
148 __ Bind(GetEntryLabel()); in EmitNativeCode()
153 __ Bind(&loop); in EmitNativeCode()
154 __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex)); in EmitNativeCode()
181 __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex)); in EmitNativeCode()
182 __ Cmp(src_curr_addr, src_stop_addr); in EmitNativeCode()
183 __ B(ne, &loop, /* is_far_target= */ false); in EmitNativeCode()
184 __ B(GetExitLabel()); in EmitNativeCode()
228 __ Vmov(LowRegisterFrom(output), HighRegisterFrom(output), DRegisterFrom(input)); in MoveFPToInt()
230 __ Vmov(RegisterFrom(output), SRegisterFrom(input)); in MoveFPToInt()
238 __ Vmov(DRegisterFrom(output), LowRegisterFrom(input), HighRegisterFrom(input)); in MoveIntToFP()
240 __ Vmov(SRegisterFrom(output), RegisterFrom(input)); in MoveIntToFP()
317 __ Clz(out, in_reg_hi); in GenNumberOfLeadingZeros()
318 __ CompareAndBranchIfNonZero(in_reg_hi, final_label, /* is_far_target= */ false); in GenNumberOfLeadingZeros()
319 __ Clz(out, in_reg_lo); in GenNumberOfLeadingZeros()
320 __ Add(out, out, 32); in GenNumberOfLeadingZeros()
322 __ Bind(&end); in GenNumberOfLeadingZeros()
325 __ Clz(out, RegisterFrom(in)); in GenNumberOfLeadingZeros()
359 __ Rbit(out, in_reg_lo); in GenNumberOfTrailingZeros()
360 __ Clz(out, out); in GenNumberOfTrailingZeros()
361 __ CompareAndBranchIfNonZero(in_reg_lo, final_label, /* is_far_target= */ false); in GenNumberOfTrailingZeros()
362 __ Rbit(out, in_reg_hi); in GenNumberOfTrailingZeros()
363 __ Clz(out, out); in GenNumberOfTrailingZeros()
364 __ Add(out, out, 32); in GenNumberOfTrailingZeros()
366 __ Bind(&end); in GenNumberOfTrailingZeros()
370 __ Rbit(out, in); in GenNumberOfTrailingZeros()
371 __ Clz(out, out); in GenNumberOfTrailingZeros()
397 __ Vsqrt(OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); in VisitMathSqrt()
409 __ Vrintn(F64, OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); in VisitMathRint()
434 __ Vcvta(S32, F32, temp1, in_reg); in VisitMathRoundFloat()
435 __ Vmov(out_reg, temp1); in VisitMathRoundFloat()
438 __ Cmp(out_reg, 0); in VisitMathRoundFloat()
439 __ B(ge, final_label, /* is_far_target= */ false); in VisitMathRoundFloat()
444 __ Vrinta(F32, temp1, in_reg); in VisitMathRoundFloat()
445 __ Vmov(temp2, 0.5); in VisitMathRoundFloat()
446 __ Vsub(F32, temp1, in_reg, temp1); in VisitMathRoundFloat()
447 __ Vcmp(F32, temp1, temp2); in VisitMathRoundFloat()
448 __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR); in VisitMathRoundFloat()
454 __ it(eq); in VisitMathRoundFloat()
455 __ add(eq, out_reg, out_reg, 1); in VisitMathRoundFloat()
459 __ Bind(&done); in VisitMathRoundFloat()
470 __ Ldrsb(OutputRegister(invoke), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPeekByte()
480 __ Ldr(OutputRegister(invoke), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPeekIntNative()
496 __ Ldr(hi, MemOperand(addr, 4)); in VisitMemoryPeekLongNative()
497 __ Ldr(lo, MemOperand(addr)); in VisitMemoryPeekLongNative()
499 __ Ldr(lo, MemOperand(addr)); in VisitMemoryPeekLongNative()
500 __ Ldr(hi, MemOperand(addr, 4)); in VisitMemoryPeekLongNative()
511 __ Ldrsh(OutputRegister(invoke), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPeekShortNative()
527 __ Strb(InputRegisterAt(invoke, 1), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPokeByte()
536 __ Str(InputRegisterAt(invoke, 1), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPokeIntNative()
549 __ Str(LowRegisterFrom(invoke->GetLocations()->InAt(1)), MemOperand(addr)); in VisitMemoryPokeLongNative()
550 __ Str(HighRegisterFrom(invoke->GetLocations()->InAt(1)), MemOperand(addr, 4)); in VisitMemoryPokeLongNative()
559 __ Strh(InputRegisterAt(invoke, 1), MemOperand(LowRegisterFrom(invoke->GetLocations()->InAt(0)))); in VisitMemoryPokeShortNative()
570 __ Ldr(OutputRegister(invoke), in VisitThreadCurrentThread()
636 __ CompareAndBranchIfZero(arg, slow_path->GetEntryLabel()); in VisitStringCompareTo()
640 __ Subs(out, str, arg); in VisitStringCompareTo()
641 __ B(eq, &end); in VisitStringCompareTo()
645 __ Ldr(temp3, MemOperand(str, count_offset)); in VisitStringCompareTo()
646 __ Ldr(temp2, MemOperand(arg, count_offset)); in VisitStringCompareTo()
648 __ Lsr(temp0, temp3, 1u); in VisitStringCompareTo()
649 __ Lsr(temp1, temp2, 1u); in VisitStringCompareTo()
652 __ Ldr(temp0, MemOperand(str, count_offset)); in VisitStringCompareTo()
653 __ Ldr(temp1, MemOperand(arg, count_offset)); in VisitStringCompareTo()
656 __ Subs(out, temp0, temp1); in VisitStringCompareTo()
664 __ it(gt); in VisitStringCompareTo()
665 __ mov(gt, temp0, temp1); in VisitStringCompareTo()
671 __ CompareAndBranchIfZero(temp0, &end, mirror::kUseStringCompression); in VisitStringCompareTo()
675 __ Eors(temp2, temp2, temp3); in VisitStringCompareTo()
676 __ Lsrs(temp2, temp2, 1u); in VisitStringCompareTo()
677 __ B(cs, &different_compression); in VisitStringCompareTo()
680 __ Lsls(temp3, temp3, 31u); // Extract purely the compression flag. in VisitStringCompareTo()
686 __ it(ne); in VisitStringCompareTo()
687 __ add(ne, temp0, temp0, temp0); in VisitStringCompareTo()
693 __ Bind(&end); in VisitStringCompareTo()
696 __ Bind(slow_path->GetExitLabel()); in VisitStringCompareTo()
723 __ Mov(temp1, value_offset); in GenerateStringCompareToLoop()
737 __ Bind(&loop); in GenerateStringCompareToLoop()
739 __ Ldr(temp_reg, MemOperand(str, temp1)); in GenerateStringCompareToLoop()
740 __ Ldr(temp2, MemOperand(arg, temp1)); in GenerateStringCompareToLoop()
741 __ Cmp(temp_reg, temp2); in GenerateStringCompareToLoop()
742 __ B(ne, &find_char_diff, /* is_far_target= */ false); in GenerateStringCompareToLoop()
743 __ Add(temp1, temp1, char_size * 2); in GenerateStringCompareToLoop()
745 __ Ldr(temp_reg, MemOperand(str, temp1)); in GenerateStringCompareToLoop()
746 __ Ldr(temp2, MemOperand(arg, temp1)); in GenerateStringCompareToLoop()
747 __ Cmp(temp_reg, temp2); in GenerateStringCompareToLoop()
748 __ B(ne, &find_char_diff_2nd_cmp, /* is_far_target= */ false); in GenerateStringCompareToLoop()
749 __ Add(temp1, temp1, char_size * 2); in GenerateStringCompareToLoop()
751 __ Subs(temp0, temp0, (mirror::kUseStringCompression ? 8 : 4)); in GenerateStringCompareToLoop()
752 __ B(hi, &loop, /* is_far_target= */ false); in GenerateStringCompareToLoop()
753 __ B(end); in GenerateStringCompareToLoop()
755 __ Bind(&find_char_diff_2nd_cmp); in GenerateStringCompareToLoop()
757 __ Subs(temp0, temp0, 4); // 4 bytes previously compared. in GenerateStringCompareToLoop()
758 __ B(ls, end, /* is_far_target= */ false); // Was the second comparison fully beyond the end? in GenerateStringCompareToLoop()
762 __ Sub(temp0, temp0, 2); in GenerateStringCompareToLoop()
766 __ Bind(&find_char_diff); in GenerateStringCompareToLoop()
768 __ Eor(temp1, temp2, temp_reg); in GenerateStringCompareToLoop()
769 __ Rbit(temp1, temp1); in GenerateStringCompareToLoop()
770 __ Clz(temp1, temp1); in GenerateStringCompareToLoop()
785 __ Cmp(temp0, Operand(temp1, vixl32::LSR, (mirror::kUseStringCompression ? 3 : 4))); in GenerateStringCompareToLoop()
786 __ B((mirror::kUseStringCompression ? ls : le), end, /* is_far_target= */ false); in GenerateStringCompareToLoop()
793 __ Orr(temp3, temp3, 0xffu << 23); // uncompressed ? 0xff800000u : 0x7ff80000u in GenerateStringCompareToLoop()
794 __ Bic(temp1, temp1, Operand(temp3, vixl32::LSR, 31 - 3)); // &= ~(uncompressed ? 0xfu : 0x7u) in GenerateStringCompareToLoop()
795 __ Asr(temp3, temp3, 7u); // uncompressed ? 0xffff0000u : 0xff0000u. in GenerateStringCompareToLoop()
796 __ Lsr(temp2, temp2, temp1); // Extract second character. in GenerateStringCompareToLoop()
797 __ Lsr(temp3, temp3, 16u); // uncompressed ? 0xffffu : 0xffu in GenerateStringCompareToLoop()
798 __ Lsr(out, temp_reg, temp1); // Extract first character. in GenerateStringCompareToLoop()
799 __ And(temp2, temp2, temp3); in GenerateStringCompareToLoop()
800 __ And(out, out, temp3); in GenerateStringCompareToLoop()
802 __ Bic(temp1, temp1, 0xf); in GenerateStringCompareToLoop()
803 __ Lsr(temp2, temp2, temp1); in GenerateStringCompareToLoop()
804 __ Lsr(out, temp_reg, temp1); in GenerateStringCompareToLoop()
805 __ Movt(temp2, 0); in GenerateStringCompareToLoop()
806 __ Movt(out, 0); in GenerateStringCompareToLoop()
809 __ Sub(out, out, temp2); in GenerateStringCompareToLoop()
813 __ B(end); in GenerateStringCompareToLoop()
814 __ Bind(different_compression); in GenerateStringCompareToLoop()
825 __ Add(temp0, temp0, temp0); // Unlike LSL, this ADD is always 16-bit. in GenerateStringCompareToLoop()
827 __ Mov(temp1, str); in GenerateStringCompareToLoop()
828 __ Mov(temp2, arg); in GenerateStringCompareToLoop()
829 __ Lsrs(temp3, temp3, 1u); // Continue the move of the compression flag. in GenerateStringCompareToLoop()
834 __ itt(cs); // Interleave with selection of temp1 and temp2. in GenerateStringCompareToLoop()
835 __ mov(cs, temp1, arg); // Preserves flags. in GenerateStringCompareToLoop()
836 __ mov(cs, temp2, str); // Preserves flags. in GenerateStringCompareToLoop()
838 __ Sbc(temp0, temp0, 0); // Complete the move of the compression flag. in GenerateStringCompareToLoop()
841 __ Add(temp1, temp1, value_offset); in GenerateStringCompareToLoop()
842 __ Add(temp2, temp2, value_offset); in GenerateStringCompareToLoop()
849 __ Bind(&different_compression_loop); in GenerateStringCompareToLoop()
850 __ Ldrb(temp_reg, MemOperand(temp1, c_char_size, PostIndex)); in GenerateStringCompareToLoop()
851 __ Ldrh(temp3, MemOperand(temp2, char_size, PostIndex)); in GenerateStringCompareToLoop()
852 __ Cmp(temp_reg, temp3); in GenerateStringCompareToLoop()
853 __ B(ne, &different_compression_diff, /* is_far_target= */ false); in GenerateStringCompareToLoop()
854 __ Subs(temp0, temp0, 2); in GenerateStringCompareToLoop()
855 __ B(hi, &different_compression_loop, /* is_far_target= */ false); in GenerateStringCompareToLoop()
856 __ B(end); in GenerateStringCompareToLoop()
859 __ Bind(&different_compression_diff); in GenerateStringCompareToLoop()
860 __ Sub(out, temp_reg, temp3); in GenerateStringCompareToLoop()
864 __ Lsrs(temp0, temp0, 1u); in GenerateStringCompareToLoop()
871 __ it(cc); in GenerateStringCompareToLoop()
872 __ rsb(cc, out, out, 0); in GenerateStringCompareToLoop()
951 __ CompareAndBranchIfZero(arg, &return_false, /* is_far_target= */ false); in VisitStringEquals()
955 __ Cmp(str, arg); in VisitStringEquals()
956 __ B(eq, &return_true, /* is_far_target= */ false); in VisitStringEquals()
968 __ Ldr(temp, MemOperand(str, class_offset)); in VisitStringEquals()
970 __ Ldr(out, MemOperand(arg, class_offset)); in VisitStringEquals()
973 __ Cmp(temp, out); in VisitStringEquals()
974 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
995 __ Ldr(temp, MemOperand(arg, count_offset)); in VisitStringEquals()
996 __ Cmp(temp, Operand(mirror::String::GetFlaggedCount(const_string_length, is_compressed))); in VisitStringEquals()
997 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1000 __ Ldr(temp, MemOperand(str, count_offset)); in VisitStringEquals()
1001 __ Ldr(out, MemOperand(arg, count_offset)); in VisitStringEquals()
1004 __ Cmp(temp, out); in VisitStringEquals()
1005 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1025 __ Ldrd(temp, temp1, MemOperand(str, offset)); in VisitStringEquals()
1026 __ Ldrd(temp2, out, MemOperand(arg, offset)); in VisitStringEquals()
1027 __ Cmp(temp, temp2); in VisitStringEquals()
1028 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1029 __ Cmp(temp1, out); in VisitStringEquals()
1030 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1035 __ Ldr(temp, MemOperand(str, offset)); in VisitStringEquals()
1036 __ Ldr(out, MemOperand(arg, offset)); in VisitStringEquals()
1037 __ Cmp(temp, out); in VisitStringEquals()
1038 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1044 __ CompareAndBranchIfZero(temp, &return_true, /* is_far_target= */ false); in VisitStringEquals()
1049 __ Lsrs(temp, temp, 1u); // Extract length and check compression flag. in VisitStringEquals()
1053 __ it(cs); // If uncompressed, in VisitStringEquals()
1054 __ add(cs, temp, temp, temp); // double the byte count. in VisitStringEquals()
1062 __ Mov(temp1, value_offset); in VisitStringEquals()
1065 __ Bind(&loop); in VisitStringEquals()
1066 __ Ldr(out, MemOperand(str, temp1)); in VisitStringEquals()
1067 __ Ldr(temp2, MemOperand(arg, temp1)); in VisitStringEquals()
1068 __ Add(temp1, temp1, Operand::From(sizeof(uint32_t))); in VisitStringEquals()
1069 __ Cmp(out, temp2); in VisitStringEquals()
1070 __ B(ne, &return_false, /* is_far_target= */ false); in VisitStringEquals()
1072 __ Subs(temp, temp, mirror::kUseStringCompression ? 4 : 2); in VisitStringEquals()
1073 __ B(hi, &loop, /* is_far_target= */ false); in VisitStringEquals()
1078 __ Bind(&return_true); in VisitStringEquals()
1079 __ Mov(out, 1); in VisitStringEquals()
1080 __ B(final_label); in VisitStringEquals()
1083 __ Bind(&return_false); in VisitStringEquals()
1084 __ Mov(out, 0); in VisitStringEquals()
1087 __ Bind(&end); in VisitStringEquals()
1111 __ B(slow_path->GetEntryLabel()); in GenerateVisitStringIndexOf()
1112 __ Bind(slow_path->GetExitLabel()); in GenerateVisitStringIndexOf()
1118 __ Cmp(char_reg, static_cast<uint32_t>(std::numeric_limits<uint16_t>::max()) + 1); in GenerateVisitStringIndexOf()
1121 __ B(hs, slow_path->GetEntryLabel()); in GenerateVisitStringIndexOf()
1128 __ Mov(tmp_reg, 0); in GenerateVisitStringIndexOf()
1135 __ Bind(slow_path->GetExitLabel()); in GenerateVisitStringIndexOf()
1187 __ Cmp(byte_array, 0); in VisitStringNewStringFromBytes()
1191 __ B(eq, slow_path->GetEntryLabel()); in VisitStringNewStringFromBytes()
1195 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromBytes()
1230 __ Cmp(string_to_copy, 0); in VisitStringNewStringFromString()
1234 __ B(eq, slow_path->GetEntryLabel()); in VisitStringNewStringFromString()
1239 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromString()
1292 __ Ldr(temp, MemOperand(input, length_offset)); in CheckPosition()
1294 __ Cmp(temp, Int32ConstantFrom(length)); in CheckPosition()
1296 __ Cmp(temp, RegisterFrom(length)); in CheckPosition()
1298 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1302 __ Ldr(temp, MemOperand(input, length_offset)); in CheckPosition()
1303 __ Subs(temp, temp, pos_const); in CheckPosition()
1304 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1308 __ Cmp(temp, Int32ConstantFrom(length)); in CheckPosition()
1310 __ Cmp(temp, RegisterFrom(length)); in CheckPosition()
1312 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1317 __ CompareAndBranchIfNonZero(pos_reg, slow_path->GetEntryLabel()); in CheckPosition()
1321 __ Cmp(pos_reg, 0); in CheckPosition()
1322 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1325 __ Ldr(temp, MemOperand(input, length_offset)); in CheckPosition()
1326 __ Subs(temp, temp, pos_reg); in CheckPosition()
1327 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1331 __ Cmp(temp, Int32ConstantFrom(length)); in CheckPosition()
1333 __ Cmp(temp, RegisterFrom(length)); in CheckPosition()
1335 __ B(lt, slow_path->GetEntryLabel()); in CheckPosition()
1382 __ Cmp(src, dest); in VisitSystemArrayCopy()
1383 __ B(eq, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1391 __ Cmp(src, dest); in VisitSystemArrayCopy()
1392 __ B(ne, &conditions_on_positions_validated, /* is_far_target= */ false); in VisitSystemArrayCopy()
1394 __ Cmp(RegisterFrom(dest_pos), src_pos_constant); in VisitSystemArrayCopy()
1395 __ B(gt, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1399 __ Cmp(src, dest); in VisitSystemArrayCopy()
1400 __ B(ne, &conditions_on_positions_validated, /* is_far_target= */ false); in VisitSystemArrayCopy()
1404 __ Cmp(RegisterFrom(src_pos), dest_pos_constant); in VisitSystemArrayCopy()
1406 __ Cmp(RegisterFrom(src_pos), RegisterFrom(dest_pos)); in VisitSystemArrayCopy()
1408 __ B(lt, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1411 __ Bind(&conditions_on_positions_validated); in VisitSystemArrayCopy()
1415 __ CompareAndBranchIfZero(src, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1420 __ CompareAndBranchIfZero(dest, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1428 __ Cmp(RegisterFrom(length), 0); in VisitSystemArrayCopy()
1429 __ B(lt, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1465 __ CompareAndBranchIfZero(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1469 __ Ldrh(temp1, MemOperand(temp1, primitive_offset)); in VisitSystemArrayCopy()
1471 __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1489 __ CompareAndBranchIfZero(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1493 __ Ldrh(temp2, MemOperand(temp2, primitive_offset)); in VisitSystemArrayCopy()
1495 __ CompareAndBranchIfNonZero(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1504 __ Cmp(temp1, temp2); in VisitSystemArrayCopy()
1508 __ B(eq, &do_copy, /* is_far_target= */ false); in VisitSystemArrayCopy()
1517 __ Ldr(temp1, MemOperand(temp1, super_offset)); in VisitSystemArrayCopy()
1518 __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1519 __ Bind(&do_copy); in VisitSystemArrayCopy()
1521 __ B(ne, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1527 __ Ldr(temp1, MemOperand(dest, class_offset)); in VisitSystemArrayCopy()
1529 __ Ldr(temp2, MemOperand(src, class_offset)); in VisitSystemArrayCopy()
1543 __ Ldr(temp3, MemOperand(temp1, component_offset)); in VisitSystemArrayCopy()
1544 __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1547 __ Ldrh(temp3, MemOperand(temp3, primitive_offset)); in VisitSystemArrayCopy()
1549 __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1555 __ Ldr(temp3, MemOperand(temp2, component_offset)); in VisitSystemArrayCopy()
1556 __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1559 __ Ldrh(temp3, MemOperand(temp3, primitive_offset)); in VisitSystemArrayCopy()
1561 __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1564 __ Cmp(temp1, temp2); in VisitSystemArrayCopy()
1568 __ B(eq, &do_copy, /* is_far_target= */ false); in VisitSystemArrayCopy()
1573 __ Ldr(temp1, MemOperand(temp1, component_offset)); in VisitSystemArrayCopy()
1576 __ Ldr(temp1, MemOperand(temp1, super_offset)); in VisitSystemArrayCopy()
1578 __ CompareAndBranchIfNonZero(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1579 __ Bind(&do_copy); in VisitSystemArrayCopy()
1581 __ B(ne, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1594 __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1599 __ Ldr(temp1, MemOperand(src, class_offset)); in VisitSystemArrayCopy()
1602 __ Ldr(temp3, MemOperand(temp1, component_offset)); in VisitSystemArrayCopy()
1603 __ CompareAndBranchIfZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1607 __ Ldrh(temp3, MemOperand(temp3, primitive_offset)); in VisitSystemArrayCopy()
1609 __ CompareAndBranchIfNonZero(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1621 __ CompareAndBranchIfZero(RegisterFrom(length), &done, /* is_far_target= */ false); in VisitSystemArrayCopy()
1646 __ Ldr(temp2, MemOperand(src, monitor_offset)); in VisitSystemArrayCopy()
1656 __ Add(src, src, Operand(temp2, vixl32::LSR, 32)); in VisitSystemArrayCopy()
1680 __ Lsrs(temp2, temp2, LockWord::kReadBarrierStateShift + 1); in VisitSystemArrayCopy()
1682 __ B(cs, read_barrier_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
1690 __ Bind(&loop); in VisitSystemArrayCopy()
1694 __ Ldr(temp_reg, MemOperand(temp1, element_size, PostIndex)); in VisitSystemArrayCopy()
1695 __ Str(temp_reg, MemOperand(temp2, element_size, PostIndex)); in VisitSystemArrayCopy()
1697 __ Cmp(temp1, temp3); in VisitSystemArrayCopy()
1698 __ B(ne, &loop, /* is_far_target= */ false); in VisitSystemArrayCopy()
1700 __ Bind(read_barrier_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
1712 __ Bind(&loop); in VisitSystemArrayCopy()
1716 __ Ldr(temp_reg, MemOperand(temp1, element_size, PostIndex)); in VisitSystemArrayCopy()
1717 __ Str(temp_reg, MemOperand(temp2, element_size, PostIndex)); in VisitSystemArrayCopy()
1719 __ Cmp(temp1, temp3); in VisitSystemArrayCopy()
1720 __ B(ne, &loop, /* is_far_target= */ false); in VisitSystemArrayCopy()
1722 __ Bind(&done); in VisitSystemArrayCopy()
1728 __ Bind(intrinsic_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
1795 __ Vmov(RegisterFrom(locations->GetTemp(0)), in GenFPToFPCall()
1799 __ Vmov(OutputDRegister(invoke), in GenFPToFPCall()
1814 __ Vmov(RegisterFrom(locations->GetTemp(0)), in GenFPFPToFPCall()
1817 __ Vmov(RegisterFrom(locations->GetTemp(2)), in GenFPFPToFPCall()
1821 __ Vmov(OutputDRegister(invoke), in GenFPFPToFPCall()
1976 __ Rbit(OutputRegister(invoke), InputRegisterAt(invoke, 0)); in VisitIntegerReverse()
1992 __ Rbit(out_reg_lo, in_reg_hi); in VisitLongReverse()
1993 __ Rbit(out_reg_hi, in_reg_lo); in VisitLongReverse()
1998 __ Rev(LowRegisterFrom(pair), LowRegisterFrom(pair)); in GenerateReverseBytesInPlaceForEachWord()
1999 __ Rev(HighRegisterFrom(pair), HighRegisterFrom(pair)); in GenerateReverseBytesInPlaceForEachWord()
2008 __ Rev16(RegisterFrom(out), RegisterFrom(in)); in GenerateReverseBytes()
2011 __ Revsh(RegisterFrom(out), RegisterFrom(in)); in GenerateReverseBytes()
2014 __ Rev(RegisterFrom(out), RegisterFrom(in)); in GenerateReverseBytes()
2018 __ Rev(LowRegisterFrom(out), HighRegisterFrom(in)); in GenerateReverseBytes()
2019 __ Rev(HighRegisterFrom(out), LowRegisterFrom(in)); in GenerateReverseBytes()
2022 __ Rev(RegisterFrom(in), RegisterFrom(in)); // Note: Clobbers `in`. in GenerateReverseBytes()
2023 __ Vmov(SRegisterFrom(out), RegisterFrom(in)); in GenerateReverseBytes()
2027 __ Vmov(DRegisterFrom(out), HighRegisterFrom(in), LowRegisterFrom(in)); // Swap high/low. in GenerateReverseBytes()
2083 __ Vmov(tmp_d, src_1, src_0); // Temp DReg |--src_1|--src_0| in GenBitCount()
2084 __ Vcnt(Untyped8, tmp_d, tmp_d); // Temp DReg |c|c|c|c|c|c|c|c| in GenBitCount()
2085 __ Vpaddl(U8, tmp_d, tmp_d); // Temp DReg |--c|--c|--c|--c| in GenBitCount()
2086 __ Vpaddl(U16, tmp_d, tmp_d); // Temp DReg |------c|------c| in GenBitCount()
2088 __ Vpaddl(U32, tmp_d, tmp_d); // Temp DReg |--------------c| in GenBitCount()
2090 __ Vmov(out_r, tmp_s); in GenBitCount()
2129 __ Mov(temp, 0x80000000); // Modified immediate. in GenHighestOneBit()
2130 __ Clz(out_reg_lo, in_reg_lo); in GenHighestOneBit()
2131 __ Clz(out_reg_hi, in_reg_hi); in GenHighestOneBit()
2132 __ Lsr(out_reg_lo, temp, out_reg_lo); in GenHighestOneBit()
2133 __ Lsrs(out_reg_hi, temp, out_reg_hi); in GenHighestOneBit()
2142 __ Mov(LeaveFlags, temp, 0); in GenHighestOneBit()
2148 __ it(ne); in GenHighestOneBit()
2149 __ mov(ne, out_reg_lo, mov_src); in GenHighestOneBit()
2154 __ Mov(temp, 0x80000000); // Modified immediate. in GenHighestOneBit()
2155 __ Clz(out, in); in GenHighestOneBit()
2156 __ Lsr(out, temp, out); in GenHighestOneBit()
2195 __ Rsb(out_reg_hi, in_reg_hi, 0); in GenLowestOneBit()
2196 __ Rsb(out_reg_lo, in_reg_lo, 0); in GenLowestOneBit()
2197 __ And(out_reg_hi, out_reg_hi, in_reg_hi); in GenLowestOneBit()
2199 __ Ands(out_reg_lo, out_reg_lo, in_reg_lo); in GenLowestOneBit()
2208 __ Mov(LeaveFlags, temp, 0); in GenLowestOneBit()
2214 __ it(ne); in GenLowestOneBit()
2215 __ mov(ne, out_reg_hi, mov_src); in GenLowestOneBit()
2220 __ Rsb(temp, in, 0); in GenLowestOneBit()
2221 __ And(out, temp, in); in GenLowestOneBit()
2285 __ Add(dst_ptr, dstObj, data_offset); in VisitStringGetCharsNoCheck()
2286 __ Add(dst_ptr, dst_ptr, Operand(dstBegin, vixl32::LSL, 1)); in VisitStringGetCharsNoCheck()
2288 __ Subs(num_chr, srcEnd, srcBegin); in VisitStringGetCharsNoCheck()
2290 __ B(eq, final_label, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2293 __ Add(src_ptr, srcObj, value_offset); in VisitStringGetCharsNoCheck()
2303 __ Ldr(temp, MemOperand(srcObj, count_offset)); in VisitStringGetCharsNoCheck()
2304 __ Tst(temp, 1); in VisitStringGetCharsNoCheck()
2306 __ B(eq, &compressed_string_preloop, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2308 __ Add(src_ptr, src_ptr, Operand(srcBegin, vixl32::LSL, 1)); in VisitStringGetCharsNoCheck()
2315 __ Subs(temp, num_chr, 4); in VisitStringGetCharsNoCheck()
2316 __ B(lt, &remainder, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2319 __ Mov(num_chr, temp); in VisitStringGetCharsNoCheck()
2324 __ Bind(&loop); in VisitStringGetCharsNoCheck()
2325 __ Ldr(temp, MemOperand(src_ptr, char_size * 2)); in VisitStringGetCharsNoCheck()
2326 __ Subs(num_chr, num_chr, 4); in VisitStringGetCharsNoCheck()
2327 __ Str(temp, MemOperand(dst_ptr, char_size * 2)); in VisitStringGetCharsNoCheck()
2328 __ Ldr(temp, MemOperand(src_ptr, char_size * 4, PostIndex)); in VisitStringGetCharsNoCheck()
2329 __ Str(temp, MemOperand(dst_ptr, char_size * 4, PostIndex)); in VisitStringGetCharsNoCheck()
2331 __ B(ge, &loop, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2333 __ Adds(num_chr, num_chr, 4); in VisitStringGetCharsNoCheck()
2334 __ B(eq, final_label, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2338 __ Bind(&remainder); in VisitStringGetCharsNoCheck()
2340 __ Ldrh(temp, MemOperand(src_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2341 __ Subs(num_chr, num_chr, 1); in VisitStringGetCharsNoCheck()
2342 __ Strh(temp, MemOperand(dst_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2344 __ B(gt, &remainder, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2347 __ B(final_label); in VisitStringGetCharsNoCheck()
2352 __ Bind(&compressed_string_preloop); in VisitStringGetCharsNoCheck()
2353 __ Add(src_ptr, src_ptr, srcBegin); in VisitStringGetCharsNoCheck()
2354 __ Bind(&compressed_string_loop); in VisitStringGetCharsNoCheck()
2356 __ Ldrb(temp, MemOperand(src_ptr, c_char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2357 __ Strh(temp, MemOperand(dst_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2359 __ Subs(num_chr, num_chr, 1); in VisitStringGetCharsNoCheck()
2360 __ B(gt, &compressed_string_loop, /* is_far_target= */ false); in VisitStringGetCharsNoCheck()
2364 __ Bind(&done); in VisitStringGetCharsNoCheck()
2379 __ Vmov(out, InputSRegisterAt(invoke, 0)); in VisitFloatIsInfinite()
2381 __ Lsl(out, out, 1); in VisitFloatIsInfinite()
2382 __ Eor(out, out, infinity); in VisitFloatIsInfinite()
2404 __ Vmov(temp, out, InputDRegisterAt(invoke, 0)); in VisitDoubleIsInfinite()
2405 __ Eor(out, out, infinity_high); in VisitDoubleIsInfinite()
2406 __ Eor(out, out, infinity_high2); in VisitDoubleIsInfinite()
2408 __ Orr(out, temp, Operand(out, vixl32::LSL, 1)); in VisitDoubleIsInfinite()
2421 __ Vrintp(F64, OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); in VisitMathCeil()
2433 __ Vrintm(F64, OutputDRegister(invoke), InputDRegisterAt(invoke, 0)); in VisitMathFloor()
2472 __ Mov(temp, value); in VisitIntegerValueOf()
2481 __ Add(out, in, -info.low); in VisitIntegerValueOf()
2482 __ Cmp(out, info.length); in VisitIntegerValueOf()
2484 __ B(hs, &allocate, /* is_far_target= */ false); in VisitIntegerValueOf()
2489 __ B(&done); in VisitIntegerValueOf()
2490 __ Bind(&allocate); in VisitIntegerValueOf()
2496 __ Bind(&done); in VisitIntegerValueOf()
2518 __ Ldr(temp, in VisitReferenceGetReferent()
2520 __ Cmp(temp, 0); in VisitReferenceGetReferent()
2521 __ B(eq, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
2535 __ Ldrh(temp, MemOperand(temp, disable_intrinsic_offset.Uint32Value())); in VisitReferenceGetReferent()
2536 __ Cmp(temp, 0); in VisitReferenceGetReferent()
2537 __ B(ne, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
2553 __ Ldr(RegisterFrom(out), MemOperand(RegisterFrom(obj), referent_offset)); in VisitReferenceGetReferent()
2559 __ Bind(slow_path->GetExitLabel()); in VisitReferenceGetReferent()
2584 __ ldr(tmp, MemOperand(obj, referent_offset)); in VisitReferenceRefersTo()
2594 __ Subs(out, tmp, other); in VisitReferenceRefersTo()
2595 __ B(eq, &calculate_result); // `out` is 0 if taken. in VisitReferenceRefersTo()
2598 __ Cmp(tmp, 0); in VisitReferenceRefersTo()
2599 __ B(eq, &calculate_result); // `out` is not 0 if taken. in VisitReferenceRefersTo()
2607 __ Ldr(tmp, MemOperand(tmp, monitor_offset)); in VisitReferenceRefersTo()
2608 __ Cmp(tmp, Operand(0xc0000000)); in VisitReferenceRefersTo()
2609 __ B(lo, &calculate_result); // `out` is not 0 if taken. in VisitReferenceRefersTo()
2612 __ Sub(out, other, Operand(tmp, LSL, LockWord::kForwardingAddressShift)); in VisitReferenceRefersTo()
2614 __ Bind(&calculate_result); in VisitReferenceRefersTo()
2617 __ Sub(out, tmp, other); in VisitReferenceRefersTo()
2621 __ Clz(out, out); in VisitReferenceRefersTo()
2622 __ Lsr(out, out, WhichPowerOf2(out.GetSizeInBits())); in VisitReferenceRefersTo()
2635 __ Ldr(out, MemOperand(tr, offset)); in VisitThreadInterrupted()
2640 __ CompareAndBranchIfZero(out, final_label, /* is_far_target= */ false); in VisitThreadInterrupted()
2641 __ Dmb(vixl32::ISH); in VisitThreadInterrupted()
2642 __ Mov(temp, 0); in VisitThreadInterrupted()
2644 __ Dmb(vixl32::ISH); in VisitThreadInterrupted()
2646 __ Bind(&done); in VisitThreadInterrupted()
2673 __ CompareAndBranchIfZero(divisor, slow_path->GetEntryLabel()); in VisitIntegerDivideUnsigned()
2675 __ Udiv(out, dividend, divisor); in VisitIntegerDivideUnsigned()
2677 __ Bind(slow_path->GetExitLabel()); in VisitIntegerDivideUnsigned()
2704 __ Ldrb(RegisterFrom(out), address); in GenerateIntrinsicGet()
2707 __ Ldrsb(RegisterFrom(out), address); in GenerateIntrinsicGet()
2710 __ Ldrh(RegisterFrom(out), address); in GenerateIntrinsicGet()
2713 __ Ldrsh(RegisterFrom(out), address); in GenerateIntrinsicGet()
2716 __ Ldr(RegisterFrom(out), address); in GenerateIntrinsicGet()
2723 __ Add(temp_reg, base, offset); in GenerateIntrinsicGet()
2725 __ Bind(&loop); in GenerateIntrinsicGet()
2726 __ Ldrexd(LowRegisterFrom(out), HighRegisterFrom(out), MemOperand(temp_reg)); in GenerateIntrinsicGet()
2727 __ Strexd(strexd_tmp, LowRegisterFrom(out), HighRegisterFrom(out), MemOperand(temp_reg)); in GenerateIntrinsicGet()
2728 __ Cmp(strexd_tmp, 0); in GenerateIntrinsicGet()
2729 __ B(ne, &loop); in GenerateIntrinsicGet()
2731 __ Ldrd(LowRegisterFrom(out), HighRegisterFrom(out), address); in GenerateIntrinsicGet()
2738 __ Add(temp, base, offset); in GenerateIntrinsicGet()
2742 __ Ldr(RegisterFrom(out), address); in GenerateIntrinsicGet()
2748 __ Add(temp_reg, base, offset); in GenerateIntrinsicGet()
2749 __ Vldr(SRegisterFrom(out), MemOperand(temp_reg)); in GenerateIntrinsicGet()
2755 __ Add(temp_reg, base, offset); in GenerateIntrinsicGet()
2761 __ Bind(&loop); in GenerateIntrinsicGet()
2762 __ Ldrexd(lo, hi, MemOperand(temp_reg)); in GenerateIntrinsicGet()
2763 __ Strexd(strexd_tmp, lo, hi, MemOperand(temp_reg)); in GenerateIntrinsicGet()
2764 __ Cmp(strexd_tmp, 0); in GenerateIntrinsicGet()
2765 __ B(ne, &loop); in GenerateIntrinsicGet()
2766 __ Vmov(DRegisterFrom(out), lo, hi); in GenerateIntrinsicGet()
2768 __ Vldr(DRegisterFrom(out), MemOperand(temp_reg)); in GenerateIntrinsicGet()
2921 __ Mov(temp, RegisterFrom(value)); in GenerateIntrinsicSet()
2928 __ Add(temp_reg, base, offset); in GenerateIntrinsicSet()
2934 __ Strb(RegisterFrom(value), address); in GenerateIntrinsicSet()
2938 __ Strh(RegisterFrom(value), address); in GenerateIntrinsicSet()
2942 __ Str(RegisterFrom(value), address); in GenerateIntrinsicSet()
2949 __ Bind(&loop); in GenerateIntrinsicSet()
2950 __ Ldrexd(lo_tmp, hi_tmp, address); // Ignore the retrieved value. in GenerateIntrinsicSet()
2951 __ Strexd(lo_tmp, LowRegisterFrom(value), HighRegisterFrom(value), address); in GenerateIntrinsicSet()
2952 __ Cmp(lo_tmp, 0); in GenerateIntrinsicSet()
2953 __ B(ne, &loop); in GenerateIntrinsicSet()
2955 __ Strd(LowRegisterFrom(value), HighRegisterFrom(value), address); in GenerateIntrinsicSet()
2959 __ Vstr(SRegisterFrom(value), address); in GenerateIntrinsicSet()
2967 __ Bind(&loop); in GenerateIntrinsicSet()
2968 __ Ldrexd(lo_tmp, hi_tmp, address); // Ignore the retrieved value. in GenerateIntrinsicSet()
2969 __ Vmov(lo_tmp, hi_tmp, DRegisterFrom(value)); in GenerateIntrinsicSet()
2970 __ Strexd(strexd_tmp, lo_tmp, hi_tmp, address); in GenerateIntrinsicSet()
2971 __ Cmp(strexd_tmp, 0); in GenerateIntrinsicSet()
2972 __ B(ne, &loop); in GenerateIntrinsicSet()
2974 __ Vstr(DRegisterFrom(value), address); in GenerateIntrinsicSet()
3163 __ Ldrexb(RegisterFrom(old_value), MemOperand(ptr)); in EmitLoadExclusive()
3167 __ Ldrexh(RegisterFrom(old_value), MemOperand(ptr)); in EmitLoadExclusive()
3171 __ Ldrex(RegisterFrom(old_value), MemOperand(ptr)); in EmitLoadExclusive()
3174 __ Ldrexd(LowRegisterFrom(old_value), HighRegisterFrom(old_value), MemOperand(ptr)); in EmitLoadExclusive()
3182 __ Sxtb(RegisterFrom(old_value), RegisterFrom(old_value)); in EmitLoadExclusive()
3185 __ Sxth(RegisterFrom(old_value), RegisterFrom(old_value)); in EmitLoadExclusive()
3207 __ Strexb(store_result, RegisterFrom(new_value), MemOperand(ptr)); in EmitStoreExclusive()
3211 __ Strexh(store_result, RegisterFrom(new_value), MemOperand(ptr)); in EmitStoreExclusive()
3215 __ Strex(store_result, RegisterFrom(new_value), MemOperand(ptr)); in EmitStoreExclusive()
3218 __ Strexd( in EmitStoreExclusive()
3269 __ Bind(&loop_head); in GenerateCompareAndSet()
3282 __ Mov(success, 0); // Indicate failure if the comparison fails. in GenerateCompareAndSet()
3285 __ Cmp(LowRegisterFrom(old_value), LowRegisterFrom(expected)); in GenerateCompareAndSet()
3287 __ it(eq); in GenerateCompareAndSet()
3288 __ cmp(eq, HighRegisterFrom(old_value), HighRegisterFrom(expected)); in GenerateCompareAndSet()
3292 __ Cmp(RegisterFrom(old_value), LowRegisterFrom(expected)); in GenerateCompareAndSet()
3294 __ it(ne); in GenerateCompareAndSet()
3295 __ cmp(ne, RegisterFrom(old_value), HighRegisterFrom(expected)); in GenerateCompareAndSet()
3297 __ Cmp(RegisterFrom(old_value), RegisterFrom(expected)); in GenerateCompareAndSet()
3300 __ Mov(LeaveFlags, success, 0); // Indicate failure if the comparison fails. in GenerateCompareAndSet()
3302 __ B(ne, cmp_failure, /*is_far_target=*/ cmp_failure_is_far_target); in GenerateCompareAndSet()
3309 __ Mov(success, 1); // Indicate success if the store succeeds. in GenerateCompareAndSet()
3311 __ Cmp(store_result, 0); in GenerateCompareAndSet()
3312 __ B(ne, &loop_head, /*is_far_target=*/ false); in GenerateCompareAndSet()
3317 __ Eor(success, store_result, 1); in GenerateCompareAndSet()
3372 __ Bind(GetEntryLabel()); in EmitNativeCode()
3380 __ B(mark_old_value_slow_path_->GetEntryLabel()); in EmitNativeCode()
3381 __ Bind(mark_old_value_slow_path_->GetExitLabel()); in EmitNativeCode()
3383 __ Cmp(old_value_temp_, expected_); in EmitNativeCode()
3385 __ Mov(LeaveFlags, success_, 0); // Indicate failure if we take the branch out. in EmitNativeCode()
3389 __ it(ne); in EmitNativeCode()
3390 __ mov(ne, old_value_, old_value_temp_); in EmitNativeCode()
3392 __ B(ne, GetExitLabel()); in EmitNativeCode()
3404 __ Add(tmp_ptr, base_, offset_); in EmitNativeCode()
3421 __ Mov(old_value_, expected_); in EmitNativeCode()
3424 __ B(GetExitLabel()); in EmitNativeCode()
3427 __ Bind(&mark_old_value); in EmitNativeCode()
3436 __ B(update_old_value_slow_path_->GetEntryLabel()); in EmitNativeCode()
3437 __ Bind(update_old_value_slow_path_->GetExitLabel()); in EmitNativeCode()
3439 __ B(GetExitLabel()); in EmitNativeCode()
3533 __ Add(tmp_ptr, base, offset); in GenUnsafeCas()
3545 __ Bind(exit_loop); in GenUnsafeCas()
3631 __ Bind(&loop_label); in GenerateGetAndUpdate()
3640 __ Vadd(sum, DRegisterFrom(old_value), DRegisterFrom(arg)); in GenerateGetAndUpdate()
3641 __ Vmov(HighRegisterFrom(new_value), LowRegisterFrom(new_value), sum); // Swap low/high. in GenerateGetAndUpdate()
3645 __ Vadd(sum, SRegisterFrom(old_value), SRegisterFrom(arg)); in GenerateGetAndUpdate()
3646 __ Vmov(RegisterFrom(new_value), sum); in GenerateGetAndUpdate()
3650 __ Adds(HighRegisterFrom(new_value), LowRegisterFrom(old_value), LowRegisterFrom(arg)); in GenerateGetAndUpdate()
3651 __ Adc(LowRegisterFrom(new_value), HighRegisterFrom(old_value), HighRegisterFrom(arg)); in GenerateGetAndUpdate()
3654 __ Add(RegisterFrom(new_value), RegisterFrom(old_value), RegisterFrom(arg)); in GenerateGetAndUpdate()
3667 __ Vmov(old_value_vreg, LowRegisterFrom(loaded_value), HighRegisterFrom(loaded_value)); in GenerateGetAndUpdate()
3668 __ Vadd(sum, old_value_vreg, DRegisterFrom(arg)); in GenerateGetAndUpdate()
3669 __ Vmov(LowRegisterFrom(new_value), HighRegisterFrom(new_value), sum); in GenerateGetAndUpdate()
3673 __ Vmov(old_value_vreg, RegisterFrom(loaded_value)); in GenerateGetAndUpdate()
3674 __ Vadd(sum, old_value_vreg, SRegisterFrom(arg)); in GenerateGetAndUpdate()
3675 __ Vmov(RegisterFrom(new_value), sum); in GenerateGetAndUpdate()
3677 __ Adds(LowRegisterFrom(new_value), LowRegisterFrom(loaded_value), LowRegisterFrom(arg)); in GenerateGetAndUpdate()
3678 __ Adc(HighRegisterFrom(new_value), HighRegisterFrom(loaded_value), HighRegisterFrom(arg)); in GenerateGetAndUpdate()
3680 __ Add(RegisterFrom(new_value), RegisterFrom(loaded_value), RegisterFrom(arg)); in GenerateGetAndUpdate()
3685 __ And(LowRegisterFrom(new_value), LowRegisterFrom(loaded_value), LowRegisterFrom(arg)); in GenerateGetAndUpdate()
3686 __ And(HighRegisterFrom(new_value), HighRegisterFrom(loaded_value), HighRegisterFrom(arg)); in GenerateGetAndUpdate()
3688 __ And(RegisterFrom(new_value), RegisterFrom(loaded_value), RegisterFrom(arg)); in GenerateGetAndUpdate()
3693 __ Orr(LowRegisterFrom(new_value), LowRegisterFrom(loaded_value), LowRegisterFrom(arg)); in GenerateGetAndUpdate()
3694 __ Orr(HighRegisterFrom(new_value), HighRegisterFrom(loaded_value), HighRegisterFrom(arg)); in GenerateGetAndUpdate()
3696 __ Orr(RegisterFrom(new_value), RegisterFrom(loaded_value), RegisterFrom(arg)); in GenerateGetAndUpdate()
3701 __ Eor(LowRegisterFrom(new_value), LowRegisterFrom(loaded_value), LowRegisterFrom(arg)); in GenerateGetAndUpdate()
3702 __ Eor(HighRegisterFrom(new_value), HighRegisterFrom(loaded_value), HighRegisterFrom(arg)); in GenerateGetAndUpdate()
3704 __ Eor(RegisterFrom(new_value), RegisterFrom(loaded_value), RegisterFrom(arg)); in GenerateGetAndUpdate()
3709 __ Cmp(store_result, 0); in GenerateGetAndUpdate()
3710 __ B(ne, &loop_label); in GenerateGetAndUpdate()
3797 __ CompareAndBranchIfZero(object, &success, /*is_far_target=*/ false); in GenerateSubTypeObjectCheckNoReadBarrier()
3803 __ Ldr(temp, MemOperand(object, class_offset.Int32Value())); in GenerateSubTypeObjectCheckNoReadBarrier()
3806 __ Bind(&loop); in GenerateSubTypeObjectCheckNoReadBarrier()
3807 __ Cmp(type, temp); in GenerateSubTypeObjectCheckNoReadBarrier()
3808 __ B(eq, &success, /*is_far_target=*/ false); in GenerateSubTypeObjectCheckNoReadBarrier()
3809 __ Ldr(temp, MemOperand(temp, super_class_offset.Int32Value())); in GenerateSubTypeObjectCheckNoReadBarrier()
3811 __ Cmp(temp, 0); in GenerateSubTypeObjectCheckNoReadBarrier()
3812 __ B(eq, slow_path->GetEntryLabel()); in GenerateSubTypeObjectCheckNoReadBarrier()
3813 __ B(&loop); in GenerateSubTypeObjectCheckNoReadBarrier()
3814 __ Bind(&success); in GenerateSubTypeObjectCheckNoReadBarrier()
3847 __ Ldrd(var_type_no_rb, temp2, MemOperand(varhandle, var_type_offset.Int32Value())); in GenerateVarHandleAccessModeAndVarTypeChecks()
3849 __ Tst(temp2, 1u << static_cast<uint32_t>(access_mode)); in GenerateVarHandleAccessModeAndVarTypeChecks()
3850 __ B(eq, slow_path->GetEntryLabel()); in GenerateVarHandleAccessModeAndVarTypeChecks()
3851 __ Ldrh(temp2, MemOperand(var_type_no_rb, primitive_type_offset.Int32Value())); in GenerateVarHandleAccessModeAndVarTypeChecks()
3852 __ Cmp(temp2, static_cast<uint16_t>(primitive_type)); in GenerateVarHandleAccessModeAndVarTypeChecks()
3853 __ B(ne, slow_path->GetEntryLabel()); in GenerateVarHandleAccessModeAndVarTypeChecks()
3886 __ Ldr(temp, MemOperand(varhandle, coordinate_type0_offset.Int32Value())); in GenerateVarHandleStaticFieldCheck()
3887 __ Cmp(temp, 0); in GenerateVarHandleStaticFieldCheck()
3888 __ B(ne, slow_path->GetEntryLabel()); in GenerateVarHandleStaticFieldCheck()
3902 __ Cmp(object, 0); in GenerateVarHandleInstanceFieldChecks()
3903 __ B(eq, slow_path->GetEntryLabel()); in GenerateVarHandleInstanceFieldChecks()
3916 __ Ldrd(temp, temp2, MemOperand(varhandle, coordinate_type0_offset.Int32Value())); in GenerateVarHandleInstanceFieldChecks()
3919 __ Cmp(temp2, 0); in GenerateVarHandleInstanceFieldChecks()
3920 __ B(ne, slow_path->GetEntryLabel()); in GenerateVarHandleInstanceFieldChecks()
3960 __ Cmp(object, 0); in GenerateVarHandleArrayChecks()
3961 __ B(eq, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
3973 __ Ldrd(temp, temp2, MemOperand(varhandle, coordinate_type0_offset.Int32Value())); in GenerateVarHandleArrayChecks()
3976 __ Cmp(temp2, 0); in GenerateVarHandleArrayChecks()
3977 __ B(eq, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
3989 __ Ldr(temp2, MemOperand(object, class_offset.Int32Value())); in GenerateVarHandleArrayChecks()
3991 __ Cmp(temp, temp2); in GenerateVarHandleArrayChecks()
3992 __ B(ne, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
3997 __ Ldr(temp2, MemOperand(temp, component_type_offset.Int32Value())); in GenerateVarHandleArrayChecks()
3999 __ Cmp(temp2, 0); in GenerateVarHandleArrayChecks()
4000 __ B(eq, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
4016 __ Ldrh(temp2, MemOperand(temp2, primitive_type_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4017 __ Cmp(temp2, static_cast<uint16_t>(primitive_type)); in GenerateVarHandleArrayChecks()
4018 __ B(ne, slow_path_label); in GenerateVarHandleArrayChecks()
4021 __ Ldr(temp, MemOperand(object, array_length_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4022 __ Cmp(index, temp); in GenerateVarHandleArrayChecks()
4023 __ B(hs, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
4091 __ Ldr(method, MemOperand(varhandle, art_field_offset.Int32Value())); in GenerateVarHandleTarget()
4092 __ Ldr(target.offset, MemOperand(method, offset_offset.Int32Value())); in GenerateVarHandleTarget()
4111 __ Lsl(shifted_index, index, size_shift); in GenerateVarHandleTarget()
4113 __ Add(target.offset, shifted_index, data_offset.Int32Value()); in GenerateVarHandleTarget()
4298 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleGet()
4363 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleGet()
4452 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleSet()
4473 __ Add(base, target.object, target.offset); in GenerateVarHandleSet()
4489 __ Vmov(HighRegisterFrom(value), LowRegisterFrom(value), DRegisterFrom(original_value)); in GenerateVarHandleSet()
4497 __ Vmov(RegisterFrom(value), SRegisterFrom(original_value)); in GenerateVarHandleSet()
4526 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleSet()
4651 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleCompareAndSetOrExchange()
4667 __ Add(tmp_ptr, target.object, target.offset); in GenerateVarHandleCompareAndSetOrExchange()
4690 __ Vmov(HighRegisterFrom(expected), LowRegisterFrom(expected), expected_vreg); in GenerateVarHandleCompareAndSetOrExchange()
4691 __ Vmov(HighRegisterFrom(new_value), LowRegisterFrom(new_value), new_value_vreg); in GenerateVarHandleCompareAndSetOrExchange()
4695 __ Vmov(LowRegisterFrom(expected), HighRegisterFrom(expected), expected_vreg); in GenerateVarHandleCompareAndSetOrExchange()
4696 __ Vmov(LowRegisterFrom(new_value), HighRegisterFrom(new_value), new_value_vreg); in GenerateVarHandleCompareAndSetOrExchange()
4706 __ Vmov(RegisterFrom(expected), expected_vreg); in GenerateVarHandleCompareAndSetOrExchange()
4707 __ Vmov(RegisterFrom(new_value), new_value_vreg); in GenerateVarHandleCompareAndSetOrExchange()
4782 __ Bind(exit_loop); in GenerateVarHandleCompareAndSetOrExchange()
4801 __ Vmov(DRegisterFrom(out), LowRegisterFrom(old_value), HighRegisterFrom(old_value)); in GenerateVarHandleCompareAndSetOrExchange()
4803 __ Vmov(SRegisterFrom(out), RegisterFrom(old_value)); in GenerateVarHandleCompareAndSetOrExchange()
4817 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleCompareAndSetOrExchange()
4970 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleGetAndUpdate()
4986 __ Add(tmp_ptr, target.object, target.offset); in GenerateVarHandleGetAndUpdate()
5010 __ Vmov(HighRegisterFrom(arg), LowRegisterFrom(arg), arg_vreg); in GenerateVarHandleGetAndUpdate()
5013 __ Vmov(LowRegisterFrom(arg), HighRegisterFrom(arg), arg_vreg); in GenerateVarHandleGetAndUpdate()
5020 __ Vmov(RegisterFrom(arg), arg_vreg); in GenerateVarHandleGetAndUpdate()
5103 __ Vmov(DRegisterFrom(out), LowRegisterFrom(old_value), HighRegisterFrom(old_value)); in GenerateVarHandleGetAndUpdate()
5105 __ Vmov(SRegisterFrom(out), RegisterFrom(old_value)); in GenerateVarHandleGetAndUpdate()
5132 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleGetAndUpdate()
5277 __ Bind(GetByteArrayViewCheckLabel()); in EmitByteArrayViewCode()
5290 __ Ldr(temp, MemOperand(varhandle, class_offset.Int32Value())); in EmitByteArrayViewCode()
5292 __ Cmp(temp, temp2); in EmitByteArrayViewCode()
5293 __ B(ne, GetEntryLabel()); in EmitByteArrayViewCode()
5296 __ Ldr(temp, MemOperand(object, array_length_offset.Int32Value())); in EmitByteArrayViewCode()
5299 __ Mov(temp2, size_operand); in EmitByteArrayViewCode()
5301 __ Subs(temp, temp, index); in EmitByteArrayViewCode()
5306 __ it(hs); in EmitByteArrayViewCode()
5308 __ cmp(hs, temp, size_operand); in EmitByteArrayViewCode()
5310 __ cmp(hs, temp, temp2); in EmitByteArrayViewCode()
5313 __ B(lo, GetEntryLabel()); in EmitByteArrayViewCode()
5316 __ Add(target.offset, index, data_offset.Int32Value()); // Note: `temp` cannot be used below. in EmitByteArrayViewCode()
5320 __ Tst(target.offset, dchecked_integral_cast<int32_t>(size - 1u)); in EmitByteArrayViewCode()
5321 __ B(ne, GetEntryLabel()); in EmitByteArrayViewCode()
5329 __ B(GetNativeByteOrderLabel()); in EmitByteArrayViewCode()
5333 __ Ldr(temp2, MemOperand(varhandle, native_byte_order_offset.Int32Value())); in EmitByteArrayViewCode()
5334 __ Cmp(temp2, 0); in EmitByteArrayViewCode()
5335 __ B(ne, GetNativeByteOrderLabel()); in EmitByteArrayViewCode()
5355 __ B(GetExitLabel()); in EmitByteArrayViewCode()
5406 #undef __