Lines Matching refs:__
88 #define __ codegen->GetVIXLAssembler()-> macro
116 __ Bind(GetEntryLabel()); in EmitNativeCode()
118 __ Bind(&slow_copy_loop); in EmitNativeCode()
119 __ Ldr(tmp_reg, MemOperand(src_curr_addr, element_size, PostIndex)); in EmitNativeCode()
146 __ Str(tmp_reg, MemOperand(dst_curr_addr, element_size, PostIndex)); in EmitNativeCode()
147 __ Cmp(src_curr_addr, src_stop_addr); in EmitNativeCode()
148 __ B(&slow_copy_loop, ne); in EmitNativeCode()
149 __ B(GetExitLabel()); in EmitNativeCode()
159 #undef __
170 #define __ masm-> macro
189 __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output), in MoveFPToInt()
196 __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output), in MoveIntToFP()
258 __ Rev16(out.W(), in.W()); in GenerateReverseBytes()
261 __ Rev16(out.W(), in.W()); in GenerateReverseBytes()
262 __ Sxth(out.W(), out.W()); in GenerateReverseBytes()
265 __ Rev(out.W(), in.W()); in GenerateReverseBytes()
268 __ Rev(out.X(), in.X()); in GenerateReverseBytes()
271 __ Rev(in.W(), in.W()); // Note: Clobbers `in`. in GenerateReverseBytes()
272 __ Fmov(out.S(), in.W()); in GenerateReverseBytes()
275 __ Rev(in.X(), in.X()); // Note: Clobbers `in`. in GenerateReverseBytes()
276 __ Fmov(out.D(), in.X()); in GenerateReverseBytes()
324 __ Clz(RegisterFrom(out, type), RegisterFrom(in, type)); in GenNumberOfLeadingZeros()
351 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type)); in GenNumberOfTrailingZeros()
352 __ Clz(RegisterFrom(out, type), RegisterFrom(out, type)); in GenNumberOfTrailingZeros()
379 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type)); in GenReverse()
409 __ Fmov(fpr, src); in GenBitCount()
410 __ Cnt(fpr.V8B(), fpr.V8B()); in GenBitCount()
411 __ Addv(fpr.B(), fpr.V8B()); in GenBitCount()
412 __ Fmov(dst, fpr); in GenBitCount()
442 __ Clz(temp, src); in GenHighestOneBit()
443 __ Mov(dst, UINT64_C(1) << high_bit); // MOV (bitmask immediate) in GenHighestOneBit()
444 __ Bic(dst, dst, Operand(temp, LSL, high_bit - clz_high_bit)); // Clear dst if src was 0. in GenHighestOneBit()
445 __ Lsr(dst, dst, temp); in GenHighestOneBit()
473 __ Neg(temp, src); in GenLowestOneBit()
474 __ And(dst, temp, src); in GenLowestOneBit()
507 __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); in VisitMathSqrt()
517 __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); in VisitMathCeil()
527 __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); in VisitMathFloor()
537 __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); in VisitMathRint()
569 __ Fcvtas(out_reg, in_reg); in GenMathRound()
572 __ Tbz(out_reg, out_reg.GetSizeInBits() - 1, &done); in GenMathRound()
577 __ Frinta(tmp_fp, in_reg); in GenMathRound()
578 __ Fsub(tmp_fp, in_reg, tmp_fp); in GenMathRound()
579 __ Fcmp(tmp_fp, 0.5); in GenMathRound()
580 __ Cinc(out_reg, out_reg, eq); in GenMathRound()
582 __ Bind(&done); in GenMathRound()
607 __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekByte()
617 __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekIntNative()
627 __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekLongNative()
637 __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekShortNative()
654 __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeByte()
664 __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeIntNative()
674 __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeLongNative()
684 __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeShortNative()
719 __ Add(temp, base, offset.W()); // Offset should not exceed 32 bits. in GenUnsafeGet()
967 __ Mov(temp.W(), value.W()); in GenUnsafePut()
1130 __ Ldaxrb(old_value, MemOperand(ptr)); in EmitLoadExclusive()
1132 __ Ldxrb(old_value, MemOperand(ptr)); in EmitLoadExclusive()
1138 __ Ldaxrh(old_value, MemOperand(ptr)); in EmitLoadExclusive()
1140 __ Ldxrh(old_value, MemOperand(ptr)); in EmitLoadExclusive()
1147 __ Ldaxr(old_value, MemOperand(ptr)); in EmitLoadExclusive()
1149 __ Ldxr(old_value, MemOperand(ptr)); in EmitLoadExclusive()
1158 __ Sxtb(old_value, old_value); in EmitLoadExclusive()
1161 __ Sxth(old_value, old_value); in EmitLoadExclusive()
1187 __ Stlxrb(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1189 __ Stxrb(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1195 __ Stlxrh(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1197 __ Stxrh(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1204 __ Stlxr(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1206 __ Stxr(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1265 __ Bind(&loop_head); in GenerateCompareAndSet()
1268 __ Cmp(old_value, expected); in GenerateCompareAndSet()
1270 __ Ccmp(old_value, expected2, ZFlag, ne); in GenerateCompareAndSet()
1275 __ B(cmp_failure, ne); in GenerateCompareAndSet()
1278 __ Cbnz(store_result, &loop_head); in GenerateCompareAndSet()
1281 __ Eor(store_result, store_result, 1); in GenerateCompareAndSet()
1339 __ Bind(GetEntryLabel()); in EmitNativeCode()
1347 __ B(mark_old_value_slow_path_->GetEntryLabel()); in EmitNativeCode()
1348 __ Bind(mark_old_value_slow_path_->GetExitLabel()); in EmitNativeCode()
1350 __ Cmp(old_value_temp_, expected_); in EmitNativeCode()
1353 __ Csel(old_value_, old_value_temp_, old_value_, ne); in EmitNativeCode()
1355 __ B(GetExitLabel(), ne); // If taken, Z=false indicates failure. in EmitNativeCode()
1369 __ Add(tmp_ptr, base_.X(), Operand(offset_)); in EmitNativeCode()
1386 __ Mov(old_value_, expected_); in EmitNativeCode()
1392 __ B(GetExitLabel()); in EmitNativeCode()
1395 __ Bind(&mark_old_value); in EmitNativeCode()
1404 __ B(update_old_value_slow_path_->GetEntryLabel()); in EmitNativeCode()
1405 __ Bind(update_old_value_slow_path_->GetExitLabel()); in EmitNativeCode()
1407 __ B(GetExitLabel()); in EmitNativeCode()
1479 __ Add(tmp_ptr, base.X(), Operand(offset)); in GenUnsafeCas()
1491 __ Bind(exit_loop); in GenUnsafeCas()
1492 __ Cset(out, eq); in GenUnsafeCas()
1637 __ Bind(&loop_label); in GenerateGetAndUpdate()
1652 __ Fmov(old_value_vreg, old_value_reg); in GenerateGetAndUpdate()
1653 __ Fadd(sum, old_value_vreg, arg.IsD() ? arg.D() : arg.S()); in GenerateGetAndUpdate()
1654 __ Fmov(new_value, sum); in GenerateGetAndUpdate()
1656 __ Add(new_value, old_value_reg, arg.IsX() ? arg.X() : arg.W()); in GenerateGetAndUpdate()
1663 __ And(new_value, old_value_reg, arg.IsX() ? arg.X() : arg.W()); in GenerateGetAndUpdate()
1666 __ Orr(new_value, old_value_reg, arg.IsX() ? arg.X() : arg.W()); in GenerateGetAndUpdate()
1669 __ Eor(new_value, old_value_reg, arg.IsX() ? arg.X() : arg.W()); in GenerateGetAndUpdate()
1673 __ Cbnz(store_result, &loop_label); in GenerateGetAndUpdate()
1731 __ Cbz(arg, slow_path->GetEntryLabel()); in VisitStringCompareTo()
1735 __ Subs(out, str, arg); in VisitStringCompareTo()
1736 __ B(&end, eq); in VisitStringCompareTo()
1740 __ Ldr(temp3, HeapOperand(str, count_offset)); in VisitStringCompareTo()
1741 __ Ldr(temp2, HeapOperand(arg, count_offset)); in VisitStringCompareTo()
1743 __ Lsr(temp0, temp3, 1u); in VisitStringCompareTo()
1744 __ Lsr(temp1, temp2, 1u); in VisitStringCompareTo()
1747 __ Ldr(temp0, HeapOperand(str, count_offset)); in VisitStringCompareTo()
1748 __ Ldr(temp1, HeapOperand(arg, count_offset)); in VisitStringCompareTo()
1751 __ Subs(out, temp0, temp1); in VisitStringCompareTo()
1753 __ Csel(temp0, temp1, temp0, ge); in VisitStringCompareTo()
1755 __ Cbz(temp0, &end); in VisitStringCompareTo()
1759 __ Eor(temp2, temp2, Operand(temp3)); in VisitStringCompareTo()
1762 __ Ands(temp3.W(), temp3.W(), Operand(1)); in VisitStringCompareTo()
1763 __ Tbnz(temp2, 0, &different_compression); // Does not use flags. in VisitStringCompareTo()
1766 __ Mov(temp1, value_offset); in VisitStringCompareTo()
1770 __ Lsl(temp0, temp0, temp3); in VisitStringCompareTo()
1787 __ Bind(&loop); in VisitStringCompareTo()
1788 __ Ldr(temp4, MemOperand(str.X(), temp1.X())); in VisitStringCompareTo()
1789 __ Ldr(temp2, MemOperand(arg.X(), temp1.X())); in VisitStringCompareTo()
1790 __ Cmp(temp4, temp2); in VisitStringCompareTo()
1791 __ B(ne, &find_char_diff); in VisitStringCompareTo()
1792 __ Add(temp1, temp1, char_size * 4); in VisitStringCompareTo()
1794 __ Subs(temp0, temp0, (mirror::kUseStringCompression) ? 8 : 4); in VisitStringCompareTo()
1795 __ B(&loop, hi); in VisitStringCompareTo()
1796 __ B(&end); in VisitStringCompareTo()
1802 __ Bind(&find_char_diff); in VisitStringCompareTo()
1804 __ Eor(temp1, temp2, temp4); in VisitStringCompareTo()
1805 __ Rbit(temp1, temp1); in VisitStringCompareTo()
1806 __ Clz(temp1, temp1); in VisitStringCompareTo()
1814 __ Cmp(temp0, Operand(temp1.W(), LSR, (mirror::kUseStringCompression) ? 3 : 4)); in VisitStringCompareTo()
1815 __ B(ls, &end); in VisitStringCompareTo()
1819 __ Bic(temp1, temp1, 0x7); in VisitStringCompareTo()
1820 __ Bic(temp1, temp1, Operand(temp3.X(), LSL, 3u)); in VisitStringCompareTo()
1822 __ Bic(temp1, temp1, 0xf); in VisitStringCompareTo()
1824 __ Lsr(temp2, temp2, temp1); in VisitStringCompareTo()
1825 __ Lsr(temp4, temp4, temp1); in VisitStringCompareTo()
1828 __ Uxtb(temp1, temp4); in VisitStringCompareTo()
1829 __ Sub(out, temp1.W(), Operand(temp2.W(), UXTB)); in VisitStringCompareTo()
1830 __ Tbz(temp3, 0u, &end); // If actually compressed, we're done. in VisitStringCompareTo()
1832 __ Uxth(temp4, temp4); in VisitStringCompareTo()
1833 __ Sub(out, temp4.W(), Operand(temp2.W(), UXTH)); in VisitStringCompareTo()
1836 __ B(&end); in VisitStringCompareTo()
1837 __ Bind(&different_compression); in VisitStringCompareTo()
1849 __ Csel(temp1, str, arg, eq); // Pointer to the compressed string. in VisitStringCompareTo()
1850 __ Csel(temp2, str, arg, ne); // Pointer to the uncompressed string. in VisitStringCompareTo()
1856 __ Lsl(temp0, temp0, 1u); in VisitStringCompareTo()
1859 __ Add(temp1, temp1, Operand(value_offset)); in VisitStringCompareTo()
1860 __ Add(temp2, temp2, Operand(value_offset)); in VisitStringCompareTo()
1863 __ Sub(temp0, temp0, Operand(temp3)); in VisitStringCompareTo()
1868 __ Bind(&different_compression_loop); in VisitStringCompareTo()
1869 __ Ldrb(temp4, MemOperand(temp1.X(), c_char_size, PostIndex)); in VisitStringCompareTo()
1870 __ Ldrh(temp3, MemOperand(temp2.X(), char_size, PostIndex)); in VisitStringCompareTo()
1871 __ Subs(temp4, temp4, Operand(temp3)); in VisitStringCompareTo()
1872 __ B(&different_compression_diff, ne); in VisitStringCompareTo()
1873 __ Subs(temp0, temp0, 2); in VisitStringCompareTo()
1874 __ B(&different_compression_loop, hi); in VisitStringCompareTo()
1875 __ B(&end); in VisitStringCompareTo()
1878 __ Bind(&different_compression_diff); in VisitStringCompareTo()
1879 __ Tst(temp0, Operand(1)); in VisitStringCompareTo()
1882 __ Cneg(out, temp4, ne); in VisitStringCompareTo()
1885 __ Bind(&end); in VisitStringCompareTo()
1888 __ Bind(slow_path->GetExitLabel()); in VisitStringCompareTo()
1963 __ Cbz(arg, &return_false); in VisitStringEquals()
1967 __ Cmp(str, arg); in VisitStringEquals()
1968 __ B(&return_true, eq); in VisitStringEquals()
1980 __ Ldr(temp, MemOperand(str.X(), class_offset)); in VisitStringEquals()
1982 __ Ldr(temp1, MemOperand(arg.X(), class_offset)); in VisitStringEquals()
1985 __ Cmp(temp, temp1); in VisitStringEquals()
1986 __ B(&return_false, ne); in VisitStringEquals()
2007 __ Ldr(temp, MemOperand(arg.X(), count_offset)); in VisitStringEquals()
2010 __ Cmp(temp, Operand(mirror::String::GetFlaggedCount(const_string_length, is_compressed))); in VisitStringEquals()
2012 __ B(&return_false, ne); in VisitStringEquals()
2015 __ Ldr(temp, MemOperand(str.X(), count_offset)); in VisitStringEquals()
2016 __ Ldr(temp1, MemOperand(arg.X(), count_offset)); in VisitStringEquals()
2019 __ Cmp(temp, temp1); in VisitStringEquals()
2020 __ B(&return_false, ne); in VisitStringEquals()
2040 __ Ldp(temp, temp1, MemOperand(str.X(), offset)); in VisitStringEquals()
2041 __ Ldp(temp2, out, MemOperand(arg.X(), offset)); in VisitStringEquals()
2042 __ Cmp(temp, temp2); in VisitStringEquals()
2043 __ Ccmp(temp1, out, NoFlag, eq); in VisitStringEquals()
2044 __ B(&return_false, ne); in VisitStringEquals()
2049 __ Ldr(temp, MemOperand(str.X(), offset)); in VisitStringEquals()
2050 __ Ldr(temp1, MemOperand(arg.X(), offset)); in VisitStringEquals()
2051 __ Cmp(temp, temp1); in VisitStringEquals()
2052 __ B(&return_false, ne); in VisitStringEquals()
2058 __ Cbz(temp, &return_true); in VisitStringEquals()
2063 __ And(temp1, temp, Operand(1)); // Extract compression flag. in VisitStringEquals()
2064 __ Lsr(temp, temp, 1u); // Extract length. in VisitStringEquals()
2065 __ Lsl(temp, temp, temp1); // Calculate number of bytes to compare. in VisitStringEquals()
2069 __ Mov(temp1, value_offset); in VisitStringEquals()
2074 __ Bind(&loop); in VisitStringEquals()
2075 __ Ldr(out, MemOperand(str.X(), temp1)); in VisitStringEquals()
2076 __ Ldr(temp2, MemOperand(arg.X(), temp1)); in VisitStringEquals()
2077 __ Add(temp1, temp1, Operand(sizeof(uint64_t))); in VisitStringEquals()
2078 __ Cmp(out, temp2); in VisitStringEquals()
2079 __ B(&return_false, ne); in VisitStringEquals()
2081 __ Sub(temp, temp, Operand(mirror::kUseStringCompression ? 8 : 4), SetFlags); in VisitStringEquals()
2082 __ B(&loop, hi); in VisitStringEquals()
2087 __ Bind(&return_true); in VisitStringEquals()
2088 __ Mov(out, 1); in VisitStringEquals()
2089 __ B(&end); in VisitStringEquals()
2092 __ Bind(&return_false); in VisitStringEquals()
2093 __ Mov(out, 0); in VisitStringEquals()
2094 __ Bind(&end); in VisitStringEquals()
2116 __ B(slow_path->GetEntryLabel()); in GenerateVisitStringIndexOf()
2117 __ Bind(slow_path->GetExitLabel()); in GenerateVisitStringIndexOf()
2122 __ Tst(char_reg, 0xFFFF0000); in GenerateVisitStringIndexOf()
2125 __ B(ne, slow_path->GetEntryLabel()); in GenerateVisitStringIndexOf()
2131 __ Mov(tmp_reg, 0); in GenerateVisitStringIndexOf()
2138 __ Bind(slow_path->GetExitLabel()); in GenerateVisitStringIndexOf()
2192 __ Cmp(byte_array, 0); in VisitStringNewStringFromBytes()
2196 __ B(eq, slow_path->GetEntryLabel()); in VisitStringNewStringFromBytes()
2200 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromBytes()
2237 __ Cmp(string_to_copy, 0); in VisitStringNewStringFromString()
2241 __ B(eq, slow_path->GetEntryLabel()); in VisitStringNewStringFromString()
2245 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromString()
2489 __ Sub(num_chr, srcEnd, srcBegin); in VisitStringGetCharsNoCheck()
2491 __ Cbz(num_chr, &done); in VisitStringGetCharsNoCheck()
2494 __ Add(dst_ptr, dstObj, Operand(data_offset)); in VisitStringGetCharsNoCheck()
2495 __ Add(dst_ptr, dst_ptr, Operand(dstBegin, LSL, 1)); in VisitStringGetCharsNoCheck()
2498 __ Add(src_ptr, srcObj, Operand(value_offset)); in VisitStringGetCharsNoCheck()
2504 __ Ldr(tmp2, MemOperand(srcObj, count_offset)); in VisitStringGetCharsNoCheck()
2505 __ Tbz(tmp2, 0, &compressed_string_preloop); in VisitStringGetCharsNoCheck()
2507 __ Add(src_ptr, src_ptr, Operand(srcBegin, LSL, 1)); in VisitStringGetCharsNoCheck()
2514 __ Subs(tmp1, num_chr, 8); in VisitStringGetCharsNoCheck()
2515 __ B(lt, &remainder); in VisitStringGetCharsNoCheck()
2518 __ Mov(num_chr, tmp1); in VisitStringGetCharsNoCheck()
2522 __ Bind(&loop); in VisitStringGetCharsNoCheck()
2523 __ Ldp(tmp1, tmp2, MemOperand(src_ptr, char_size * 8, PostIndex)); in VisitStringGetCharsNoCheck()
2524 __ Subs(num_chr, num_chr, 8); in VisitStringGetCharsNoCheck()
2525 __ Stp(tmp1, tmp2, MemOperand(dst_ptr, char_size * 8, PostIndex)); in VisitStringGetCharsNoCheck()
2526 __ B(ge, &loop); in VisitStringGetCharsNoCheck()
2528 __ Adds(num_chr, num_chr, 8); in VisitStringGetCharsNoCheck()
2529 __ B(eq, &done); in VisitStringGetCharsNoCheck()
2533 __ Bind(&remainder); in VisitStringGetCharsNoCheck()
2534 __ Ldrh(tmp1, MemOperand(src_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2535 __ Subs(num_chr, num_chr, 1); in VisitStringGetCharsNoCheck()
2536 __ Strh(tmp1, MemOperand(dst_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2537 __ B(gt, &remainder); in VisitStringGetCharsNoCheck()
2538 __ B(&done); in VisitStringGetCharsNoCheck()
2545 __ Bind(&compressed_string_preloop); in VisitStringGetCharsNoCheck()
2546 __ Add(src_ptr, src_ptr, Operand(srcBegin)); in VisitStringGetCharsNoCheck()
2549 __ Subs(tmp1, num_chr, 8); in VisitStringGetCharsNoCheck()
2550 __ B(lt, &compressed_string_remainder); in VisitStringGetCharsNoCheck()
2553 __ Mov(num_chr, tmp1); in VisitStringGetCharsNoCheck()
2557 __ Bind(&compressed_string_vector_loop); in VisitStringGetCharsNoCheck()
2558 __ Ld1(vtmp1.V8B(), MemOperand(src_ptr, c_char_size * 8, PostIndex)); in VisitStringGetCharsNoCheck()
2559 __ Subs(num_chr, num_chr, 8); in VisitStringGetCharsNoCheck()
2560 __ Uxtl(vtmp1.V8H(), vtmp1.V8B()); in VisitStringGetCharsNoCheck()
2561 __ St1(vtmp1.V8H(), MemOperand(dst_ptr, char_size * 8, PostIndex)); in VisitStringGetCharsNoCheck()
2562 __ B(ge, &compressed_string_vector_loop); in VisitStringGetCharsNoCheck()
2564 __ Adds(num_chr, num_chr, 8); in VisitStringGetCharsNoCheck()
2565 __ B(eq, &done); in VisitStringGetCharsNoCheck()
2569 __ Bind(&compressed_string_remainder); in VisitStringGetCharsNoCheck()
2570 __ Ldrb(tmp1, MemOperand(src_ptr, c_char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2571 __ Strh(tmp1, MemOperand(dst_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2572 __ Subs(num_chr, num_chr, Operand(1)); in VisitStringGetCharsNoCheck()
2573 __ B(gt, &compressed_string_remainder); in VisitStringGetCharsNoCheck()
2576 __ Bind(&done); in VisitStringGetCharsNoCheck()
2646 __ Ldr(temp, MemOperand(input, length_offset)); in CheckSystemArrayCopyPosition()
2647 __ Cmp(temp, OperandFrom(length, DataType::Type::kInt32)); in CheckSystemArrayCopyPosition()
2648 __ B(slow_path->GetEntryLabel(), lt); in CheckSystemArrayCopyPosition()
2652 __ Ldr(temp, MemOperand(input, length_offset)); in CheckSystemArrayCopyPosition()
2653 __ Subs(temp, temp, pos_const); in CheckSystemArrayCopyPosition()
2654 __ B(slow_path->GetEntryLabel(), lt); in CheckSystemArrayCopyPosition()
2657 __ Cmp(temp, OperandFrom(length, DataType::Type::kInt32)); in CheckSystemArrayCopyPosition()
2658 __ B(slow_path->GetEntryLabel(), lt); in CheckSystemArrayCopyPosition()
2662 __ Cbnz(WRegisterFrom(pos), slow_path->GetEntryLabel()); in CheckSystemArrayCopyPosition()
2666 __ Tbnz(pos_reg, pos_reg.GetSizeInBits() - 1, slow_path->GetEntryLabel()); in CheckSystemArrayCopyPosition()
2669 __ Ldr(temp, MemOperand(input, length_offset)); in CheckSystemArrayCopyPosition()
2670 __ Subs(temp, temp, pos_reg); in CheckSystemArrayCopyPosition()
2672 __ Ccmp(temp, OperandFrom(length, DataType::Type::kInt32), NFlag, ge); in CheckSystemArrayCopyPosition()
2673 __ B(slow_path->GetEntryLabel(), lt); in CheckSystemArrayCopyPosition()
2699 __ Add(src_base, src, element_size * constant + data_offset); in GenSystemArrayCopyAddresses()
2701 __ Add(src_base, src, data_offset); in GenSystemArrayCopyAddresses()
2702 __ Add(src_base, src_base, Operand(XRegisterFrom(src_pos), LSL, element_size_shift)); in GenSystemArrayCopyAddresses()
2707 __ Add(dst_base, dst, element_size * constant + data_offset); in GenSystemArrayCopyAddresses()
2709 __ Add(dst_base, dst, data_offset); in GenSystemArrayCopyAddresses()
2710 __ Add(dst_base, dst_base, Operand(XRegisterFrom(dst_pos), LSL, element_size_shift)); in GenSystemArrayCopyAddresses()
2715 __ Add(src_end, src_base, element_size * constant); in GenSystemArrayCopyAddresses()
2717 __ Add(src_end, src_base, Operand(XRegisterFrom(copy_length), LSL, element_size_shift)); in GenSystemArrayCopyAddresses()
2736 __ Cmp(src, dst); in VisitSystemArrayCopyChar()
2737 __ B(slow_path->GetEntryLabel(), eq); in VisitSystemArrayCopyChar()
2740 __ Cbz(src, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
2743 __ Cbz(dst, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
2749 __ Cmp(WRegisterFrom(length), kSystemArrayCopyCharThreshold); in VisitSystemArrayCopyChar()
2750 __ B(slow_path->GetEntryLabel(), hi); in VisitSystemArrayCopyChar()
2797 __ Bind(&loop); in VisitSystemArrayCopyChar()
2798 __ Cmp(src_curr_addr, src_stop_addr); in VisitSystemArrayCopyChar()
2799 __ B(&done, eq); in VisitSystemArrayCopyChar()
2800 __ Ldrh(tmp, MemOperand(src_curr_addr, char_size, PostIndex)); in VisitSystemArrayCopyChar()
2801 __ Strh(tmp, MemOperand(dst_curr_addr, char_size, PostIndex)); in VisitSystemArrayCopyChar()
2802 __ B(&loop); in VisitSystemArrayCopyChar()
2803 __ Bind(&done); in VisitSystemArrayCopyChar()
2805 __ Bind(slow_path->GetExitLabel()); in VisitSystemArrayCopyChar()
2925 __ Cmp(src, dest); in VisitSystemArrayCopy()
2926 __ B(intrinsic_slow_path->GetEntryLabel(), eq); in VisitSystemArrayCopy()
2933 __ Cmp(src, dest); in VisitSystemArrayCopy()
2934 __ B(&conditions_on_positions_validated, ne); in VisitSystemArrayCopy()
2936 __ Cmp(WRegisterFrom(dest_pos), src_pos_constant); in VisitSystemArrayCopy()
2937 __ B(intrinsic_slow_path->GetEntryLabel(), gt); in VisitSystemArrayCopy()
2941 __ Cmp(src, dest); in VisitSystemArrayCopy()
2942 __ B(&conditions_on_positions_validated, ne); in VisitSystemArrayCopy()
2944 __ Cmp(RegisterFrom(src_pos, invoke->InputAt(1)->GetType()), in VisitSystemArrayCopy()
2946 __ B(intrinsic_slow_path->GetEntryLabel(), lt); in VisitSystemArrayCopy()
2949 __ Bind(&conditions_on_positions_validated); in VisitSystemArrayCopy()
2953 __ Cbz(src, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2958 __ Cbz(dest, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2968 __ Cmp(WRegisterFrom(length), kSystemArrayCopyThreshold); in VisitSystemArrayCopy()
2969 __ B(intrinsic_slow_path->GetEntryLabel(), hs); in VisitSystemArrayCopy()
3026 __ Cbz(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3030 __ Ldrh(temp1, HeapOperand(temp1, primitive_offset)); in VisitSystemArrayCopy()
3032 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3060 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3064 __ Ldrh(temp2, HeapOperand(temp2, primitive_offset)); in VisitSystemArrayCopy()
3066 __ Cbnz(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3080 __ Cmp(temp1, temp2); in VisitSystemArrayCopy()
3084 __ B(&do_copy, eq); in VisitSystemArrayCopy()
3098 __ Ldr(temp1, HeapOperand(temp1, super_offset)); in VisitSystemArrayCopy()
3099 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3100 __ Bind(&do_copy); in VisitSystemArrayCopy()
3102 __ B(intrinsic_slow_path->GetEntryLabel(), ne); in VisitSystemArrayCopy()
3108 __ Ldr(temp1, MemOperand(dest, class_offset)); in VisitSystemArrayCopy()
3110 __ Ldr(temp2, MemOperand(src, class_offset)); in VisitSystemArrayCopy()
3124 __ Ldr(temp3, HeapOperand(temp1, component_offset)); in VisitSystemArrayCopy()
3125 __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3128 __ Ldrh(temp3, HeapOperand(temp3, primitive_offset)); in VisitSystemArrayCopy()
3130 __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3136 __ Ldr(temp3, HeapOperand(temp2, component_offset)); in VisitSystemArrayCopy()
3137 __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3140 __ Ldrh(temp3, HeapOperand(temp3, primitive_offset)); in VisitSystemArrayCopy()
3142 __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3145 __ Cmp(temp1, temp2); in VisitSystemArrayCopy()
3149 __ B(&do_copy, eq); in VisitSystemArrayCopy()
3154 __ Ldr(temp1, HeapOperand(temp1, component_offset)); in VisitSystemArrayCopy()
3157 __ Ldr(temp1, HeapOperand(temp1, super_offset)); in VisitSystemArrayCopy()
3159 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3160 __ Bind(&do_copy); in VisitSystemArrayCopy()
3162 __ B(intrinsic_slow_path->GetEntryLabel(), ne); in VisitSystemArrayCopy()
3185 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3190 __ Ldr(temp1, HeapOperand(src.W(), class_offset)); in VisitSystemArrayCopy()
3193 __ Ldr(temp2, HeapOperand(temp1, component_offset)); in VisitSystemArrayCopy()
3194 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3198 __ Ldrh(temp2, HeapOperand(temp2, primitive_offset)); in VisitSystemArrayCopy()
3200 __ Cbnz(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3215 __ Cbz(WRegisterFrom(length), &done); in VisitSystemArrayCopy()
3255 __ Ldr(tmp, HeapOperand(src.W(), monitor_offset)); in VisitSystemArrayCopy()
3265 __ Add(src.X(), src.X(), Operand(tmp.X(), LSR, 32)); in VisitSystemArrayCopy()
3293 __ Tbnz(tmp, LockWord::kReadBarrierStateShift, read_barrier_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3299 __ Bind(&loop); in VisitSystemArrayCopy()
3300 __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex)); in VisitSystemArrayCopy()
3301 __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex)); in VisitSystemArrayCopy()
3302 __ Cmp(src_curr_addr, src_stop_addr); in VisitSystemArrayCopy()
3303 __ B(&loop, ne); in VisitSystemArrayCopy()
3305 __ Bind(read_barrier_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
3324 __ Bind(&loop); in VisitSystemArrayCopy()
3327 __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex)); in VisitSystemArrayCopy()
3328 __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex)); in VisitSystemArrayCopy()
3330 __ Cmp(src_curr_addr, src_stop_addr); in VisitSystemArrayCopy()
3331 __ B(&loop, ne); in VisitSystemArrayCopy()
3333 __ Bind(&done); in VisitSystemArrayCopy()
3340 __ Bind(intrinsic_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
3362 __ Eor(out, out, infinity); in GenIsInfinite()
3365 __ Tst(out, tst_mask); in GenIsInfinite()
3366 __ Cset(out, eq); in GenIsInfinite()
3421 __ Mov(temp.W(), value); in VisitIntegerValueOf()
3422 __ Str(temp.W(), HeapOperand(out.W(), info.value_offset)); in VisitIntegerValueOf()
3430 __ Add(out.W(), in.W(), -info.low); in VisitIntegerValueOf()
3431 __ Cmp(out.W(), info.length); in VisitIntegerValueOf()
3433 __ B(&allocate, hs); in VisitIntegerValueOf()
3440 __ B(&done); in VisitIntegerValueOf()
3441 __ Bind(&allocate); in VisitIntegerValueOf()
3444 __ Str(in.W(), HeapOperand(out.W(), info.value_offset)); in VisitIntegerValueOf()
3447 __ Bind(&done); in VisitIntegerValueOf()
3473 __ Ldr(temp, in VisitReferenceGetReferent()
3476 __ Cbnz(temp, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
3490 __ Ldrh(temp, HeapOperand(temp, disable_intrinsic_offset.Uint32Value())); in VisitReferenceGetReferent()
3491 __ Cbnz(temp, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
3510 __ Bind(slow_path->GetExitLabel()); in VisitReferenceGetReferent()
3534 __ Cmp(tmp, other); in VisitReferenceRefersTo()
3542 __ Cbz(mr, &calculate_result); in VisitReferenceRefersTo()
3544 __ B(&calculate_result, eq); // ZF set if taken. in VisitReferenceRefersTo()
3547 __ Cbz(tmp, &calculate_result); // ZF clear if taken. in VisitReferenceRefersTo()
3555 __ Ldr(tmp, HeapOperand(tmp, monitor_offset)); in VisitReferenceRefersTo()
3556 __ Cmp(tmp, Operand(0xc0000000)); in VisitReferenceRefersTo()
3557 __ B(&calculate_result, lo); // ZF clear if taken. in VisitReferenceRefersTo()
3560 __ Cmp(other, Operand(tmp, LSL, LockWord::kForwardingAddressShift)); in VisitReferenceRefersTo()
3562 __ Bind(&calculate_result); in VisitReferenceRefersTo()
3566 __ Cset(out, eq); in VisitReferenceRefersTo()
3581 __ Add(temp, tr, Thread::InterruptedOffset<kArm64PointerSize>().Int32Value()); in VisitThreadInterrupted()
3582 __ Ldar(out.W(), MemOperand(temp)); in VisitThreadInterrupted()
3585 __ Cbz(out.W(), &done); in VisitThreadInterrupted()
3586 __ Stlr(wzr, MemOperand(temp)); in VisitThreadInterrupted()
3587 __ Bind(&done); in VisitThreadInterrupted()
3631 __ Mvn(tmp, crc); in VisitCRC32Update()
3632 __ Crc32b(tmp, tmp, val); in VisitCRC32Update()
3633 __ Mvn(out, tmp); in VisitCRC32Update()
3673 __ Mvn(out, crc); in GenerateCodeForCalculationCRC32ValueOfBytes()
3674 __ Mov(len, length); in GenerateCodeForCalculationCRC32ValueOfBytes()
3676 __ Tbz(ptr, 0, &aligned2); in GenerateCodeForCalculationCRC32ValueOfBytes()
3677 __ Subs(len, len, 1); in GenerateCodeForCalculationCRC32ValueOfBytes()
3678 __ B(&done, lo); in GenerateCodeForCalculationCRC32ValueOfBytes()
3679 __ Ldrb(array_elem, MemOperand(ptr, 1, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3680 __ Crc32b(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3682 __ Bind(&aligned2); in GenerateCodeForCalculationCRC32ValueOfBytes()
3683 __ Tbz(ptr, 1, &aligned4); in GenerateCodeForCalculationCRC32ValueOfBytes()
3684 __ Subs(len, len, 2); in GenerateCodeForCalculationCRC32ValueOfBytes()
3685 __ B(&process_1byte, lo); in GenerateCodeForCalculationCRC32ValueOfBytes()
3686 __ Ldrh(array_elem, MemOperand(ptr, 2, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3687 __ Crc32h(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3689 __ Bind(&aligned4); in GenerateCodeForCalculationCRC32ValueOfBytes()
3690 __ Tbz(ptr, 2, &aligned8); in GenerateCodeForCalculationCRC32ValueOfBytes()
3691 __ Subs(len, len, 4); in GenerateCodeForCalculationCRC32ValueOfBytes()
3692 __ B(&process_2bytes, lo); in GenerateCodeForCalculationCRC32ValueOfBytes()
3693 __ Ldr(array_elem, MemOperand(ptr, 4, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3694 __ Crc32w(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3696 __ Bind(&aligned8); in GenerateCodeForCalculationCRC32ValueOfBytes()
3697 __ Subs(len, len, 8); in GenerateCodeForCalculationCRC32ValueOfBytes()
3699 __ B(&process_4bytes, lo); in GenerateCodeForCalculationCRC32ValueOfBytes()
3702 __ Bind(&loop); in GenerateCodeForCalculationCRC32ValueOfBytes()
3703 __ Ldr(array_elem.X(), MemOperand(ptr, 8, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3704 __ Subs(len, len, 8); in GenerateCodeForCalculationCRC32ValueOfBytes()
3705 __ Crc32x(out, out, array_elem.X()); in GenerateCodeForCalculationCRC32ValueOfBytes()
3707 __ B(&loop, hs); in GenerateCodeForCalculationCRC32ValueOfBytes()
3718 __ Bind(&process_4bytes); in GenerateCodeForCalculationCRC32ValueOfBytes()
3720 __ Tbz(len, 2, &process_2bytes); in GenerateCodeForCalculationCRC32ValueOfBytes()
3721 __ Ldr(array_elem, MemOperand(ptr, 4, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3722 __ Crc32w(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3724 __ Bind(&process_2bytes); in GenerateCodeForCalculationCRC32ValueOfBytes()
3726 __ Tbz(len, 1, &process_1byte); in GenerateCodeForCalculationCRC32ValueOfBytes()
3727 __ Ldrh(array_elem, MemOperand(ptr, 2, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3728 __ Crc32h(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3730 __ Bind(&process_1byte); in GenerateCodeForCalculationCRC32ValueOfBytes()
3732 __ Tbz(len, 0, &done); in GenerateCodeForCalculationCRC32ValueOfBytes()
3733 __ Ldrb(array_elem, MemOperand(ptr)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3734 __ Crc32b(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3736 __ Bind(&done); in GenerateCodeForCalculationCRC32ValueOfBytes()
3737 __ Mvn(out, out); in GenerateCodeForCalculationCRC32ValueOfBytes()
3776 __ Cmp(length, kCRC32UpdateBytesThreshold); in VisitCRC32UpdateBytes()
3777 __ B(slow_path->GetEntryLabel(), hi); in VisitCRC32UpdateBytes()
3786 __ Add(ptr, array, array_data_offset + offset_value); in VisitCRC32UpdateBytes()
3788 __ Add(ptr, array, array_data_offset); in VisitCRC32UpdateBytes()
3789 __ Add(ptr, ptr, XRegisterFrom(offset)); in VisitCRC32UpdateBytes()
3797 __ Bind(slow_path->GetExitLabel()); in VisitCRC32UpdateBytes()
3835 __ Add(ptr, addr, XRegisterFrom(locations->InAt(2))); in VisitCRC32UpdateByteBuffer()
3862 __ Fmov(half, bits); // ARMv8.2 in VisitFP16ToFloat()
3863 __ Fcvt(out, half); in VisitFP16ToFloat()
3885 __ Fcvt(half, in); in VisitFP16ToHalf()
3886 __ Fmov(out, half); in VisitFP16ToHalf()
3887 __ Sxth(out, out); // sign extend due to returning a short type. in VisitFP16ToHalf()
3900 __ Fmov(half, WRegisterFrom(locations->InAt(0))); in GenerateFP16Round()
3902 __ Fmov(out, half); in GenerateFP16Round()
3903 __ Sxth(out, out); in GenerateFP16Round()
3917 __ Frintm(out, in); // Round towards Minus infinity in VisitFP16Floor()
3933 __ Frintp(out, in); // Round towards Plus infinity in VisitFP16Ceil()
3949 __ Frintn(out, in); // Round to nearest, with ties to even in VisitFP16Rint()
3978 __ Fmov(half0, WRegisterFrom(locations->InAt(0))); in GenerateFP16Compare()
3979 __ Fmov(half1, WRegisterFrom(locations->InAt(1))); in GenerateFP16Compare()
3988 __ Fcmp(in0, in1); in GenerateFP16Compare()
3989 __ Cset(out, cond); in GenerateFP16Compare()
4051 __ Fcmp(in0, in1); in VisitFP16Compare()
4052 __ B(eq, &equal); // in0==in1 or +0 -0 case. in VisitFP16Compare()
4053 __ B(vc, &normal); // in0 and in1 are ordered (not NaN). in VisitFP16Compare()
4060 __ Fcmp(in0, 0.0); in VisitFP16Compare()
4061 __ Mov(out, -1); in VisitFP16Compare()
4062 __ B(vc, &end); // in0 != NaN => out = -1. in VisitFP16Compare()
4063 __ Fcmp(in1, 0.0); in VisitFP16Compare()
4064 __ Cset(out, vc); // if in1 != NaN => out = 1, otherwise both are NaNs => out = 0. in VisitFP16Compare()
4065 __ B(&end); in VisitFP16Compare()
4068 __ Bind(&equal); in VisitFP16Compare()
4074 __ Fcmp(in1.S(), in0.S()); in VisitFP16Compare()
4076 __ Bind(&normal); in VisitFP16Compare()
4077 __ Cset(out, gt); // if in0 > in1 => out = 1, otherwise out = 0. in VisitFP16Compare()
4079 __ Csinv(out, out, wzr, pl); // if in0 >= in1 out=out, otherwise out=-1. in VisitFP16Compare()
4081 __ Bind(&end); in VisitFP16Compare()
4115 __ Fmov(half0, in0); in GenerateFP16MinMax()
4116 __ Fmov(half1, in1); in GenerateFP16MinMax()
4117 __ Fcmp(half0, half1); in GenerateFP16MinMax()
4118 __ B(eq, &equal); // half0 = half1 or +0/-0 case. in GenerateFP16MinMax()
4119 __ Csel(out, in0, in1, cond); // if half0 <cond> half1 => out = in0, otherwise out = in1. in GenerateFP16MinMax()
4120 __ B(vc, &end); // None of the inputs were NaN. in GenerateFP16MinMax()
4123 __ Mov(out, kFP16NaN); // out=NaN. in GenerateFP16MinMax()
4124 __ B(&end); in GenerateFP16MinMax()
4127 __ Bind(&equal); in GenerateFP16MinMax()
4132 __ Fcmp(half1.S(), half0.S()); in GenerateFP16MinMax()
4134 __ Csel(out, in0, in1, cond); // if half0 <cond> half1 => out = in0, otherwise out = in1. in GenerateFP16MinMax()
4136 __ Bind(&end); in GenerateFP16MinMax()
4173 __ Cbz(divisor, slow_path->GetEntryLabel()); in GenerateDivideUnsigned()
4175 __ Udiv(out, dividend, divisor); in GenerateDivideUnsigned()
4177 __ Bind(slow_path->GetExitLabel()); in GenerateDivideUnsigned()
4210 __ Smulh(out, x, y); in VisitMathMultiplyHigh()
4221 __ Fmadd(out, n, m, a); in GenerateMathFma()
4315 __ Cbz(object, &success); in GenerateSubTypeObjectCheckNoReadBarrier()
4321 __ Ldr(temp, HeapOperand(object, class_offset.Int32Value())); in GenerateSubTypeObjectCheckNoReadBarrier()
4324 __ Bind(&loop); in GenerateSubTypeObjectCheckNoReadBarrier()
4325 __ Cmp(type, temp); in GenerateSubTypeObjectCheckNoReadBarrier()
4326 __ B(&success, eq); in GenerateSubTypeObjectCheckNoReadBarrier()
4327 __ Ldr(temp, HeapOperand(temp, super_class_offset.Int32Value())); in GenerateSubTypeObjectCheckNoReadBarrier()
4329 __ Cbz(temp, slow_path->GetEntryLabel()); in GenerateSubTypeObjectCheckNoReadBarrier()
4330 __ B(&loop); in GenerateSubTypeObjectCheckNoReadBarrier()
4331 __ Bind(&success); in GenerateSubTypeObjectCheckNoReadBarrier()
4360 __ Ldp(var_type_no_rb, temp2, HeapOperand(varhandle, var_type_offset.Int32Value())); in GenerateVarHandleAccessModeAndVarTypeChecks()
4362 __ Tbz(temp2, static_cast<uint32_t>(access_mode), slow_path->GetEntryLabel()); in GenerateVarHandleAccessModeAndVarTypeChecks()
4363 __ Ldrh(temp2, HeapOperand(var_type_no_rb, primitive_type_offset.Int32Value())); in GenerateVarHandleAccessModeAndVarTypeChecks()
4366 __ Cbnz(temp2, slow_path->GetEntryLabel()); in GenerateVarHandleAccessModeAndVarTypeChecks()
4368 __ Cmp(temp2, static_cast<uint16_t>(primitive_type)); in GenerateVarHandleAccessModeAndVarTypeChecks()
4369 __ B(slow_path->GetEntryLabel(), ne); in GenerateVarHandleAccessModeAndVarTypeChecks()
4405 __ Ldr(temp, HeapOperand(varhandle, coordinate_type0_offset.Int32Value())); in GenerateVarHandleStaticFieldCheck()
4406 __ Cbnz(temp, slow_path->GetEntryLabel()); in GenerateVarHandleStaticFieldCheck()
4422 __ Cbz(object, slow_path->GetEntryLabel()); in GenerateVarHandleInstanceFieldChecks()
4434 __ Ldp(temp, temp2, HeapOperand(varhandle, coordinate_type0_offset.Int32Value())); in GenerateVarHandleInstanceFieldChecks()
4437 __ Cbnz(temp2, slow_path->GetEntryLabel()); in GenerateVarHandleInstanceFieldChecks()
4468 __ Cbz(object, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
4479 __ Ldp(temp, temp2, HeapOperand(varhandle, coordinate_type0_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4482 __ Cbz(temp2, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
4494 __ Ldr(temp2, HeapOperand(object, class_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4496 __ Cmp(temp, temp2); in GenerateVarHandleArrayChecks()
4497 __ B(slow_path->GetEntryLabel(), ne); in GenerateVarHandleArrayChecks()
4502 __ Ldr(temp2, HeapOperand(temp, component_type_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4504 __ Cbz(temp2, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
4507 __ Ldrh(temp2, HeapOperand(temp2, primitive_type_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4510 __ Cbnz(temp2, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
4522 __ Cmp(temp2, static_cast<uint16_t>(primitive_type)); in GenerateVarHandleArrayChecks()
4523 __ B(slow_path_label, ne); in GenerateVarHandleArrayChecks()
4527 __ Ldr(temp, HeapOperand(object, array_length_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4528 __ Cmp(index, temp); in GenerateVarHandleArrayChecks()
4529 __ B(slow_path->GetEntryLabel(), hs); in GenerateVarHandleArrayChecks()
4612 __ Mov(target.offset, target_field->GetOffset().Uint32Value()); in GenerateVarHandleTarget()
4624 __ Ldr(method.X(), HeapOperand(varhandle, art_field_offset.Int32Value())); in GenerateVarHandleTarget()
4625 __ Ldr(target.offset, MemOperand(method.X(), offset_offset.Int32Value())); in GenerateVarHandleTarget()
4646 __ Lsl(shifted_index, index, size_shift); in GenerateVarHandleTarget()
4648 __ Add(target.offset, shifted_index, data_offset.Int32Value()); in GenerateVarHandleTarget()
4739 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleGet()
4753 __ Add(tmp_ptr, target.object.X(), target.offset.X()); in GenerateVarHandleGet()
4797 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleGet()
4858 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleSet()
4874 __ Mov(temp, value.W()); in GenerateVarHandleSet()
4885 __ Fmov(temp, source.Is64Bits() ? source.D() : source.S()); in GenerateVarHandleSet()
4906 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleSet()
5018 __ Fmov(reg, cpu_reg.S()); in MoveToTempIfFpRegister()
5023 __ Fmov(reg, cpu_reg.D()); in MoveToTempIfFpRegister()
5056 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleCompareAndSetOrExchange()
5074 __ Add(tmp_ptr, target.object.X(), target.offset.X()); in GenerateVarHandleCompareAndSetOrExchange()
5088 __ Uxth(temp, expected_reg); in GenerateVarHandleCompareAndSetOrExchange()
5093 __ Uxtb(temp, expected_reg); in GenerateVarHandleCompareAndSetOrExchange()
5192 __ Bind(exit_loop); in GenerateVarHandleCompareAndSetOrExchange()
5196 __ Cset(out.W(), eq); in GenerateVarHandleCompareAndSetOrExchange()
5201 __ Csel(out.W(), store_result, wzr, eq); in GenerateVarHandleCompareAndSetOrExchange()
5207 __ Fmov((value_type == DataType::Type::kFloat64) ? out.D() : out.S(), old_value); in GenerateVarHandleCompareAndSetOrExchange()
5209 __ Sxtb(out.W(), old_value); in GenerateVarHandleCompareAndSetOrExchange()
5211 __ Sxth(out.W(), old_value); in GenerateVarHandleCompareAndSetOrExchange()
5216 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleCompareAndSetOrExchange()
5359 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleGetAndUpdate()
5379 __ Add(tmp_ptr, target.object.X(), target.offset.X()); in GenerateVarHandleGetAndUpdate()
5440 __ Sxth(out.W(), old_value.W()); in GenerateVarHandleGetAndUpdate()
5446 __ Fmov(out.D(), old_value.X()); in GenerateVarHandleGetAndUpdate()
5448 __ Fmov(out.S(), old_value.W()); in GenerateVarHandleGetAndUpdate()
5450 __ Sxtb(out.W(), old_value.W()); in GenerateVarHandleGetAndUpdate()
5452 __ Sxth(out.W(), old_value.W()); in GenerateVarHandleGetAndUpdate()
5469 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleGetAndUpdate()
5613 __ Bind(GetByteArrayViewCheckLabel()); in EmitByteArrayViewCode()
5624 __ Ldr(temp, HeapOperand(varhandle, class_offset.Int32Value())); in EmitByteArrayViewCode()
5627 __ Cmp(temp, temp2); in EmitByteArrayViewCode()
5628 __ B(GetEntryLabel(), ne); in EmitByteArrayViewCode()
5631 __ Ldr(temp, HeapOperand(object, array_length_offset.Int32Value())); in EmitByteArrayViewCode()
5632 __ Subs(temp, temp, index); in EmitByteArrayViewCode()
5633 __ Ccmp(temp, size, NoFlag, hs); // If SUBS yields LO (C=false), keep the C flag clear. in EmitByteArrayViewCode()
5634 __ B(GetEntryLabel(), lo); in EmitByteArrayViewCode()
5637 __ Add(target.offset, index, data_offset.Int32Value()); in EmitByteArrayViewCode()
5642 __ Tbnz(target.offset, 0, GetEntryLabel()); in EmitByteArrayViewCode()
5644 __ Tst(target.offset, size - 1u); in EmitByteArrayViewCode()
5645 __ B(GetEntryLabel(), ne); in EmitByteArrayViewCode()
5653 __ B(GetNativeByteOrderLabel()); in EmitByteArrayViewCode()
5656 __ Ldr(temp, HeapOperand(varhandle, native_byte_order_offset.Int32Value())); in EmitByteArrayViewCode()
5657 __ Cbnz(temp, GetNativeByteOrderLabel()); in EmitByteArrayViewCode()
5677 __ B(GetExitLabel()); in EmitByteArrayViewCode()
5719 #undef __