Lines Matching refs:__
88 #define __ codegen->GetVIXLAssembler()-> macro
116 __ Bind(GetEntryLabel()); in EmitNativeCode()
118 __ Bind(&slow_copy_loop); in EmitNativeCode()
119 __ Ldr(tmp_reg, MemOperand(src_curr_addr, element_size, PostIndex)); in EmitNativeCode()
146 __ Str(tmp_reg, MemOperand(dst_curr_addr, element_size, PostIndex)); in EmitNativeCode()
147 __ Cmp(src_curr_addr, src_stop_addr); in EmitNativeCode()
148 __ B(&slow_copy_loop, ne); in EmitNativeCode()
149 __ B(GetExitLabel()); in EmitNativeCode()
159 #undef __
170 #define __ masm-> macro
189 __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output), in MoveFPToInt()
196 __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output), in MoveIntToFP()
258 __ Rev16(out.W(), in.W()); in GenerateReverseBytes()
261 __ Rev16(out.W(), in.W()); in GenerateReverseBytes()
262 __ Sxth(out.W(), out.W()); in GenerateReverseBytes()
265 __ Rev(out.W(), in.W()); in GenerateReverseBytes()
268 __ Rev(out.X(), in.X()); in GenerateReverseBytes()
271 __ Rev(in.W(), in.W()); // Note: Clobbers `in`. in GenerateReverseBytes()
272 __ Fmov(out.S(), in.W()); in GenerateReverseBytes()
275 __ Rev(in.X(), in.X()); // Note: Clobbers `in`. in GenerateReverseBytes()
276 __ Fmov(out.D(), in.X()); in GenerateReverseBytes()
324 __ Clz(RegisterFrom(out, type), RegisterFrom(in, type)); in GenNumberOfLeadingZeros()
351 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type)); in GenNumberOfTrailingZeros()
352 __ Clz(RegisterFrom(out, type), RegisterFrom(out, type)); in GenNumberOfTrailingZeros()
379 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type)); in GenReverse()
409 __ Fmov(fpr, src); in GenBitCount()
410 __ Cnt(fpr.V8B(), fpr.V8B()); in GenBitCount()
411 __ Addv(fpr.B(), fpr.V8B()); in GenBitCount()
412 __ Fmov(dst, fpr); in GenBitCount()
442 __ Clz(temp, src); in GenHighestOneBit()
443 __ Mov(dst, UINT64_C(1) << high_bit); // MOV (bitmask immediate) in GenHighestOneBit()
444 __ Bic(dst, dst, Operand(temp, LSL, high_bit - clz_high_bit)); // Clear dst if src was 0. in GenHighestOneBit()
445 __ Lsr(dst, dst, temp); in GenHighestOneBit()
473 __ Neg(temp, src); in GenLowestOneBit()
474 __ And(dst, temp, src); in GenLowestOneBit()
507 __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); in VisitMathSqrt()
517 __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); in VisitMathCeil()
527 __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); in VisitMathFloor()
537 __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); in VisitMathRint()
569 __ Fcvtas(out_reg, in_reg); in GenMathRound()
572 __ Tbz(out_reg, out_reg.GetSizeInBits() - 1, &done); in GenMathRound()
577 __ Frinta(tmp_fp, in_reg); in GenMathRound()
578 __ Fsub(tmp_fp, in_reg, tmp_fp); in GenMathRound()
579 __ Fcmp(tmp_fp, 0.5); in GenMathRound()
580 __ Cinc(out_reg, out_reg, eq); in GenMathRound()
582 __ Bind(&done); in GenMathRound()
607 __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekByte()
617 __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekIntNative()
627 __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekLongNative()
637 __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekShortNative()
654 __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeByte()
664 __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeIntNative()
674 __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeLongNative()
684 __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeShortNative()
719 __ Add(temp, base, offset.W()); // Offset should not exceed 32 bits. in GenUnsafeGet()
862 __ Mov(temp.W(), value.W()); in GenUnsafePut()
977 __ Ldaxrb(old_value, MemOperand(ptr)); in EmitLoadExclusive()
979 __ Ldxrb(old_value, MemOperand(ptr)); in EmitLoadExclusive()
985 __ Ldaxrh(old_value, MemOperand(ptr)); in EmitLoadExclusive()
987 __ Ldxrh(old_value, MemOperand(ptr)); in EmitLoadExclusive()
994 __ Ldaxr(old_value, MemOperand(ptr)); in EmitLoadExclusive()
996 __ Ldxr(old_value, MemOperand(ptr)); in EmitLoadExclusive()
1005 __ Sxtb(old_value, old_value); in EmitLoadExclusive()
1008 __ Sxth(old_value, old_value); in EmitLoadExclusive()
1034 __ Stlxrb(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1036 __ Stxrb(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1042 __ Stlxrh(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1044 __ Stxrh(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1051 __ Stlxr(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1053 __ Stxr(store_result, new_value, MemOperand(ptr)); in EmitStoreExclusive()
1112 __ Bind(&loop_head); in GenerateCompareAndSet()
1115 __ Cmp(old_value, expected); in GenerateCompareAndSet()
1117 __ Ccmp(old_value, expected2, ZFlag, ne); in GenerateCompareAndSet()
1122 __ B(cmp_failure, ne); in GenerateCompareAndSet()
1125 __ Cbnz(store_result, &loop_head); in GenerateCompareAndSet()
1128 __ Eor(store_result, store_result, 1); in GenerateCompareAndSet()
1186 __ Bind(GetEntryLabel()); in EmitNativeCode()
1194 __ B(mark_old_value_slow_path_->GetEntryLabel()); in EmitNativeCode()
1195 __ Bind(mark_old_value_slow_path_->GetExitLabel()); in EmitNativeCode()
1197 __ Cmp(old_value_temp_, expected_); in EmitNativeCode()
1200 __ Csel(old_value_, old_value_temp_, old_value_, ne); in EmitNativeCode()
1202 __ B(GetExitLabel(), ne); // If taken, Z=false indicates failure. in EmitNativeCode()
1216 __ Add(tmp_ptr, base_.X(), Operand(offset_)); in EmitNativeCode()
1233 __ Mov(old_value_, expected_); in EmitNativeCode()
1239 __ B(GetExitLabel()); in EmitNativeCode()
1242 __ Bind(&mark_old_value); in EmitNativeCode()
1251 __ B(update_old_value_slow_path_->GetEntryLabel()); in EmitNativeCode()
1252 __ Bind(update_old_value_slow_path_->GetExitLabel()); in EmitNativeCode()
1254 __ B(GetExitLabel()); in EmitNativeCode()
1326 __ Add(tmp_ptr, base.X(), Operand(offset)); in GenUnsafeCas()
1338 __ Bind(exit_loop); in GenUnsafeCas()
1339 __ Cset(out, eq); in GenUnsafeCas()
1436 __ Bind(&loop_label); in GenerateGetAndUpdate()
1451 __ Fmov(old_value_vreg, old_value_reg); in GenerateGetAndUpdate()
1452 __ Fadd(sum, old_value_vreg, arg.IsD() ? arg.D() : arg.S()); in GenerateGetAndUpdate()
1453 __ Fmov(new_value, sum); in GenerateGetAndUpdate()
1455 __ Add(new_value, old_value_reg, arg.IsX() ? arg.X() : arg.W()); in GenerateGetAndUpdate()
1462 __ And(new_value, old_value_reg, arg.IsX() ? arg.X() : arg.W()); in GenerateGetAndUpdate()
1465 __ Orr(new_value, old_value_reg, arg.IsX() ? arg.X() : arg.W()); in GenerateGetAndUpdate()
1468 __ Eor(new_value, old_value_reg, arg.IsX() ? arg.X() : arg.W()); in GenerateGetAndUpdate()
1472 __ Cbnz(store_result, &loop_label); in GenerateGetAndUpdate()
1530 __ Cbz(arg, slow_path->GetEntryLabel()); in VisitStringCompareTo()
1534 __ Subs(out, str, arg); in VisitStringCompareTo()
1535 __ B(&end, eq); in VisitStringCompareTo()
1539 __ Ldr(temp3, HeapOperand(str, count_offset)); in VisitStringCompareTo()
1540 __ Ldr(temp2, HeapOperand(arg, count_offset)); in VisitStringCompareTo()
1542 __ Lsr(temp0, temp3, 1u); in VisitStringCompareTo()
1543 __ Lsr(temp1, temp2, 1u); in VisitStringCompareTo()
1546 __ Ldr(temp0, HeapOperand(str, count_offset)); in VisitStringCompareTo()
1547 __ Ldr(temp1, HeapOperand(arg, count_offset)); in VisitStringCompareTo()
1550 __ Subs(out, temp0, temp1); in VisitStringCompareTo()
1552 __ Csel(temp0, temp1, temp0, ge); in VisitStringCompareTo()
1554 __ Cbz(temp0, &end); in VisitStringCompareTo()
1558 __ Eor(temp2, temp2, Operand(temp3)); in VisitStringCompareTo()
1561 __ Ands(temp3.W(), temp3.W(), Operand(1)); in VisitStringCompareTo()
1562 __ Tbnz(temp2, 0, &different_compression); // Does not use flags. in VisitStringCompareTo()
1565 __ Mov(temp1, value_offset); in VisitStringCompareTo()
1569 __ Lsl(temp0, temp0, temp3); in VisitStringCompareTo()
1586 __ Bind(&loop); in VisitStringCompareTo()
1587 __ Ldr(temp4, MemOperand(str.X(), temp1.X())); in VisitStringCompareTo()
1588 __ Ldr(temp2, MemOperand(arg.X(), temp1.X())); in VisitStringCompareTo()
1589 __ Cmp(temp4, temp2); in VisitStringCompareTo()
1590 __ B(ne, &find_char_diff); in VisitStringCompareTo()
1591 __ Add(temp1, temp1, char_size * 4); in VisitStringCompareTo()
1593 __ Subs(temp0, temp0, (mirror::kUseStringCompression) ? 8 : 4); in VisitStringCompareTo()
1594 __ B(&loop, hi); in VisitStringCompareTo()
1595 __ B(&end); in VisitStringCompareTo()
1601 __ Bind(&find_char_diff); in VisitStringCompareTo()
1603 __ Eor(temp1, temp2, temp4); in VisitStringCompareTo()
1604 __ Rbit(temp1, temp1); in VisitStringCompareTo()
1605 __ Clz(temp1, temp1); in VisitStringCompareTo()
1613 __ Cmp(temp0, Operand(temp1.W(), LSR, (mirror::kUseStringCompression) ? 3 : 4)); in VisitStringCompareTo()
1614 __ B(ls, &end); in VisitStringCompareTo()
1618 __ Bic(temp1, temp1, 0x7); in VisitStringCompareTo()
1619 __ Bic(temp1, temp1, Operand(temp3.X(), LSL, 3u)); in VisitStringCompareTo()
1621 __ Bic(temp1, temp1, 0xf); in VisitStringCompareTo()
1623 __ Lsr(temp2, temp2, temp1); in VisitStringCompareTo()
1624 __ Lsr(temp4, temp4, temp1); in VisitStringCompareTo()
1627 __ Uxtb(temp1, temp4); in VisitStringCompareTo()
1628 __ Sub(out, temp1.W(), Operand(temp2.W(), UXTB)); in VisitStringCompareTo()
1629 __ Tbz(temp3, 0u, &end); // If actually compressed, we're done. in VisitStringCompareTo()
1631 __ Uxth(temp4, temp4); in VisitStringCompareTo()
1632 __ Sub(out, temp4.W(), Operand(temp2.W(), UXTH)); in VisitStringCompareTo()
1635 __ B(&end); in VisitStringCompareTo()
1636 __ Bind(&different_compression); in VisitStringCompareTo()
1648 __ Csel(temp1, str, arg, eq); // Pointer to the compressed string. in VisitStringCompareTo()
1649 __ Csel(temp2, str, arg, ne); // Pointer to the uncompressed string. in VisitStringCompareTo()
1655 __ Lsl(temp0, temp0, 1u); in VisitStringCompareTo()
1658 __ Add(temp1, temp1, Operand(value_offset)); in VisitStringCompareTo()
1659 __ Add(temp2, temp2, Operand(value_offset)); in VisitStringCompareTo()
1662 __ Sub(temp0, temp0, Operand(temp3)); in VisitStringCompareTo()
1667 __ Bind(&different_compression_loop); in VisitStringCompareTo()
1668 __ Ldrb(temp4, MemOperand(temp1.X(), c_char_size, PostIndex)); in VisitStringCompareTo()
1669 __ Ldrh(temp3, MemOperand(temp2.X(), char_size, PostIndex)); in VisitStringCompareTo()
1670 __ Subs(temp4, temp4, Operand(temp3)); in VisitStringCompareTo()
1671 __ B(&different_compression_diff, ne); in VisitStringCompareTo()
1672 __ Subs(temp0, temp0, 2); in VisitStringCompareTo()
1673 __ B(&different_compression_loop, hi); in VisitStringCompareTo()
1674 __ B(&end); in VisitStringCompareTo()
1677 __ Bind(&different_compression_diff); in VisitStringCompareTo()
1678 __ Tst(temp0, Operand(1)); in VisitStringCompareTo()
1681 __ Cneg(out, temp4, ne); in VisitStringCompareTo()
1684 __ Bind(&end); in VisitStringCompareTo()
1687 __ Bind(slow_path->GetExitLabel()); in VisitStringCompareTo()
1762 __ Cbz(arg, &return_false); in VisitStringEquals()
1766 __ Cmp(str, arg); in VisitStringEquals()
1767 __ B(&return_true, eq); in VisitStringEquals()
1779 __ Ldr(temp, MemOperand(str.X(), class_offset)); in VisitStringEquals()
1781 __ Ldr(temp1, MemOperand(arg.X(), class_offset)); in VisitStringEquals()
1784 __ Cmp(temp, temp1); in VisitStringEquals()
1785 __ B(&return_false, ne); in VisitStringEquals()
1806 __ Ldr(temp, MemOperand(arg.X(), count_offset)); in VisitStringEquals()
1809 __ Cmp(temp, Operand(mirror::String::GetFlaggedCount(const_string_length, is_compressed))); in VisitStringEquals()
1811 __ B(&return_false, ne); in VisitStringEquals()
1814 __ Ldr(temp, MemOperand(str.X(), count_offset)); in VisitStringEquals()
1815 __ Ldr(temp1, MemOperand(arg.X(), count_offset)); in VisitStringEquals()
1818 __ Cmp(temp, temp1); in VisitStringEquals()
1819 __ B(&return_false, ne); in VisitStringEquals()
1839 __ Ldp(temp, temp1, MemOperand(str.X(), offset)); in VisitStringEquals()
1840 __ Ldp(temp2, out, MemOperand(arg.X(), offset)); in VisitStringEquals()
1841 __ Cmp(temp, temp2); in VisitStringEquals()
1842 __ Ccmp(temp1, out, NoFlag, eq); in VisitStringEquals()
1843 __ B(&return_false, ne); in VisitStringEquals()
1848 __ Ldr(temp, MemOperand(str.X(), offset)); in VisitStringEquals()
1849 __ Ldr(temp1, MemOperand(arg.X(), offset)); in VisitStringEquals()
1850 __ Cmp(temp, temp1); in VisitStringEquals()
1851 __ B(&return_false, ne); in VisitStringEquals()
1857 __ Cbz(temp, &return_true); in VisitStringEquals()
1862 __ And(temp1, temp, Operand(1)); // Extract compression flag. in VisitStringEquals()
1863 __ Lsr(temp, temp, 1u); // Extract length. in VisitStringEquals()
1864 __ Lsl(temp, temp, temp1); // Calculate number of bytes to compare. in VisitStringEquals()
1868 __ Mov(temp1, value_offset); in VisitStringEquals()
1873 __ Bind(&loop); in VisitStringEquals()
1874 __ Ldr(out, MemOperand(str.X(), temp1)); in VisitStringEquals()
1875 __ Ldr(temp2, MemOperand(arg.X(), temp1)); in VisitStringEquals()
1876 __ Add(temp1, temp1, Operand(sizeof(uint64_t))); in VisitStringEquals()
1877 __ Cmp(out, temp2); in VisitStringEquals()
1878 __ B(&return_false, ne); in VisitStringEquals()
1880 __ Sub(temp, temp, Operand(mirror::kUseStringCompression ? 8 : 4), SetFlags); in VisitStringEquals()
1881 __ B(&loop, hi); in VisitStringEquals()
1886 __ Bind(&return_true); in VisitStringEquals()
1887 __ Mov(out, 1); in VisitStringEquals()
1888 __ B(&end); in VisitStringEquals()
1891 __ Bind(&return_false); in VisitStringEquals()
1892 __ Mov(out, 0); in VisitStringEquals()
1893 __ Bind(&end); in VisitStringEquals()
1915 __ B(slow_path->GetEntryLabel()); in GenerateVisitStringIndexOf()
1916 __ Bind(slow_path->GetExitLabel()); in GenerateVisitStringIndexOf()
1921 __ Tst(char_reg, 0xFFFF0000); in GenerateVisitStringIndexOf()
1924 __ B(ne, slow_path->GetEntryLabel()); in GenerateVisitStringIndexOf()
1930 __ Mov(tmp_reg, 0); in GenerateVisitStringIndexOf()
1937 __ Bind(slow_path->GetExitLabel()); in GenerateVisitStringIndexOf()
1991 __ Cmp(byte_array, 0); in VisitStringNewStringFromBytes()
1995 __ B(eq, slow_path->GetEntryLabel()); in VisitStringNewStringFromBytes()
1999 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromBytes()
2036 __ Cmp(string_to_copy, 0); in VisitStringNewStringFromString()
2040 __ B(eq, slow_path->GetEntryLabel()); in VisitStringNewStringFromString()
2044 __ Bind(slow_path->GetExitLabel()); in VisitStringNewStringFromString()
2272 __ Sub(num_chr, srcEnd, srcBegin); in VisitStringGetCharsNoCheck()
2274 __ Cbz(num_chr, &done); in VisitStringGetCharsNoCheck()
2277 __ Add(dst_ptr, dstObj, Operand(data_offset)); in VisitStringGetCharsNoCheck()
2278 __ Add(dst_ptr, dst_ptr, Operand(dstBegin, LSL, 1)); in VisitStringGetCharsNoCheck()
2281 __ Add(src_ptr, srcObj, Operand(value_offset)); in VisitStringGetCharsNoCheck()
2287 __ Ldr(tmp2, MemOperand(srcObj, count_offset)); in VisitStringGetCharsNoCheck()
2288 __ Tbz(tmp2, 0, &compressed_string_preloop); in VisitStringGetCharsNoCheck()
2290 __ Add(src_ptr, src_ptr, Operand(srcBegin, LSL, 1)); in VisitStringGetCharsNoCheck()
2297 __ Subs(tmp1, num_chr, 8); in VisitStringGetCharsNoCheck()
2298 __ B(lt, &remainder); in VisitStringGetCharsNoCheck()
2301 __ Mov(num_chr, tmp1); in VisitStringGetCharsNoCheck()
2305 __ Bind(&loop); in VisitStringGetCharsNoCheck()
2306 __ Ldp(tmp1, tmp2, MemOperand(src_ptr, char_size * 8, PostIndex)); in VisitStringGetCharsNoCheck()
2307 __ Subs(num_chr, num_chr, 8); in VisitStringGetCharsNoCheck()
2308 __ Stp(tmp1, tmp2, MemOperand(dst_ptr, char_size * 8, PostIndex)); in VisitStringGetCharsNoCheck()
2309 __ B(ge, &loop); in VisitStringGetCharsNoCheck()
2311 __ Adds(num_chr, num_chr, 8); in VisitStringGetCharsNoCheck()
2312 __ B(eq, &done); in VisitStringGetCharsNoCheck()
2316 __ Bind(&remainder); in VisitStringGetCharsNoCheck()
2317 __ Ldrh(tmp1, MemOperand(src_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2318 __ Subs(num_chr, num_chr, 1); in VisitStringGetCharsNoCheck()
2319 __ Strh(tmp1, MemOperand(dst_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2320 __ B(gt, &remainder); in VisitStringGetCharsNoCheck()
2321 __ B(&done); in VisitStringGetCharsNoCheck()
2328 __ Bind(&compressed_string_preloop); in VisitStringGetCharsNoCheck()
2329 __ Add(src_ptr, src_ptr, Operand(srcBegin)); in VisitStringGetCharsNoCheck()
2332 __ Subs(tmp1, num_chr, 8); in VisitStringGetCharsNoCheck()
2333 __ B(lt, &compressed_string_remainder); in VisitStringGetCharsNoCheck()
2336 __ Mov(num_chr, tmp1); in VisitStringGetCharsNoCheck()
2340 __ Bind(&compressed_string_vector_loop); in VisitStringGetCharsNoCheck()
2341 __ Ld1(vtmp1.V8B(), MemOperand(src_ptr, c_char_size * 8, PostIndex)); in VisitStringGetCharsNoCheck()
2342 __ Subs(num_chr, num_chr, 8); in VisitStringGetCharsNoCheck()
2343 __ Uxtl(vtmp1.V8H(), vtmp1.V8B()); in VisitStringGetCharsNoCheck()
2344 __ St1(vtmp1.V8H(), MemOperand(dst_ptr, char_size * 8, PostIndex)); in VisitStringGetCharsNoCheck()
2345 __ B(ge, &compressed_string_vector_loop); in VisitStringGetCharsNoCheck()
2347 __ Adds(num_chr, num_chr, 8); in VisitStringGetCharsNoCheck()
2348 __ B(eq, &done); in VisitStringGetCharsNoCheck()
2352 __ Bind(&compressed_string_remainder); in VisitStringGetCharsNoCheck()
2353 __ Ldrb(tmp1, MemOperand(src_ptr, c_char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2354 __ Strh(tmp1, MemOperand(dst_ptr, char_size, PostIndex)); in VisitStringGetCharsNoCheck()
2355 __ Subs(num_chr, num_chr, Operand(1)); in VisitStringGetCharsNoCheck()
2356 __ B(gt, &compressed_string_remainder); in VisitStringGetCharsNoCheck()
2359 __ Bind(&done); in VisitStringGetCharsNoCheck()
2429 __ Ldr(temp, MemOperand(input, length_offset)); in CheckSystemArrayCopyPosition()
2430 __ Cmp(temp, OperandFrom(length, DataType::Type::kInt32)); in CheckSystemArrayCopyPosition()
2431 __ B(slow_path->GetEntryLabel(), lt); in CheckSystemArrayCopyPosition()
2435 __ Ldr(temp, MemOperand(input, length_offset)); in CheckSystemArrayCopyPosition()
2436 __ Subs(temp, temp, pos_const); in CheckSystemArrayCopyPosition()
2437 __ B(slow_path->GetEntryLabel(), lt); in CheckSystemArrayCopyPosition()
2440 __ Cmp(temp, OperandFrom(length, DataType::Type::kInt32)); in CheckSystemArrayCopyPosition()
2441 __ B(slow_path->GetEntryLabel(), lt); in CheckSystemArrayCopyPosition()
2445 __ Cbnz(WRegisterFrom(pos), slow_path->GetEntryLabel()); in CheckSystemArrayCopyPosition()
2449 __ Tbnz(pos_reg, pos_reg.GetSizeInBits() - 1, slow_path->GetEntryLabel()); in CheckSystemArrayCopyPosition()
2452 __ Ldr(temp, MemOperand(input, length_offset)); in CheckSystemArrayCopyPosition()
2453 __ Subs(temp, temp, pos_reg); in CheckSystemArrayCopyPosition()
2455 __ Ccmp(temp, OperandFrom(length, DataType::Type::kInt32), NFlag, ge); in CheckSystemArrayCopyPosition()
2456 __ B(slow_path->GetEntryLabel(), lt); in CheckSystemArrayCopyPosition()
2482 __ Add(src_base, src, element_size * constant + data_offset); in GenSystemArrayCopyAddresses()
2484 __ Add(src_base, src, data_offset); in GenSystemArrayCopyAddresses()
2485 __ Add(src_base, src_base, Operand(XRegisterFrom(src_pos), LSL, element_size_shift)); in GenSystemArrayCopyAddresses()
2490 __ Add(dst_base, dst, element_size * constant + data_offset); in GenSystemArrayCopyAddresses()
2492 __ Add(dst_base, dst, data_offset); in GenSystemArrayCopyAddresses()
2493 __ Add(dst_base, dst_base, Operand(XRegisterFrom(dst_pos), LSL, element_size_shift)); in GenSystemArrayCopyAddresses()
2498 __ Add(src_end, src_base, element_size * constant); in GenSystemArrayCopyAddresses()
2500 __ Add(src_end, src_base, Operand(XRegisterFrom(copy_length), LSL, element_size_shift)); in GenSystemArrayCopyAddresses()
2519 __ Cmp(src, dst); in VisitSystemArrayCopyChar()
2520 __ B(slow_path->GetEntryLabel(), eq); in VisitSystemArrayCopyChar()
2523 __ Cbz(src, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
2526 __ Cbz(dst, slow_path->GetEntryLabel()); in VisitSystemArrayCopyChar()
2532 __ Cmp(WRegisterFrom(length), kSystemArrayCopyCharThreshold); in VisitSystemArrayCopyChar()
2533 __ B(slow_path->GetEntryLabel(), hi); in VisitSystemArrayCopyChar()
2580 __ Bind(&loop); in VisitSystemArrayCopyChar()
2581 __ Cmp(src_curr_addr, src_stop_addr); in VisitSystemArrayCopyChar()
2582 __ B(&done, eq); in VisitSystemArrayCopyChar()
2583 __ Ldrh(tmp, MemOperand(src_curr_addr, char_size, PostIndex)); in VisitSystemArrayCopyChar()
2584 __ Strh(tmp, MemOperand(dst_curr_addr, char_size, PostIndex)); in VisitSystemArrayCopyChar()
2585 __ B(&loop); in VisitSystemArrayCopyChar()
2586 __ Bind(&done); in VisitSystemArrayCopyChar()
2588 __ Bind(slow_path->GetExitLabel()); in VisitSystemArrayCopyChar()
2708 __ Cmp(src, dest); in VisitSystemArrayCopy()
2709 __ B(intrinsic_slow_path->GetEntryLabel(), eq); in VisitSystemArrayCopy()
2716 __ Cmp(src, dest); in VisitSystemArrayCopy()
2717 __ B(&conditions_on_positions_validated, ne); in VisitSystemArrayCopy()
2719 __ Cmp(WRegisterFrom(dest_pos), src_pos_constant); in VisitSystemArrayCopy()
2720 __ B(intrinsic_slow_path->GetEntryLabel(), gt); in VisitSystemArrayCopy()
2724 __ Cmp(src, dest); in VisitSystemArrayCopy()
2725 __ B(&conditions_on_positions_validated, ne); in VisitSystemArrayCopy()
2727 __ Cmp(RegisterFrom(src_pos, invoke->InputAt(1)->GetType()), in VisitSystemArrayCopy()
2729 __ B(intrinsic_slow_path->GetEntryLabel(), lt); in VisitSystemArrayCopy()
2732 __ Bind(&conditions_on_positions_validated); in VisitSystemArrayCopy()
2736 __ Cbz(src, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2741 __ Cbz(dest, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2751 __ Cmp(WRegisterFrom(length), kSystemArrayCopyThreshold); in VisitSystemArrayCopy()
2752 __ B(intrinsic_slow_path->GetEntryLabel(), hs); in VisitSystemArrayCopy()
2809 __ Cbz(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2813 __ Ldrh(temp1, HeapOperand(temp1, primitive_offset)); in VisitSystemArrayCopy()
2815 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2843 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2847 __ Ldrh(temp2, HeapOperand(temp2, primitive_offset)); in VisitSystemArrayCopy()
2849 __ Cbnz(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2863 __ Cmp(temp1, temp2); in VisitSystemArrayCopy()
2867 __ B(&do_copy, eq); in VisitSystemArrayCopy()
2881 __ Ldr(temp1, HeapOperand(temp1, super_offset)); in VisitSystemArrayCopy()
2882 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2883 __ Bind(&do_copy); in VisitSystemArrayCopy()
2885 __ B(intrinsic_slow_path->GetEntryLabel(), ne); in VisitSystemArrayCopy()
2891 __ Ldr(temp1, MemOperand(dest, class_offset)); in VisitSystemArrayCopy()
2893 __ Ldr(temp2, MemOperand(src, class_offset)); in VisitSystemArrayCopy()
2907 __ Ldr(temp3, HeapOperand(temp1, component_offset)); in VisitSystemArrayCopy()
2908 __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2911 __ Ldrh(temp3, HeapOperand(temp3, primitive_offset)); in VisitSystemArrayCopy()
2913 __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2919 __ Ldr(temp3, HeapOperand(temp2, component_offset)); in VisitSystemArrayCopy()
2920 __ Cbz(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2923 __ Ldrh(temp3, HeapOperand(temp3, primitive_offset)); in VisitSystemArrayCopy()
2925 __ Cbnz(temp3, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2928 __ Cmp(temp1, temp2); in VisitSystemArrayCopy()
2932 __ B(&do_copy, eq); in VisitSystemArrayCopy()
2937 __ Ldr(temp1, HeapOperand(temp1, component_offset)); in VisitSystemArrayCopy()
2940 __ Ldr(temp1, HeapOperand(temp1, super_offset)); in VisitSystemArrayCopy()
2942 __ Cbnz(temp1, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2943 __ Bind(&do_copy); in VisitSystemArrayCopy()
2945 __ B(intrinsic_slow_path->GetEntryLabel(), ne); in VisitSystemArrayCopy()
2968 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2973 __ Ldr(temp1, HeapOperand(src.W(), class_offset)); in VisitSystemArrayCopy()
2976 __ Ldr(temp2, HeapOperand(temp1, component_offset)); in VisitSystemArrayCopy()
2977 __ Cbz(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2981 __ Ldrh(temp2, HeapOperand(temp2, primitive_offset)); in VisitSystemArrayCopy()
2983 __ Cbnz(temp2, intrinsic_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
2998 __ Cbz(WRegisterFrom(length), &done); in VisitSystemArrayCopy()
3038 __ Ldr(tmp, HeapOperand(src.W(), monitor_offset)); in VisitSystemArrayCopy()
3048 __ Add(src.X(), src.X(), Operand(tmp.X(), LSR, 32)); in VisitSystemArrayCopy()
3076 __ Tbnz(tmp, LockWord::kReadBarrierStateShift, read_barrier_slow_path->GetEntryLabel()); in VisitSystemArrayCopy()
3082 __ Bind(&loop); in VisitSystemArrayCopy()
3083 __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex)); in VisitSystemArrayCopy()
3084 __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex)); in VisitSystemArrayCopy()
3085 __ Cmp(src_curr_addr, src_stop_addr); in VisitSystemArrayCopy()
3086 __ B(&loop, ne); in VisitSystemArrayCopy()
3088 __ Bind(read_barrier_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
3107 __ Bind(&loop); in VisitSystemArrayCopy()
3110 __ Ldr(tmp, MemOperand(src_curr_addr, element_size, PostIndex)); in VisitSystemArrayCopy()
3111 __ Str(tmp, MemOperand(dst_curr_addr, element_size, PostIndex)); in VisitSystemArrayCopy()
3113 __ Cmp(src_curr_addr, src_stop_addr); in VisitSystemArrayCopy()
3114 __ B(&loop, ne); in VisitSystemArrayCopy()
3116 __ Bind(&done); in VisitSystemArrayCopy()
3123 __ Bind(intrinsic_slow_path->GetExitLabel()); in VisitSystemArrayCopy()
3145 __ Eor(out, out, infinity); in GenIsInfinite()
3148 __ Tst(out, tst_mask); in GenIsInfinite()
3149 __ Cset(out, eq); in GenIsInfinite()
3204 __ Mov(temp.W(), value); in VisitIntegerValueOf()
3205 __ Str(temp.W(), HeapOperand(out.W(), info.value_offset)); in VisitIntegerValueOf()
3213 __ Add(out.W(), in.W(), -info.low); in VisitIntegerValueOf()
3214 __ Cmp(out.W(), info.length); in VisitIntegerValueOf()
3216 __ B(&allocate, hs); in VisitIntegerValueOf()
3223 __ B(&done); in VisitIntegerValueOf()
3224 __ Bind(&allocate); in VisitIntegerValueOf()
3227 __ Str(in.W(), HeapOperand(out.W(), info.value_offset)); in VisitIntegerValueOf()
3230 __ Bind(&done); in VisitIntegerValueOf()
3256 __ Ldr(temp, in VisitReferenceGetReferent()
3258 __ Cbz(temp, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
3272 __ Ldrh(temp, HeapOperand(temp, disable_intrinsic_offset.Uint32Value())); in VisitReferenceGetReferent()
3273 __ Cbnz(temp, slow_path->GetEntryLabel()); in VisitReferenceGetReferent()
3292 __ Bind(slow_path->GetExitLabel()); in VisitReferenceGetReferent()
3316 __ Cmp(tmp, other); in VisitReferenceRefersTo()
3324 __ Cbz(mr, &calculate_result); in VisitReferenceRefersTo()
3326 __ B(&calculate_result, eq); // ZF set if taken. in VisitReferenceRefersTo()
3329 __ Cbz(tmp, &calculate_result); // ZF clear if taken. in VisitReferenceRefersTo()
3337 __ Ldr(tmp, HeapOperand(tmp, monitor_offset)); in VisitReferenceRefersTo()
3338 __ Cmp(tmp, Operand(0xc0000000)); in VisitReferenceRefersTo()
3339 __ B(&calculate_result, lo); // ZF clear if taken. in VisitReferenceRefersTo()
3342 __ Cmp(other, Operand(tmp, LSL, LockWord::kForwardingAddressShift)); in VisitReferenceRefersTo()
3344 __ Bind(&calculate_result); in VisitReferenceRefersTo()
3348 __ Cset(out, eq); in VisitReferenceRefersTo()
3363 __ Add(temp, tr, Thread::InterruptedOffset<kArm64PointerSize>().Int32Value()); in VisitThreadInterrupted()
3364 __ Ldar(out.W(), MemOperand(temp)); in VisitThreadInterrupted()
3367 __ Cbz(out.W(), &done); in VisitThreadInterrupted()
3368 __ Stlr(wzr, MemOperand(temp)); in VisitThreadInterrupted()
3369 __ Bind(&done); in VisitThreadInterrupted()
3413 __ Mvn(tmp, crc); in VisitCRC32Update()
3414 __ Crc32b(tmp, tmp, val); in VisitCRC32Update()
3415 __ Mvn(out, tmp); in VisitCRC32Update()
3455 __ Mvn(out, crc); in GenerateCodeForCalculationCRC32ValueOfBytes()
3456 __ Mov(len, length); in GenerateCodeForCalculationCRC32ValueOfBytes()
3458 __ Tbz(ptr, 0, &aligned2); in GenerateCodeForCalculationCRC32ValueOfBytes()
3459 __ Subs(len, len, 1); in GenerateCodeForCalculationCRC32ValueOfBytes()
3460 __ B(&done, lo); in GenerateCodeForCalculationCRC32ValueOfBytes()
3461 __ Ldrb(array_elem, MemOperand(ptr, 1, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3462 __ Crc32b(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3464 __ Bind(&aligned2); in GenerateCodeForCalculationCRC32ValueOfBytes()
3465 __ Tbz(ptr, 1, &aligned4); in GenerateCodeForCalculationCRC32ValueOfBytes()
3466 __ Subs(len, len, 2); in GenerateCodeForCalculationCRC32ValueOfBytes()
3467 __ B(&process_1byte, lo); in GenerateCodeForCalculationCRC32ValueOfBytes()
3468 __ Ldrh(array_elem, MemOperand(ptr, 2, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3469 __ Crc32h(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3471 __ Bind(&aligned4); in GenerateCodeForCalculationCRC32ValueOfBytes()
3472 __ Tbz(ptr, 2, &aligned8); in GenerateCodeForCalculationCRC32ValueOfBytes()
3473 __ Subs(len, len, 4); in GenerateCodeForCalculationCRC32ValueOfBytes()
3474 __ B(&process_2bytes, lo); in GenerateCodeForCalculationCRC32ValueOfBytes()
3475 __ Ldr(array_elem, MemOperand(ptr, 4, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3476 __ Crc32w(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3478 __ Bind(&aligned8); in GenerateCodeForCalculationCRC32ValueOfBytes()
3479 __ Subs(len, len, 8); in GenerateCodeForCalculationCRC32ValueOfBytes()
3481 __ B(&process_4bytes, lo); in GenerateCodeForCalculationCRC32ValueOfBytes()
3484 __ Bind(&loop); in GenerateCodeForCalculationCRC32ValueOfBytes()
3485 __ Ldr(array_elem.X(), MemOperand(ptr, 8, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3486 __ Subs(len, len, 8); in GenerateCodeForCalculationCRC32ValueOfBytes()
3487 __ Crc32x(out, out, array_elem.X()); in GenerateCodeForCalculationCRC32ValueOfBytes()
3489 __ B(&loop, hs); in GenerateCodeForCalculationCRC32ValueOfBytes()
3500 __ Bind(&process_4bytes); in GenerateCodeForCalculationCRC32ValueOfBytes()
3502 __ Tbz(len, 2, &process_2bytes); in GenerateCodeForCalculationCRC32ValueOfBytes()
3503 __ Ldr(array_elem, MemOperand(ptr, 4, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3504 __ Crc32w(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3506 __ Bind(&process_2bytes); in GenerateCodeForCalculationCRC32ValueOfBytes()
3508 __ Tbz(len, 1, &process_1byte); in GenerateCodeForCalculationCRC32ValueOfBytes()
3509 __ Ldrh(array_elem, MemOperand(ptr, 2, PostIndex)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3510 __ Crc32h(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3512 __ Bind(&process_1byte); in GenerateCodeForCalculationCRC32ValueOfBytes()
3514 __ Tbz(len, 0, &done); in GenerateCodeForCalculationCRC32ValueOfBytes()
3515 __ Ldrb(array_elem, MemOperand(ptr)); in GenerateCodeForCalculationCRC32ValueOfBytes()
3516 __ Crc32b(out, out, array_elem); in GenerateCodeForCalculationCRC32ValueOfBytes()
3518 __ Bind(&done); in GenerateCodeForCalculationCRC32ValueOfBytes()
3519 __ Mvn(out, out); in GenerateCodeForCalculationCRC32ValueOfBytes()
3558 __ Cmp(length, kCRC32UpdateBytesThreshold); in VisitCRC32UpdateBytes()
3559 __ B(slow_path->GetEntryLabel(), hi); in VisitCRC32UpdateBytes()
3568 __ Add(ptr, array, array_data_offset + offset_value); in VisitCRC32UpdateBytes()
3570 __ Add(ptr, array, array_data_offset); in VisitCRC32UpdateBytes()
3571 __ Add(ptr, ptr, XRegisterFrom(offset)); in VisitCRC32UpdateBytes()
3579 __ Bind(slow_path->GetExitLabel()); in VisitCRC32UpdateBytes()
3617 __ Add(ptr, addr, XRegisterFrom(locations->InAt(2))); in VisitCRC32UpdateByteBuffer()
3644 __ Fmov(half, bits); // ARMv8.2 in VisitFP16ToFloat()
3645 __ Fcvt(out, half); in VisitFP16ToFloat()
3667 __ Fcvt(half, in); in VisitFP16ToHalf()
3668 __ Fmov(out, half); in VisitFP16ToHalf()
3669 __ Sxth(out, out); // sign extend due to returning a short type. in VisitFP16ToHalf()
3682 __ Fmov(half, WRegisterFrom(locations->InAt(0))); in GenerateFP16Round()
3684 __ Fmov(out, half); in GenerateFP16Round()
3685 __ Sxth(out, out); in GenerateFP16Round()
3699 __ Frintm(out, in); // Round towards Minus infinity in VisitFP16Floor()
3715 __ Frintp(out, in); // Round towards Plus infinity in VisitFP16Ceil()
3731 __ Frintn(out, in); // Round to nearest, with ties to even in VisitFP16Rint()
3746 __ Fmov(half0, WRegisterFrom(locations->InAt(0))); in GenerateFP16Compare()
3747 __ Fmov(half1, WRegisterFrom(locations->InAt(1))); in GenerateFP16Compare()
3756 __ Fcmp(in0, in1); in GenerateFP16Compare()
3757 __ Cset(out, cond); in GenerateFP16Compare()
3836 __ Cbz(divisor, slow_path->GetEntryLabel()); in GenerateDivideUnsigned()
3838 __ Udiv(out, dividend, divisor); in GenerateDivideUnsigned()
3840 __ Bind(slow_path->GetExitLabel()); in GenerateDivideUnsigned()
3873 __ Smulh(out, x, y); in VisitMathMultiplyHigh()
3951 __ Cbz(object, &success); in GenerateSubTypeObjectCheckNoReadBarrier()
3957 __ Ldr(temp, HeapOperand(object, class_offset.Int32Value())); in GenerateSubTypeObjectCheckNoReadBarrier()
3960 __ Bind(&loop); in GenerateSubTypeObjectCheckNoReadBarrier()
3961 __ Cmp(type, temp); in GenerateSubTypeObjectCheckNoReadBarrier()
3962 __ B(&success, eq); in GenerateSubTypeObjectCheckNoReadBarrier()
3963 __ Ldr(temp, HeapOperand(temp, super_class_offset.Int32Value())); in GenerateSubTypeObjectCheckNoReadBarrier()
3965 __ Cbz(temp, slow_path->GetEntryLabel()); in GenerateSubTypeObjectCheckNoReadBarrier()
3966 __ B(&loop); in GenerateSubTypeObjectCheckNoReadBarrier()
3967 __ Bind(&success); in GenerateSubTypeObjectCheckNoReadBarrier()
3996 __ Ldp(var_type_no_rb, temp2, HeapOperand(varhandle, var_type_offset.Int32Value())); in GenerateVarHandleAccessModeAndVarTypeChecks()
3998 __ Tbz(temp2, static_cast<uint32_t>(access_mode), slow_path->GetEntryLabel()); in GenerateVarHandleAccessModeAndVarTypeChecks()
3999 __ Ldrh(temp2, HeapOperand(var_type_no_rb, primitive_type_offset.Int32Value())); in GenerateVarHandleAccessModeAndVarTypeChecks()
4002 __ Cbnz(temp2, slow_path->GetEntryLabel()); in GenerateVarHandleAccessModeAndVarTypeChecks()
4004 __ Cmp(temp2, static_cast<uint16_t>(primitive_type)); in GenerateVarHandleAccessModeAndVarTypeChecks()
4005 __ B(slow_path->GetEntryLabel(), ne); in GenerateVarHandleAccessModeAndVarTypeChecks()
4041 __ Ldr(temp, HeapOperand(varhandle, coordinate_type0_offset.Int32Value())); in GenerateVarHandleStaticFieldCheck()
4042 __ Cbnz(temp, slow_path->GetEntryLabel()); in GenerateVarHandleStaticFieldCheck()
4056 __ Cbz(object, slow_path->GetEntryLabel()); in GenerateVarHandleInstanceFieldChecks()
4066 __ Ldp(temp, temp2, HeapOperand(varhandle, coordinate_type0_offset.Int32Value())); in GenerateVarHandleInstanceFieldChecks()
4069 __ Cbnz(temp2, slow_path->GetEntryLabel()); in GenerateVarHandleInstanceFieldChecks()
4109 __ Cbz(object, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
4119 __ Ldp(temp, temp2, HeapOperand(varhandle, coordinate_type0_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4122 __ Cbz(temp2, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
4134 __ Ldr(temp2, HeapOperand(object, class_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4136 __ Cmp(temp, temp2); in GenerateVarHandleArrayChecks()
4137 __ B(slow_path->GetEntryLabel(), ne); in GenerateVarHandleArrayChecks()
4142 __ Ldr(temp2, HeapOperand(temp, component_type_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4144 __ Cbz(temp2, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
4147 __ Ldrh(temp2, HeapOperand(temp2, primitive_type_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4150 __ Cbnz(temp2, slow_path->GetEntryLabel()); in GenerateVarHandleArrayChecks()
4163 __ Cmp(temp2, static_cast<uint16_t>(primitive_type)); in GenerateVarHandleArrayChecks()
4164 __ B(slow_path_label, ne); in GenerateVarHandleArrayChecks()
4168 __ Ldr(temp, HeapOperand(object, array_length_offset.Int32Value())); in GenerateVarHandleArrayChecks()
4169 __ Cmp(index, temp); in GenerateVarHandleArrayChecks()
4170 __ B(slow_path->GetEntryLabel(), hs); in GenerateVarHandleArrayChecks()
4238 __ Ldr(method.X(), HeapOperand(varhandle, art_field_offset.Int32Value())); in GenerateVarHandleTarget()
4239 __ Ldr(target.offset, MemOperand(method.X(), offset_offset.Int32Value())); in GenerateVarHandleTarget()
4259 __ Lsl(shifted_index, index, size_shift); in GenerateVarHandleTarget()
4261 __ Add(target.offset, shifted_index, data_offset.Int32Value()); in GenerateVarHandleTarget()
4435 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleGet()
4448 __ Add(tmp_ptr, target.object.X(), target.offset.X()); in GenerateVarHandleGet()
4491 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleGet()
4550 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleSet()
4565 __ Mov(temp, value.W()); in GenerateVarHandleSet()
4576 __ Fmov(temp, source.Is64Bits() ? source.D() : source.S()); in GenerateVarHandleSet()
4596 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleSet()
4707 __ Fmov(reg, cpu_reg.S()); in MoveToTempIfFpRegister()
4712 __ Fmov(reg, cpu_reg.D()); in MoveToTempIfFpRegister()
4744 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleCompareAndSetOrExchange()
4761 __ Add(tmp_ptr, target.object.X(), target.offset.X()); in GenerateVarHandleCompareAndSetOrExchange()
4775 __ Uxth(temp, expected_reg); in GenerateVarHandleCompareAndSetOrExchange()
4780 __ Uxtb(temp, expected_reg); in GenerateVarHandleCompareAndSetOrExchange()
4879 __ Bind(exit_loop); in GenerateVarHandleCompareAndSetOrExchange()
4883 __ Cset(out.W(), eq); in GenerateVarHandleCompareAndSetOrExchange()
4888 __ Csel(out.W(), store_result, wzr, eq); in GenerateVarHandleCompareAndSetOrExchange()
4894 __ Fmov((value_type == DataType::Type::kFloat64) ? out.D() : out.S(), old_value); in GenerateVarHandleCompareAndSetOrExchange()
4896 __ Sxtb(out.W(), old_value); in GenerateVarHandleCompareAndSetOrExchange()
4898 __ Sxth(out.W(), old_value); in GenerateVarHandleCompareAndSetOrExchange()
4902 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleCompareAndSetOrExchange()
5043 __ Bind(slow_path->GetNativeByteOrderLabel()); in GenerateVarHandleGetAndUpdate()
5062 __ Add(tmp_ptr, target.object.X(), target.offset.X()); in GenerateVarHandleGetAndUpdate()
5123 __ Sxth(out.W(), old_value.W()); in GenerateVarHandleGetAndUpdate()
5129 __ Fmov(out.D(), old_value.X()); in GenerateVarHandleGetAndUpdate()
5131 __ Fmov(out.S(), old_value.W()); in GenerateVarHandleGetAndUpdate()
5133 __ Sxtb(out.W(), old_value.W()); in GenerateVarHandleGetAndUpdate()
5135 __ Sxth(out.W(), old_value.W()); in GenerateVarHandleGetAndUpdate()
5151 __ Bind(slow_path->GetExitLabel()); in GenerateVarHandleGetAndUpdate()
5295 __ Bind(GetByteArrayViewCheckLabel()); in EmitByteArrayViewCode()
5306 __ Ldr(temp, HeapOperand(varhandle, class_offset.Int32Value())); in EmitByteArrayViewCode()
5308 __ Cmp(temp, temp2); in EmitByteArrayViewCode()
5309 __ B(GetEntryLabel(), ne); in EmitByteArrayViewCode()
5312 __ Ldr(temp, HeapOperand(object, array_length_offset.Int32Value())); in EmitByteArrayViewCode()
5313 __ Subs(temp, temp, index); in EmitByteArrayViewCode()
5314 __ Ccmp(temp, size, NoFlag, hs); // If SUBS yields LO (C=false), keep the C flag clear. in EmitByteArrayViewCode()
5315 __ B(GetEntryLabel(), lo); in EmitByteArrayViewCode()
5318 __ Add(target.offset, index, data_offset.Int32Value()); in EmitByteArrayViewCode()
5323 __ Tbnz(target.offset, 0, GetEntryLabel()); in EmitByteArrayViewCode()
5325 __ Tst(target.offset, size - 1u); in EmitByteArrayViewCode()
5326 __ B(GetEntryLabel(), ne); in EmitByteArrayViewCode()
5334 __ B(GetNativeByteOrderLabel()); in EmitByteArrayViewCode()
5337 __ Ldr(temp, HeapOperand(varhandle, native_byte_order_offset.Int32Value())); in EmitByteArrayViewCode()
5338 __ Cbnz(temp, GetNativeByteOrderLabel()); in EmitByteArrayViewCode()
5358 __ B(GetExitLabel()); in EmitByteArrayViewCode()
5391 #undef __