Lines Matching refs:invoke
161 bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) { in TryDispatch() argument
162 Dispatch(invoke); in TryDispatch()
163 LocationSummary* res = invoke->GetLocations(); in TryDispatch()
172 static void CreateFPToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToIntLocations() argument
174 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPToIntLocations()
179 static void CreateIntToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntToFPLocations() argument
181 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntToFPLocations()
200 void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { in VisitDoubleDoubleToRawLongBits() argument
201 CreateFPToIntLocations(allocator_, invoke); in VisitDoubleDoubleToRawLongBits()
203 void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { in VisitDoubleLongBitsToDouble() argument
204 CreateIntToFPLocations(allocator_, invoke); in VisitDoubleLongBitsToDouble()
207 void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { in VisitDoubleDoubleToRawLongBits() argument
208 MoveFPToInt(invoke->GetLocations(), /* is64bit= */ true, GetVIXLAssembler()); in VisitDoubleDoubleToRawLongBits()
210 void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { in VisitDoubleLongBitsToDouble() argument
211 MoveIntToFP(invoke->GetLocations(), /* is64bit= */ true, GetVIXLAssembler()); in VisitDoubleLongBitsToDouble()
214 void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { in VisitFloatFloatToRawIntBits() argument
215 CreateFPToIntLocations(allocator_, invoke); in VisitFloatFloatToRawIntBits()
217 void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) { in VisitFloatIntBitsToFloat() argument
218 CreateIntToFPLocations(allocator_, invoke); in VisitFloatIntBitsToFloat()
221 void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { in VisitFloatFloatToRawIntBits() argument
222 MoveFPToInt(invoke->GetLocations(), /* is64bit= */ false, GetVIXLAssembler()); in VisitFloatFloatToRawIntBits()
224 void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) { in VisitFloatIntBitsToFloat() argument
225 MoveIntToFP(invoke->GetLocations(), /* is64bit= */ false, GetVIXLAssembler()); in VisitFloatIntBitsToFloat()
228 static void CreateIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntToIntLocations() argument
230 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntToIntLocations()
235 static void CreateIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntIntToIntLocations() argument
237 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntToIntLocations()
243 static void CreateIntIntToIntSlowPathCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntIntToIntSlowPathCallLocations() argument
245 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateIntIntToIntSlowPathCallLocations()
292 void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) { in VisitIntegerReverseBytes() argument
293 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerReverseBytes()
296 void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) { in VisitIntegerReverseBytes() argument
297 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerReverseBytes()
300 void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) { in VisitLongReverseBytes() argument
301 CreateIntToIntLocations(allocator_, invoke); in VisitLongReverseBytes()
304 void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) { in VisitLongReverseBytes() argument
305 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongReverseBytes()
308 void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) { in VisitShortReverseBytes() argument
309 CreateIntToIntLocations(allocator_, invoke); in VisitShortReverseBytes()
312 void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) { in VisitShortReverseBytes() argument
313 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt16, GetVIXLAssembler()); in VisitShortReverseBytes()
327 void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { in VisitIntegerNumberOfLeadingZeros() argument
328 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerNumberOfLeadingZeros()
331 void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { in VisitIntegerNumberOfLeadingZeros() argument
332 GenNumberOfLeadingZeros(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerNumberOfLeadingZeros()
335 void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { in VisitLongNumberOfLeadingZeros() argument
336 CreateIntToIntLocations(allocator_, invoke); in VisitLongNumberOfLeadingZeros()
339 void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { in VisitLongNumberOfLeadingZeros() argument
340 GenNumberOfLeadingZeros(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongNumberOfLeadingZeros()
355 void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { in VisitIntegerNumberOfTrailingZeros() argument
356 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerNumberOfTrailingZeros()
359 void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { in VisitIntegerNumberOfTrailingZeros() argument
360 GenNumberOfTrailingZeros(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerNumberOfTrailingZeros()
363 void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { in VisitLongNumberOfTrailingZeros() argument
364 CreateIntToIntLocations(allocator_, invoke); in VisitLongNumberOfTrailingZeros()
367 void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { in VisitLongNumberOfTrailingZeros() argument
368 GenNumberOfTrailingZeros(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongNumberOfTrailingZeros()
382 void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) { in VisitIntegerReverse() argument
383 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerReverse()
386 void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) { in VisitIntegerReverse() argument
387 GenReverse(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerReverse()
390 void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) { in VisitLongReverse() argument
391 CreateIntToIntLocations(allocator_, invoke); in VisitLongReverse()
394 void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) { in VisitLongReverse() argument
395 GenReverse(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongReverse()
415 void IntrinsicLocationsBuilderARM64::VisitLongBitCount(HInvoke* invoke) { in VisitLongBitCount() argument
416 CreateIntToIntLocations(allocator_, invoke); in VisitLongBitCount()
419 void IntrinsicCodeGeneratorARM64::VisitLongBitCount(HInvoke* invoke) { in VisitLongBitCount() argument
420 GenBitCount(invoke, DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongBitCount()
423 void IntrinsicLocationsBuilderARM64::VisitIntegerBitCount(HInvoke* invoke) { in VisitIntegerBitCount() argument
424 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerBitCount()
427 void IntrinsicCodeGeneratorARM64::VisitIntegerBitCount(HInvoke* invoke) { in VisitIntegerBitCount() argument
428 GenBitCount(invoke, DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerBitCount()
431 static void GenHighestOneBit(HInvoke* invoke, DataType::Type type, MacroAssembler* masm) { in GenHighestOneBit() argument
436 Register src = InputRegisterAt(invoke, 0); in GenHighestOneBit()
437 Register dst = RegisterFrom(invoke->GetLocations()->Out(), type); in GenHighestOneBit()
448 void IntrinsicLocationsBuilderARM64::VisitIntegerHighestOneBit(HInvoke* invoke) { in VisitIntegerHighestOneBit() argument
449 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerHighestOneBit()
452 void IntrinsicCodeGeneratorARM64::VisitIntegerHighestOneBit(HInvoke* invoke) { in VisitIntegerHighestOneBit() argument
453 GenHighestOneBit(invoke, DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerHighestOneBit()
456 void IntrinsicLocationsBuilderARM64::VisitLongHighestOneBit(HInvoke* invoke) { in VisitLongHighestOneBit() argument
457 CreateIntToIntLocations(allocator_, invoke); in VisitLongHighestOneBit()
460 void IntrinsicCodeGeneratorARM64::VisitLongHighestOneBit(HInvoke* invoke) { in VisitLongHighestOneBit() argument
461 GenHighestOneBit(invoke, DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongHighestOneBit()
464 static void GenLowestOneBit(HInvoke* invoke, DataType::Type type, MacroAssembler* masm) { in GenLowestOneBit() argument
469 Register src = InputRegisterAt(invoke, 0); in GenLowestOneBit()
470 Register dst = RegisterFrom(invoke->GetLocations()->Out(), type); in GenLowestOneBit()
477 void IntrinsicLocationsBuilderARM64::VisitIntegerLowestOneBit(HInvoke* invoke) { in VisitIntegerLowestOneBit() argument
478 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerLowestOneBit()
481 void IntrinsicCodeGeneratorARM64::VisitIntegerLowestOneBit(HInvoke* invoke) { in VisitIntegerLowestOneBit() argument
482 GenLowestOneBit(invoke, DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerLowestOneBit()
485 void IntrinsicLocationsBuilderARM64::VisitLongLowestOneBit(HInvoke* invoke) { in VisitLongLowestOneBit() argument
486 CreateIntToIntLocations(allocator_, invoke); in VisitLongLowestOneBit()
489 void IntrinsicCodeGeneratorARM64::VisitLongLowestOneBit(HInvoke* invoke) { in VisitLongLowestOneBit() argument
490 GenLowestOneBit(invoke, DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongLowestOneBit()
493 static void CreateFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToFPLocations() argument
495 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPToFPLocations()
500 void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) { in VisitMathSqrt() argument
501 CreateFPToFPLocations(allocator_, invoke); in VisitMathSqrt()
504 void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) { in VisitMathSqrt() argument
505 LocationSummary* locations = invoke->GetLocations(); in VisitMathSqrt()
510 void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) { in VisitMathCeil() argument
511 CreateFPToFPLocations(allocator_, invoke); in VisitMathCeil()
514 void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) { in VisitMathCeil() argument
515 LocationSummary* locations = invoke->GetLocations(); in VisitMathCeil()
520 void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) { in VisitMathFloor() argument
521 CreateFPToFPLocations(allocator_, invoke); in VisitMathFloor()
524 void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) { in VisitMathFloor() argument
525 LocationSummary* locations = invoke->GetLocations(); in VisitMathFloor()
530 void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) { in VisitMathRint() argument
531 CreateFPToFPLocations(allocator_, invoke); in VisitMathRint()
534 void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) { in VisitMathRint() argument
535 LocationSummary* locations = invoke->GetLocations(); in VisitMathRint()
540 static void CreateFPToIntPlusFPTempLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToIntPlusFPTempLocations() argument
542 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPToIntPlusFPTempLocations()
548 static void GenMathRound(HInvoke* invoke, bool is_double, vixl::aarch64::MacroAssembler* masm) { in GenMathRound() argument
562 LocationSummary* l = invoke->GetLocations(); in GenMathRound()
585 void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) { in VisitMathRoundDouble() argument
586 CreateFPToIntPlusFPTempLocations(allocator_, invoke); in VisitMathRoundDouble()
589 void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) { in VisitMathRoundDouble() argument
590 GenMathRound(invoke, /* is_double= */ true, GetVIXLAssembler()); in VisitMathRoundDouble()
593 void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) { in VisitMathRoundFloat() argument
594 CreateFPToIntPlusFPTempLocations(allocator_, invoke); in VisitMathRoundFloat()
597 void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) { in VisitMathRoundFloat() argument
598 GenMathRound(invoke, /* is_double= */ false, GetVIXLAssembler()); in VisitMathRoundFloat()
601 void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) { in VisitMemoryPeekByte() argument
602 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekByte()
605 void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) { in VisitMemoryPeekByte() argument
607 __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekByte()
608 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPeekByte()
611 void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) { in VisitMemoryPeekIntNative() argument
612 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekIntNative()
615 void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) { in VisitMemoryPeekIntNative() argument
617 __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekIntNative()
618 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPeekIntNative()
621 void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) { in VisitMemoryPeekLongNative() argument
622 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekLongNative()
625 void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) { in VisitMemoryPeekLongNative() argument
627 __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekLongNative()
628 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPeekLongNative()
631 void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) { in VisitMemoryPeekShortNative() argument
632 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekShortNative()
635 void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) { in VisitMemoryPeekShortNative() argument
637 __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekShortNative()
638 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPeekShortNative()
641 static void CreateIntIntToVoidLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntIntToVoidLocations() argument
643 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntToVoidLocations()
648 void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) { in VisitMemoryPokeByte() argument
649 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeByte()
652 void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) { in VisitMemoryPokeByte() argument
654 __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeByte()
655 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPokeByte()
658 void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) { in VisitMemoryPokeIntNative() argument
659 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeIntNative()
662 void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) { in VisitMemoryPokeIntNative() argument
664 __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeIntNative()
665 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPokeIntNative()
668 void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) { in VisitMemoryPokeLongNative() argument
669 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeLongNative()
672 void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) { in VisitMemoryPokeLongNative() argument
674 __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeLongNative()
675 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPokeLongNative()
678 void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) { in VisitMemoryPokeShortNative() argument
679 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeShortNative()
682 void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) { in VisitMemoryPokeShortNative() argument
684 __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeShortNative()
685 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPokeShortNative()
688 void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) { in VisitThreadCurrentThread() argument
690 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitThreadCurrentThread()
694 void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) { in VisitThreadCurrentThread() argument
695 codegen_->Load(DataType::Type::kReference, WRegisterFrom(invoke->GetLocations()->Out()), in VisitThreadCurrentThread()
699 static void GenUnsafeGet(HInvoke* invoke, in GenUnsafeGet() argument
703 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeGet()
720 codegen->GenerateFieldLoadWithBakerReadBarrier(invoke, in GenUnsafeGet()
730 codegen->LoadAcquire(invoke, type, trg, mem_op, /* needs_null_check= */ true); in GenUnsafeGet()
737 codegen->MaybeGenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0u, offset_loc); in GenUnsafeGet()
742 static void CreateIntIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntIntIntToIntLocations() argument
744 (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || in CreateIntIntIntToIntLocations()
745 invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); in CreateIntIntIntToIntLocations()
747 new (allocator) LocationSummary(invoke, in CreateIntIntIntToIntLocations()
765 void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) { in VisitUnsafeGet() argument
766 CreateIntIntIntToIntLocations(allocator_, invoke); in VisitUnsafeGet()
768 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) { in VisitUnsafeGetVolatile() argument
769 CreateIntIntIntToIntLocations(allocator_, invoke); in VisitUnsafeGetVolatile()
771 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) { in VisitUnsafeGetLong() argument
772 CreateIntIntIntToIntLocations(allocator_, invoke); in VisitUnsafeGetLong()
774 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { in VisitUnsafeGetLongVolatile() argument
775 CreateIntIntIntToIntLocations(allocator_, invoke); in VisitUnsafeGetLongVolatile()
777 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) { in VisitUnsafeGetObject() argument
778 CreateIntIntIntToIntLocations(allocator_, invoke); in VisitUnsafeGetObject()
780 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { in VisitUnsafeGetObjectVolatile() argument
781 CreateIntIntIntToIntLocations(allocator_, invoke); in VisitUnsafeGetObjectVolatile()
784 void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) { in VisitUnsafeGet() argument
785 GenUnsafeGet(invoke, DataType::Type::kInt32, /* is_volatile= */ false, codegen_); in VisitUnsafeGet()
787 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) { in VisitUnsafeGetVolatile() argument
788 GenUnsafeGet(invoke, DataType::Type::kInt32, /* is_volatile= */ true, codegen_); in VisitUnsafeGetVolatile()
790 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) { in VisitUnsafeGetLong() argument
791 GenUnsafeGet(invoke, DataType::Type::kInt64, /* is_volatile= */ false, codegen_); in VisitUnsafeGetLong()
793 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { in VisitUnsafeGetLongVolatile() argument
794 GenUnsafeGet(invoke, DataType::Type::kInt64, /* is_volatile= */ true, codegen_); in VisitUnsafeGetLongVolatile()
796 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) { in VisitUnsafeGetObject() argument
797 GenUnsafeGet(invoke, DataType::Type::kReference, /* is_volatile= */ false, codegen_); in VisitUnsafeGetObject()
799 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { in VisitUnsafeGetObjectVolatile() argument
800 GenUnsafeGet(invoke, DataType::Type::kReference, /* is_volatile= */ true, codegen_); in VisitUnsafeGetObjectVolatile()
803 static void CreateIntIntIntIntToVoid(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntIntIntIntToVoid() argument
805 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntIntIntToVoid()
812 void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) { in VisitUnsafePut() argument
813 CreateIntIntIntIntToVoid(allocator_, invoke); in VisitUnsafePut()
815 void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) { in VisitUnsafePutOrdered() argument
816 CreateIntIntIntIntToVoid(allocator_, invoke); in VisitUnsafePutOrdered()
818 void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) { in VisitUnsafePutVolatile() argument
819 CreateIntIntIntIntToVoid(allocator_, invoke); in VisitUnsafePutVolatile()
821 void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) { in VisitUnsafePutObject() argument
822 CreateIntIntIntIntToVoid(allocator_, invoke); in VisitUnsafePutObject()
824 void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { in VisitUnsafePutObjectOrdered() argument
825 CreateIntIntIntIntToVoid(allocator_, invoke); in VisitUnsafePutObjectOrdered()
827 void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { in VisitUnsafePutObjectVolatile() argument
828 CreateIntIntIntIntToVoid(allocator_, invoke); in VisitUnsafePutObjectVolatile()
830 void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) { in VisitUnsafePutLong() argument
831 CreateIntIntIntIntToVoid(allocator_, invoke); in VisitUnsafePutLong()
833 void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { in VisitUnsafePutLongOrdered() argument
834 CreateIntIntIntIntToVoid(allocator_, invoke); in VisitUnsafePutLongOrdered()
836 void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { in VisitUnsafePutLongVolatile() argument
837 CreateIntIntIntIntToVoid(allocator_, invoke); in VisitUnsafePutLongVolatile()
840 static void GenUnsafePut(HInvoke* invoke, in GenUnsafePut() argument
845 LocationSummary* locations = invoke->GetLocations(); in GenUnsafePut()
868 codegen->StoreRelease(invoke, type, source, mem_op, /* needs_null_check= */ false); in GenUnsafePut()
880 void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) { in VisitUnsafePut() argument
881 GenUnsafePut(invoke, in VisitUnsafePut()
887 void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) { in VisitUnsafePutOrdered() argument
888 GenUnsafePut(invoke, in VisitUnsafePutOrdered()
894 void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) { in VisitUnsafePutVolatile() argument
895 GenUnsafePut(invoke, in VisitUnsafePutVolatile()
901 void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) { in VisitUnsafePutObject() argument
902 GenUnsafePut(invoke, in VisitUnsafePutObject()
908 void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { in VisitUnsafePutObjectOrdered() argument
909 GenUnsafePut(invoke, in VisitUnsafePutObjectOrdered()
915 void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { in VisitUnsafePutObjectVolatile() argument
916 GenUnsafePut(invoke, in VisitUnsafePutObjectVolatile()
922 void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) { in VisitUnsafePutLong() argument
923 GenUnsafePut(invoke, in VisitUnsafePutLong()
929 void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { in VisitUnsafePutLongOrdered() argument
930 GenUnsafePut(invoke, in VisitUnsafePutLongOrdered()
936 void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { in VisitUnsafePutLongVolatile() argument
937 GenUnsafePut(invoke, in VisitUnsafePutLongVolatile()
944 static void CreateUnsafeCASLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateUnsafeCASLocations() argument
946 (invoke->GetIntrinsic() == Intrinsics::kUnsafeCASObject); in CreateUnsafeCASLocations()
948 new (allocator) LocationSummary(invoke, in CreateUnsafeCASLocations()
1134 ReadBarrierCasSlowPathARM64(HInvoke* invoke, in ReadBarrierCasSlowPathARM64() argument
1146 : SlowPathCodeARM64(invoke), in ReadBarrierCasSlowPathARM64()
1162 invoke, in ReadBarrierCasSlowPathARM64()
1170 invoke, in ReadBarrierCasSlowPathARM64()
1273 static void GenUnsafeCas(HInvoke* invoke, DataType::Type type, CodeGeneratorARM64* codegen) { in GenUnsafeCas() argument
1275 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeCas()
1307 invoke, in GenUnsafeCas()
1342 void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) { in VisitUnsafeCASInt() argument
1343 CreateUnsafeCASLocations(allocator_, invoke); in VisitUnsafeCASInt()
1345 void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) { in VisitUnsafeCASLong() argument
1346 CreateUnsafeCASLocations(allocator_, invoke); in VisitUnsafeCASLong()
1348 void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) { in VisitUnsafeCASObject() argument
1355 CreateUnsafeCASLocations(allocator_, invoke); in VisitUnsafeCASObject()
1358 LocationSummary* locations = invoke->GetLocations(); in VisitUnsafeCASObject()
1375 void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) { in VisitUnsafeCASInt() argument
1376 GenUnsafeCas(invoke, DataType::Type::kInt32, codegen_); in VisitUnsafeCASInt()
1378 void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) { in VisitUnsafeCASLong() argument
1379 GenUnsafeCas(invoke, DataType::Type::kInt64, codegen_); in VisitUnsafeCASLong()
1381 void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) { in VisitUnsafeCASObject() argument
1382 GenUnsafeCas(invoke, DataType::Type::kReference, codegen_); in VisitUnsafeCASObject()
1475 void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) { in VisitStringCompareTo() argument
1477 new (allocator_) LocationSummary(invoke, in VisitStringCompareTo()
1478 invoke->InputAt(1)->CanBeNull() in VisitStringCompareTo()
1494 void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) { in VisitStringCompareTo() argument
1496 LocationSummary* locations = invoke->GetLocations(); in VisitStringCompareTo()
1498 Register str = InputRegisterAt(invoke, 0); in VisitStringCompareTo()
1499 Register arg = InputRegisterAt(invoke, 1); in VisitStringCompareTo()
1502 Register out = OutputRegister(invoke); in VisitStringCompareTo()
1522 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in VisitStringCompareTo()
1526 const bool can_slow_path = invoke->InputAt(1)->CanBeNull(); in VisitStringCompareTo()
1528 slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitStringCompareTo()
1707 void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) { in VisitStringEquals() argument
1709 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitStringEquals()
1716 const char* const_string = GetConstString(invoke->InputAt(0), &const_string_length); in VisitStringEquals()
1718 const_string = GetConstString(invoke->InputAt(1), &const_string_length); in VisitStringEquals()
1734 void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) { in VisitStringEquals() argument
1736 LocationSummary* locations = invoke->GetLocations(); in VisitStringEquals()
1757 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in VisitStringEquals()
1759 StringEqualsOptimizations optimizations(invoke); in VisitStringEquals()
1791 const char* const_string = GetConstString(invoke->InputAt(0), &const_string_length); in VisitStringEquals()
1793 const_string = GetConstString(invoke->InputAt(1), &const_string_length); in VisitStringEquals()
1896 static void GenerateVisitStringIndexOf(HInvoke* invoke, in GenerateVisitStringIndexOf() argument
1900 LocationSummary* locations = invoke->GetLocations(); in GenerateVisitStringIndexOf()
1903 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in GenerateVisitStringIndexOf()
1908 HInstruction* code_point = invoke->InputAt(1); in GenerateVisitStringIndexOf()
1913 slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in GenerateVisitStringIndexOf()
1922 slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in GenerateVisitStringIndexOf()
1933 codegen->InvokeRuntime(kQuickIndexOf, invoke, invoke->GetDexPc(), slow_path); in GenerateVisitStringIndexOf()
1941 void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) { in VisitStringIndexOf() argument
1943 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringIndexOf()
1955 void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) { in VisitStringIndexOf() argument
1956 GenerateVisitStringIndexOf(invoke, GetVIXLAssembler(), codegen_, /* start_at_zero= */ true); in VisitStringIndexOf()
1959 void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) { in VisitStringIndexOfAfter() argument
1961 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringIndexOfAfter()
1971 void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) { in VisitStringIndexOfAfter() argument
1972 GenerateVisitStringIndexOf(invoke, GetVIXLAssembler(), codegen_, /* start_at_zero= */ false); in VisitStringIndexOfAfter()
1975 void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) { in VisitStringNewStringFromBytes() argument
1977 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringNewStringFromBytes()
1986 void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) { in VisitStringNewStringFromBytes() argument
1988 LocationSummary* locations = invoke->GetLocations(); in VisitStringNewStringFromBytes()
1993 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitStringNewStringFromBytes()
1997 codegen_->InvokeRuntime(kQuickAllocStringFromBytes, invoke, invoke->GetDexPc(), slow_path); in VisitStringNewStringFromBytes()
2002 void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) { in VisitStringNewStringFromChars() argument
2004 new (allocator_) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in VisitStringNewStringFromChars()
2012 void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) { in VisitStringNewStringFromChars() argument
2019 codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke, invoke->GetDexPc()); in VisitStringNewStringFromChars()
2023 void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) { in VisitStringNewStringFromString() argument
2025 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringNewStringFromString()
2031 void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) { in VisitStringNewStringFromString() argument
2033 LocationSummary* locations = invoke->GetLocations(); in VisitStringNewStringFromString()
2038 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitStringNewStringFromString()
2042 codegen_->InvokeRuntime(kQuickAllocStringFromString, invoke, invoke->GetDexPc(), slow_path); in VisitStringNewStringFromString()
2047 static void CreateFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToFPCallLocations() argument
2048 DCHECK_EQ(invoke->GetNumberOfArguments(), 1U); in CreateFPToFPCallLocations()
2049 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(0)->GetType())); in CreateFPToFPCallLocations()
2050 DCHECK(DataType::IsFloatingPointType(invoke->GetType())); in CreateFPToFPCallLocations()
2053 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in CreateFPToFPCallLocations()
2057 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType())); in CreateFPToFPCallLocations()
2060 static void CreateFPFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPFPToFPCallLocations() argument
2061 DCHECK_EQ(invoke->GetNumberOfArguments(), 2U); in CreateFPFPToFPCallLocations()
2062 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(0)->GetType())); in CreateFPFPToFPCallLocations()
2063 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(1)->GetType())); in CreateFPFPToFPCallLocations()
2064 DCHECK(DataType::IsFloatingPointType(invoke->GetType())); in CreateFPFPToFPCallLocations()
2067 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in CreateFPFPToFPCallLocations()
2072 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType())); in CreateFPFPToFPCallLocations()
2075 static void GenFPToFPCall(HInvoke* invoke, in GenFPToFPCall() argument
2078 codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); in GenFPToFPCall()
2081 void IntrinsicLocationsBuilderARM64::VisitMathCos(HInvoke* invoke) { in VisitMathCos() argument
2082 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCos()
2085 void IntrinsicCodeGeneratorARM64::VisitMathCos(HInvoke* invoke) { in VisitMathCos() argument
2086 GenFPToFPCall(invoke, codegen_, kQuickCos); in VisitMathCos()
2089 void IntrinsicLocationsBuilderARM64::VisitMathSin(HInvoke* invoke) { in VisitMathSin() argument
2090 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathSin()
2093 void IntrinsicCodeGeneratorARM64::VisitMathSin(HInvoke* invoke) { in VisitMathSin() argument
2094 GenFPToFPCall(invoke, codegen_, kQuickSin); in VisitMathSin()
2097 void IntrinsicLocationsBuilderARM64::VisitMathAcos(HInvoke* invoke) { in VisitMathAcos() argument
2098 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAcos()
2101 void IntrinsicCodeGeneratorARM64::VisitMathAcos(HInvoke* invoke) { in VisitMathAcos() argument
2102 GenFPToFPCall(invoke, codegen_, kQuickAcos); in VisitMathAcos()
2105 void IntrinsicLocationsBuilderARM64::VisitMathAsin(HInvoke* invoke) { in VisitMathAsin() argument
2106 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAsin()
2109 void IntrinsicCodeGeneratorARM64::VisitMathAsin(HInvoke* invoke) { in VisitMathAsin() argument
2110 GenFPToFPCall(invoke, codegen_, kQuickAsin); in VisitMathAsin()
2113 void IntrinsicLocationsBuilderARM64::VisitMathAtan(HInvoke* invoke) { in VisitMathAtan() argument
2114 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAtan()
2117 void IntrinsicCodeGeneratorARM64::VisitMathAtan(HInvoke* invoke) { in VisitMathAtan() argument
2118 GenFPToFPCall(invoke, codegen_, kQuickAtan); in VisitMathAtan()
2121 void IntrinsicLocationsBuilderARM64::VisitMathCbrt(HInvoke* invoke) { in VisitMathCbrt() argument
2122 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCbrt()
2125 void IntrinsicCodeGeneratorARM64::VisitMathCbrt(HInvoke* invoke) { in VisitMathCbrt() argument
2126 GenFPToFPCall(invoke, codegen_, kQuickCbrt); in VisitMathCbrt()
2129 void IntrinsicLocationsBuilderARM64::VisitMathCosh(HInvoke* invoke) { in VisitMathCosh() argument
2130 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCosh()
2133 void IntrinsicCodeGeneratorARM64::VisitMathCosh(HInvoke* invoke) { in VisitMathCosh() argument
2134 GenFPToFPCall(invoke, codegen_, kQuickCosh); in VisitMathCosh()
2137 void IntrinsicLocationsBuilderARM64::VisitMathExp(HInvoke* invoke) { in VisitMathExp() argument
2138 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathExp()
2141 void IntrinsicCodeGeneratorARM64::VisitMathExp(HInvoke* invoke) { in VisitMathExp() argument
2142 GenFPToFPCall(invoke, codegen_, kQuickExp); in VisitMathExp()
2145 void IntrinsicLocationsBuilderARM64::VisitMathExpm1(HInvoke* invoke) { in VisitMathExpm1() argument
2146 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathExpm1()
2149 void IntrinsicCodeGeneratorARM64::VisitMathExpm1(HInvoke* invoke) { in VisitMathExpm1() argument
2150 GenFPToFPCall(invoke, codegen_, kQuickExpm1); in VisitMathExpm1()
2153 void IntrinsicLocationsBuilderARM64::VisitMathLog(HInvoke* invoke) { in VisitMathLog() argument
2154 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathLog()
2157 void IntrinsicCodeGeneratorARM64::VisitMathLog(HInvoke* invoke) { in VisitMathLog() argument
2158 GenFPToFPCall(invoke, codegen_, kQuickLog); in VisitMathLog()
2161 void IntrinsicLocationsBuilderARM64::VisitMathLog10(HInvoke* invoke) { in VisitMathLog10() argument
2162 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathLog10()
2165 void IntrinsicCodeGeneratorARM64::VisitMathLog10(HInvoke* invoke) { in VisitMathLog10() argument
2166 GenFPToFPCall(invoke, codegen_, kQuickLog10); in VisitMathLog10()
2169 void IntrinsicLocationsBuilderARM64::VisitMathSinh(HInvoke* invoke) { in VisitMathSinh() argument
2170 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathSinh()
2173 void IntrinsicCodeGeneratorARM64::VisitMathSinh(HInvoke* invoke) { in VisitMathSinh() argument
2174 GenFPToFPCall(invoke, codegen_, kQuickSinh); in VisitMathSinh()
2177 void IntrinsicLocationsBuilderARM64::VisitMathTan(HInvoke* invoke) { in VisitMathTan() argument
2178 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathTan()
2181 void IntrinsicCodeGeneratorARM64::VisitMathTan(HInvoke* invoke) { in VisitMathTan() argument
2182 GenFPToFPCall(invoke, codegen_, kQuickTan); in VisitMathTan()
2185 void IntrinsicLocationsBuilderARM64::VisitMathTanh(HInvoke* invoke) { in VisitMathTanh() argument
2186 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathTanh()
2189 void IntrinsicCodeGeneratorARM64::VisitMathTanh(HInvoke* invoke) { in VisitMathTanh() argument
2190 GenFPToFPCall(invoke, codegen_, kQuickTanh); in VisitMathTanh()
2193 void IntrinsicLocationsBuilderARM64::VisitMathAtan2(HInvoke* invoke) { in VisitMathAtan2() argument
2194 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathAtan2()
2197 void IntrinsicCodeGeneratorARM64::VisitMathAtan2(HInvoke* invoke) { in VisitMathAtan2() argument
2198 GenFPToFPCall(invoke, codegen_, kQuickAtan2); in VisitMathAtan2()
2201 void IntrinsicLocationsBuilderARM64::VisitMathPow(HInvoke* invoke) { in VisitMathPow() argument
2202 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathPow()
2205 void IntrinsicCodeGeneratorARM64::VisitMathPow(HInvoke* invoke) { in VisitMathPow() argument
2206 GenFPToFPCall(invoke, codegen_, kQuickPow); in VisitMathPow()
2209 void IntrinsicLocationsBuilderARM64::VisitMathHypot(HInvoke* invoke) { in VisitMathHypot() argument
2210 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathHypot()
2213 void IntrinsicCodeGeneratorARM64::VisitMathHypot(HInvoke* invoke) { in VisitMathHypot() argument
2214 GenFPToFPCall(invoke, codegen_, kQuickHypot); in VisitMathHypot()
2217 void IntrinsicLocationsBuilderARM64::VisitMathNextAfter(HInvoke* invoke) { in VisitMathNextAfter() argument
2218 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathNextAfter()
2221 void IntrinsicCodeGeneratorARM64::VisitMathNextAfter(HInvoke* invoke) { in VisitMathNextAfter() argument
2222 GenFPToFPCall(invoke, codegen_, kQuickNextAfter); in VisitMathNextAfter()
2225 void IntrinsicLocationsBuilderARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) { in VisitStringGetCharsNoCheck() argument
2227 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitStringGetCharsNoCheck()
2239 void IntrinsicCodeGeneratorARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) { in VisitStringGetCharsNoCheck() argument
2241 LocationSummary* locations = invoke->GetLocations(); in VisitStringGetCharsNoCheck()
2377 void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopyChar(HInvoke* invoke) { in VisitSystemArrayCopyChar() argument
2380 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant(); in VisitSystemArrayCopyChar()
2381 HIntConstant* dst_pos = invoke->InputAt(3)->AsIntConstant(); in VisitSystemArrayCopyChar()
2392 HIntConstant* length = invoke->InputAt(4)->AsIntConstant(); in VisitSystemArrayCopyChar()
2401 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in VisitSystemArrayCopyChar()
2403 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in VisitSystemArrayCopyChar()
2406 SetSystemArrayCopyLocationRequires(locations, 1, invoke->InputAt(1)); in VisitSystemArrayCopyChar()
2408 SetSystemArrayCopyLocationRequires(locations, 3, invoke->InputAt(3)); in VisitSystemArrayCopyChar()
2409 SetSystemArrayCopyLocationRequires(locations, 4, invoke->InputAt(4)); in VisitSystemArrayCopyChar()
2504 void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopyChar(HInvoke* invoke) { in VisitSystemArrayCopyChar() argument
2506 LocationSummary* locations = invoke->GetLocations(); in VisitSystemArrayCopyChar()
2514 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitSystemArrayCopyChar()
2597 void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopy(HInvoke* invoke) { in VisitSystemArrayCopy() argument
2606 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant(); in VisitSystemArrayCopy()
2607 HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant(); in VisitSystemArrayCopy()
2617 HIntConstant* length = invoke->InputAt(4)->AsIntConstant(); in VisitSystemArrayCopy()
2626 SystemArrayCopyOptimizations optimizations(invoke); in VisitSystemArrayCopy()
2640 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in VisitSystemArrayCopy()
2642 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in VisitSystemArrayCopy()
2645 SetSystemArrayCopyLocationRequires(locations, 1, invoke->InputAt(1)); in VisitSystemArrayCopy()
2647 SetSystemArrayCopyLocationRequires(locations, 3, invoke->InputAt(3)); in VisitSystemArrayCopy()
2648 SetSystemArrayCopyLocationRequires(locations, 4, invoke->InputAt(4)); in VisitSystemArrayCopy()
2667 void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopy(HInvoke* invoke) { in VisitSystemArrayCopy() argument
2673 LocationSummary* locations = invoke->GetLocations(); in VisitSystemArrayCopy()
2692 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitSystemArrayCopy()
2696 SystemArrayCopyOptimizations optimizations(invoke); in VisitSystemArrayCopy()
2727 __ Cmp(RegisterFrom(src_pos, invoke->InputAt(1)->GetType()), in VisitSystemArrayCopy()
2728 OperandFrom(dest_pos, invoke->InputAt(3)->GetType())); in VisitSystemArrayCopy()
2793 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitSystemArrayCopy()
2802 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitSystemArrayCopy()
2819 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitSystemArrayCopy()
2836 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitSystemArrayCopy()
2855 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitSystemArrayCopy()
2869 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitSystemArrayCopy()
2953 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitSystemArrayCopy()
2961 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitSystemArrayCopy()
3070 invoke, LocationFrom(tmp)); in VisitSystemArrayCopy()
3152 void IntrinsicLocationsBuilderARM64::VisitFloatIsInfinite(HInvoke* invoke) { in VisitFloatIsInfinite() argument
3153 CreateFPToIntLocations(allocator_, invoke); in VisitFloatIsInfinite()
3156 void IntrinsicCodeGeneratorARM64::VisitFloatIsInfinite(HInvoke* invoke) { in VisitFloatIsInfinite() argument
3157 GenIsInfinite(invoke->GetLocations(), /* is64bit= */ false, GetVIXLAssembler()); in VisitFloatIsInfinite()
3160 void IntrinsicLocationsBuilderARM64::VisitDoubleIsInfinite(HInvoke* invoke) { in VisitDoubleIsInfinite() argument
3161 CreateFPToIntLocations(allocator_, invoke); in VisitDoubleIsInfinite()
3164 void IntrinsicCodeGeneratorARM64::VisitDoubleIsInfinite(HInvoke* invoke) { in VisitDoubleIsInfinite() argument
3165 GenIsInfinite(invoke->GetLocations(), /* is64bit= */ true, GetVIXLAssembler()); in VisitDoubleIsInfinite()
3168 void IntrinsicLocationsBuilderARM64::VisitIntegerValueOf(HInvoke* invoke) { in VisitIntegerValueOf() argument
3171 invoke, in VisitIntegerValueOf()
3177 void IntrinsicCodeGeneratorARM64::VisitIntegerValueOf(HInvoke* invoke) { in VisitIntegerValueOf() argument
3179 IntrinsicVisitor::ComputeIntegerValueOfInfo(invoke, codegen_->GetCompilerOptions()); in VisitIntegerValueOf()
3180 LocationSummary* locations = invoke->GetLocations(); in VisitIntegerValueOf()
3188 codegen_->LoadIntrinsicDeclaringClass(out, invoke); in VisitIntegerValueOf()
3189 codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc()); in VisitIntegerValueOf()
3192 if (invoke->InputAt(0)->IsConstant()) { in VisitIntegerValueOf()
3193 int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue(); in VisitIntegerValueOf()
3234 void IntrinsicLocationsBuilderARM64::VisitReferenceGetReferent(HInvoke* invoke) { in VisitReferenceGetReferent() argument
3235 IntrinsicVisitor::CreateReferenceGetReferentLocations(invoke, codegen_); in VisitReferenceGetReferent()
3237 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier && invoke->GetLocations() != nullptr) { in VisitReferenceGetReferent()
3238 invoke->GetLocations()->AddTemp(Location::RequiresRegister()); in VisitReferenceGetReferent()
3242 void IntrinsicCodeGeneratorARM64::VisitReferenceGetReferent(HInvoke* invoke) { in VisitReferenceGetReferent() argument
3244 LocationSummary* locations = invoke->GetLocations(); in VisitReferenceGetReferent()
3249 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); in VisitReferenceGetReferent()
3265 codegen_->LoadIntrinsicDeclaringClass(temp, invoke); in VisitReferenceGetReferent()
3279 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitReferenceGetReferent()
3289 invoke, DataType::Type::kReference, WRegisterFrom(out), field, /*needs_null_check=*/ true); in VisitReferenceGetReferent()
3290 codegen_->MaybeGenerateReadBarrierSlow(invoke, out, out, obj, referent_offset); in VisitReferenceGetReferent()
3295 void IntrinsicLocationsBuilderARM64::VisitReferenceRefersTo(HInvoke* invoke) { in VisitReferenceRefersTo() argument
3296 IntrinsicVisitor::CreateReferenceRefersToLocations(invoke); in VisitReferenceRefersTo()
3299 void IntrinsicCodeGeneratorARM64::VisitReferenceRefersTo(HInvoke* invoke) { in VisitReferenceRefersTo() argument
3300 LocationSummary* locations = invoke->GetLocations(); in VisitReferenceRefersTo()
3313 codegen_->LoadAcquire(invoke, DataType::Type::kReference, tmp, field, /*needs_null_check=*/ true); in VisitReferenceRefersTo()
3351 void IntrinsicLocationsBuilderARM64::VisitThreadInterrupted(HInvoke* invoke) { in VisitThreadInterrupted() argument
3353 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitThreadInterrupted()
3357 void IntrinsicCodeGeneratorARM64::VisitThreadInterrupted(HInvoke* invoke) { in VisitThreadInterrupted() argument
3359 Register out = RegisterFrom(invoke->GetLocations()->Out(), DataType::Type::kInt32); in VisitThreadInterrupted()
3372 void IntrinsicLocationsBuilderARM64::VisitReachabilityFence(HInvoke* invoke) { in VisitReachabilityFence() argument
3374 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitReachabilityFence()
3378 void IntrinsicCodeGeneratorARM64::VisitReachabilityFence(HInvoke* invoke ATTRIBUTE_UNUSED) { } in VisitReachabilityFence()
3380 void IntrinsicLocationsBuilderARM64::VisitCRC32Update(HInvoke* invoke) { in VisitCRC32Update() argument
3385 LocationSummary* locations = new (allocator_) LocationSummary(invoke, in VisitCRC32Update()
3395 void IntrinsicCodeGeneratorARM64::VisitCRC32Update(HInvoke* invoke) { in VisitCRC32Update() argument
3400 Register crc = InputRegisterAt(invoke, 0); in VisitCRC32Update()
3401 Register val = InputRegisterAt(invoke, 1); in VisitCRC32Update()
3402 Register out = OutputRegister(invoke); in VisitCRC32Update()
3526 void IntrinsicLocationsBuilderARM64::VisitCRC32UpdateBytes(HInvoke* invoke) { in VisitCRC32UpdateBytes() argument
3532 new (allocator_) LocationSummary(invoke, in VisitCRC32UpdateBytes()
3538 locations->SetInAt(2, Location::RegisterOrConstant(invoke->InputAt(2))); in VisitCRC32UpdateBytes()
3547 void IntrinsicCodeGeneratorARM64::VisitCRC32UpdateBytes(HInvoke* invoke) { in VisitCRC32UpdateBytes() argument
3551 LocationSummary* locations = invoke->GetLocations(); in VisitCRC32UpdateBytes()
3554 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitCRC32UpdateBytes()
3582 void IntrinsicLocationsBuilderARM64::VisitCRC32UpdateByteBuffer(HInvoke* invoke) { in VisitCRC32UpdateByteBuffer() argument
3588 new (allocator_) LocationSummary(invoke, in VisitCRC32UpdateByteBuffer()
3609 void IntrinsicCodeGeneratorARM64::VisitCRC32UpdateByteBuffer(HInvoke* invoke) { in VisitCRC32UpdateByteBuffer() argument
3613 LocationSummary* locations = invoke->GetLocations(); in VisitCRC32UpdateByteBuffer()
3625 void IntrinsicLocationsBuilderARM64::VisitFP16ToFloat(HInvoke* invoke) { in VisitFP16ToFloat() argument
3630 LocationSummary* locations = new (allocator_) LocationSummary(invoke, in VisitFP16ToFloat()
3637 void IntrinsicCodeGeneratorARM64::VisitFP16ToFloat(HInvoke* invoke) { in VisitFP16ToFloat() argument
3641 Register bits = InputRegisterAt(invoke, 0); in VisitFP16ToFloat()
3642 VRegister out = SRegisterFrom(invoke->GetLocations()->Out()); in VisitFP16ToFloat()
3648 void IntrinsicLocationsBuilderARM64::VisitFP16ToHalf(HInvoke* invoke) { in VisitFP16ToHalf() argument
3653 LocationSummary* locations = new (allocator_) LocationSummary(invoke, in VisitFP16ToHalf()
3660 void IntrinsicCodeGeneratorARM64::VisitFP16ToHalf(HInvoke* invoke) { in VisitFP16ToHalf() argument
3664 VRegister in = SRegisterFrom(invoke->GetLocations()->InAt(0)); in VisitFP16ToHalf()
3666 Register out = WRegisterFrom(invoke->GetLocations()->Out()); in VisitFP16ToHalf()
3673 void GenerateFP16Round(HInvoke* invoke, in GenerateFP16Round() argument
3678 LocationSummary* locations = invoke->GetLocations(); in GenerateFP16Round()
3688 void IntrinsicLocationsBuilderARM64::VisitFP16Floor(HInvoke* invoke) { in VisitFP16Floor() argument
3693 CreateIntToIntLocations(allocator_, invoke); in VisitFP16Floor()
3696 void IntrinsicCodeGeneratorARM64::VisitFP16Floor(HInvoke* invoke) { in VisitFP16Floor() argument
3701 GenerateFP16Round(invoke, codegen_, masm, roundOp); in VisitFP16Floor()
3704 void IntrinsicLocationsBuilderARM64::VisitFP16Ceil(HInvoke* invoke) { in VisitFP16Ceil() argument
3709 CreateIntToIntLocations(allocator_, invoke); in VisitFP16Ceil()
3712 void IntrinsicCodeGeneratorARM64::VisitFP16Ceil(HInvoke* invoke) { in VisitFP16Ceil() argument
3717 GenerateFP16Round(invoke, codegen_, masm, roundOp); in VisitFP16Ceil()
3720 void IntrinsicLocationsBuilderARM64::VisitFP16Rint(HInvoke* invoke) { in VisitFP16Rint() argument
3725 CreateIntToIntLocations(allocator_, invoke); in VisitFP16Rint()
3728 void IntrinsicCodeGeneratorARM64::VisitFP16Rint(HInvoke* invoke) { in VisitFP16Rint() argument
3733 GenerateFP16Round(invoke, codegen_, masm, roundOp); in VisitFP16Rint()
3737 void GenerateFP16Compare(HInvoke* invoke, in GenerateFP16Compare() argument
3742 LocationSummary* locations = invoke->GetLocations(); in GenerateFP16Compare()
3751 static inline void GenerateFP16Compare(HInvoke* invoke, in GenerateFP16Compare() argument
3759 GenerateFP16Compare(invoke, codegen, masm, compareOp); in GenerateFP16Compare()
3762 void IntrinsicLocationsBuilderARM64::VisitFP16Greater(HInvoke* invoke) { in VisitFP16Greater() argument
3767 CreateIntIntToIntLocations(allocator_, invoke); in VisitFP16Greater()
3768 invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); in VisitFP16Greater()
3769 invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); in VisitFP16Greater()
3772 void IntrinsicCodeGeneratorARM64::VisitFP16Greater(HInvoke* invoke) { in VisitFP16Greater() argument
3774 GenerateFP16Compare(invoke, codegen_, masm, gt); in VisitFP16Greater()
3777 void IntrinsicLocationsBuilderARM64::VisitFP16GreaterEquals(HInvoke* invoke) { in VisitFP16GreaterEquals() argument
3782 CreateIntIntToIntLocations(allocator_, invoke); in VisitFP16GreaterEquals()
3783 invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); in VisitFP16GreaterEquals()
3784 invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); in VisitFP16GreaterEquals()
3787 void IntrinsicCodeGeneratorARM64::VisitFP16GreaterEquals(HInvoke* invoke) { in VisitFP16GreaterEquals() argument
3789 GenerateFP16Compare(invoke, codegen_, masm, ge); in VisitFP16GreaterEquals()
3792 void IntrinsicLocationsBuilderARM64::VisitFP16Less(HInvoke* invoke) { in VisitFP16Less() argument
3797 CreateIntIntToIntLocations(allocator_, invoke); in VisitFP16Less()
3798 invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); in VisitFP16Less()
3799 invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); in VisitFP16Less()
3802 void IntrinsicCodeGeneratorARM64::VisitFP16Less(HInvoke* invoke) { in VisitFP16Less() argument
3804 GenerateFP16Compare(invoke, codegen_, masm, mi); in VisitFP16Less()
3807 void IntrinsicLocationsBuilderARM64::VisitFP16LessEquals(HInvoke* invoke) { in VisitFP16LessEquals() argument
3812 CreateIntIntToIntLocations(allocator_, invoke); in VisitFP16LessEquals()
3813 invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); in VisitFP16LessEquals()
3814 invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); in VisitFP16LessEquals()
3817 void IntrinsicCodeGeneratorARM64::VisitFP16LessEquals(HInvoke* invoke) { in VisitFP16LessEquals() argument
3819 GenerateFP16Compare(invoke, codegen_, masm, ls); in VisitFP16LessEquals()
3822 static void GenerateDivideUnsigned(HInvoke* invoke, CodeGeneratorARM64* codegen) { in GenerateDivideUnsigned() argument
3823 LocationSummary* locations = invoke->GetLocations(); in GenerateDivideUnsigned()
3825 DataType::Type type = invoke->GetType(); in GenerateDivideUnsigned()
3834 new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in GenerateDivideUnsigned()
3843 void IntrinsicLocationsBuilderARM64::VisitIntegerDivideUnsigned(HInvoke* invoke) { in VisitIntegerDivideUnsigned() argument
3844 CreateIntIntToIntSlowPathCallLocations(allocator_, invoke); in VisitIntegerDivideUnsigned()
3847 void IntrinsicCodeGeneratorARM64::VisitIntegerDivideUnsigned(HInvoke* invoke) { in VisitIntegerDivideUnsigned() argument
3848 GenerateDivideUnsigned(invoke, codegen_); in VisitIntegerDivideUnsigned()
3851 void IntrinsicLocationsBuilderARM64::VisitLongDivideUnsigned(HInvoke* invoke) { in VisitLongDivideUnsigned() argument
3852 CreateIntIntToIntSlowPathCallLocations(allocator_, invoke); in VisitLongDivideUnsigned()
3855 void IntrinsicCodeGeneratorARM64::VisitLongDivideUnsigned(HInvoke* invoke) { in VisitLongDivideUnsigned() argument
3856 GenerateDivideUnsigned(invoke, codegen_); in VisitLongDivideUnsigned()
3859 void IntrinsicLocationsBuilderARM64::VisitMathMultiplyHigh(HInvoke* invoke) { in VisitMathMultiplyHigh() argument
3860 CreateIntIntToIntLocations(allocator_, invoke); in VisitMathMultiplyHigh()
3863 void IntrinsicCodeGeneratorARM64::VisitMathMultiplyHigh(HInvoke* invoke) { in VisitMathMultiplyHigh() argument
3864 LocationSummary* locations = invoke->GetLocations(); in VisitMathMultiplyHigh()
3866 DataType::Type type = invoke->GetType(); in VisitMathMultiplyHigh()
3878 VarHandleSlowPathARM64(HInvoke* invoke, std::memory_order order) in VarHandleSlowPathARM64() argument
3879 : IntrinsicSlowPathARM64(invoke), in VarHandleSlowPathARM64()
3973 static void GenerateVarHandleAccessModeAndVarTypeChecks(HInvoke* invoke, in GenerateVarHandleAccessModeAndVarTypeChecks() argument
3978 mirror::VarHandle::GetAccessModeByIntrinsic(invoke->GetIntrinsic()); in GenerateVarHandleAccessModeAndVarTypeChecks()
3982 Register varhandle = InputRegisterAt(invoke, 0); in GenerateVarHandleAccessModeAndVarTypeChecks()
4014 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleAccessModeAndVarTypeChecks()
4016 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in GenerateVarHandleAccessModeAndVarTypeChecks()
4018 HInstruction* arg = invoke->InputAt(arg_index); in GenerateVarHandleAccessModeAndVarTypeChecks()
4021 Register arg_reg = WRegisterFrom(invoke->GetLocations()->InAt(arg_index)); in GenerateVarHandleAccessModeAndVarTypeChecks()
4028 static void GenerateVarHandleStaticFieldCheck(HInvoke* invoke, in GenerateVarHandleStaticFieldCheck() argument
4032 Register varhandle = InputRegisterAt(invoke, 0); in GenerateVarHandleStaticFieldCheck()
4045 static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, in GenerateVarHandleInstanceFieldChecks() argument
4049 Register varhandle = InputRegisterAt(invoke, 0); in GenerateVarHandleInstanceFieldChecks()
4050 Register object = InputRegisterAt(invoke, 1); in GenerateVarHandleInstanceFieldChecks()
4078 static DataType::Type GetVarHandleExpectedValueType(HInvoke* invoke, in GetVarHandleExpectedValueType() argument
4080 DCHECK_EQ(expected_coordinates_count, GetExpectedVarHandleCoordinatesCount(invoke)); in GetVarHandleExpectedValueType()
4081 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in GetVarHandleExpectedValueType()
4084 return invoke->GetType(); in GetVarHandleExpectedValueType()
4086 return GetDataTypeFromShorty(invoke, number_of_arguments - 1u); in GetVarHandleExpectedValueType()
4090 static void GenerateVarHandleArrayChecks(HInvoke* invoke, in GenerateVarHandleArrayChecks() argument
4094 Register varhandle = InputRegisterAt(invoke, 0); in GenerateVarHandleArrayChecks()
4095 Register object = InputRegisterAt(invoke, 1); in GenerateVarHandleArrayChecks()
4096 Register index = InputRegisterAt(invoke, 2); in GenerateVarHandleArrayChecks()
4098 GetVarHandleExpectedValueType(invoke, /*expected_coordinates_count=*/ 2u); in GenerateVarHandleArrayChecks()
4173 static void GenerateVarHandleCoordinateChecks(HInvoke* invoke, in GenerateVarHandleCoordinateChecks() argument
4176 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleCoordinateChecks()
4178 GenerateVarHandleStaticFieldCheck(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4180 GenerateVarHandleInstanceFieldChecks(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4183 GenerateVarHandleArrayChecks(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4187 static VarHandleSlowPathARM64* GenerateVarHandleChecks(HInvoke* invoke, in GenerateVarHandleChecks() argument
4192 new (codegen->GetScopedAllocator()) VarHandleSlowPathARM64(invoke, order); in GenerateVarHandleChecks()
4195 GenerateVarHandleAccessModeAndVarTypeChecks(invoke, codegen, slow_path, type); in GenerateVarHandleChecks()
4196 GenerateVarHandleCoordinateChecks(invoke, codegen, slow_path); in GenerateVarHandleChecks()
4206 static VarHandleTarget GetVarHandleTarget(HInvoke* invoke) { in GetVarHandleTarget() argument
4207 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GetVarHandleTarget()
4208 LocationSummary* locations = invoke->GetLocations(); in GetVarHandleTarget()
4216 : InputRegisterAt(invoke, 1); in GetVarHandleTarget()
4220 static void GenerateVarHandleTarget(HInvoke* invoke, in GenerateVarHandleTarget() argument
4224 Register varhandle = InputRegisterAt(invoke, 0); in GenerateVarHandleTarget()
4225 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleTarget()
4241 codegen->GenerateGcRootFieldLoad(invoke, in GenerateVarHandleTarget()
4251 GetVarHandleExpectedValueType(invoke, /*expected_coordinates_count=*/ 2u); in GenerateVarHandleTarget()
4255 Register index = InputRegisterAt(invoke, 2); in GenerateVarHandleTarget()
4265 static bool HasVarHandleIntrinsicImplementation(HInvoke* invoke) { in HasVarHandleIntrinsicImplementation() argument
4266 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in HasVarHandleIntrinsicImplementation()
4272 invoke->InputAt(1)->GetType() != DataType::Type::kReference) { in HasVarHandleIntrinsicImplementation()
4280 DataType::Type index_type = GetDataTypeFromShorty(invoke, 2); in HasVarHandleIntrinsicImplementation()
4287 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in HasVarHandleIntrinsicImplementation()
4288 DataType::Type return_type = invoke->GetType(); in HasVarHandleIntrinsicImplementation()
4290 mirror::VarHandle::GetAccessModeTemplateByIntrinsic(invoke->GetIntrinsic()); in HasVarHandleIntrinsicImplementation()
4309 DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index); in HasVarHandleIntrinsicImplementation()
4310 DataType::Type new_value_type = GetDataTypeFromShorty(invoke, new_value_index); in HasVarHandleIntrinsicImplementation()
4319 DataType::Type expected_value_type = GetDataTypeFromShorty(invoke, expected_value_index); in HasVarHandleIntrinsicImplementation()
4320 DataType::Type new_value_type = GetDataTypeFromShorty(invoke, new_value_index); in HasVarHandleIntrinsicImplementation()
4327 DataType::Type value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1); in HasVarHandleIntrinsicImplementation()
4328 if (IsVarHandleGetAndAdd(invoke) && in HasVarHandleIntrinsicImplementation()
4332 } else if (IsVarHandleGetAndBitwiseOp(invoke) && !DataType::IsIntegralType(value_type)) { in HasVarHandleIntrinsicImplementation()
4350 static LocationSummary* CreateVarHandleCommonLocations(HInvoke* invoke) { in CreateVarHandleCommonLocations() argument
4351 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in CreateVarHandleCommonLocations()
4352 DataType::Type return_type = invoke->GetType(); in CreateVarHandleCommonLocations()
4354 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in CreateVarHandleCommonLocations()
4356 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateVarHandleCommonLocations()
4371 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleCommonLocations()
4373 HInstruction* arg = invoke->InputAt(arg_index); in CreateVarHandleCommonLocations()
4385 GetExpectedVarHandleCoordinatesCount(invoke) == 0u) { // For static fields. in CreateVarHandleCommonLocations()
4401 static void CreateVarHandleGetLocations(HInvoke* invoke) { in CreateVarHandleGetLocations() argument
4402 if (!HasVarHandleIntrinsicImplementation(invoke)) { in CreateVarHandleGetLocations()
4407 invoke->GetType() == DataType::Type::kReference && in CreateVarHandleGetLocations()
4408 invoke->GetIntrinsic() != Intrinsics::kVarHandleGet && in CreateVarHandleGetLocations()
4409 invoke->GetIntrinsic() != Intrinsics::kVarHandleGetOpaque) { in CreateVarHandleGetLocations()
4416 CreateVarHandleCommonLocations(invoke); in CreateVarHandleGetLocations()
4419 static void GenerateVarHandleGet(HInvoke* invoke, in GenerateVarHandleGet() argument
4423 DataType::Type type = invoke->GetType(); in GenerateVarHandleGet()
4426 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGet()
4428 CPURegister out = helpers::OutputCPURegister(invoke); in GenerateVarHandleGet()
4430 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleGet()
4433 slow_path = GenerateVarHandleChecks(invoke, codegen, order, type); in GenerateVarHandleGet()
4434 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleGet()
4449 codegen->GenerateFieldLoadWithBakerReadBarrier(invoke, in GenerateVarHandleGet()
4474 codegen->LoadAcquire(invoke, load_type, load_reg, address, /*needs_null_check=*/ false); in GenerateVarHandleGet()
4484 codegen->MaybeGenerateReadBarrierSlow(invoke, out_loc, out_loc, object_loc, 0u, offset_loc); in GenerateVarHandleGet()
4495 void IntrinsicLocationsBuilderARM64::VisitVarHandleGet(HInvoke* invoke) { in VisitVarHandleGet() argument
4496 CreateVarHandleGetLocations(invoke); in VisitVarHandleGet()
4499 void IntrinsicCodeGeneratorARM64::VisitVarHandleGet(HInvoke* invoke) { in VisitVarHandleGet() argument
4500 GenerateVarHandleGet(invoke, codegen_, std::memory_order_relaxed); in VisitVarHandleGet()
4503 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetOpaque(HInvoke* invoke) { in VisitVarHandleGetOpaque() argument
4504 CreateVarHandleGetLocations(invoke); in VisitVarHandleGetOpaque()
4507 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetOpaque(HInvoke* invoke) { in VisitVarHandleGetOpaque() argument
4508 GenerateVarHandleGet(invoke, codegen_, std::memory_order_relaxed); in VisitVarHandleGetOpaque()
4511 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAcquire(HInvoke* invoke) { in VisitVarHandleGetAcquire() argument
4512 CreateVarHandleGetLocations(invoke); in VisitVarHandleGetAcquire()
4515 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAcquire(HInvoke* invoke) { in VisitVarHandleGetAcquire() argument
4516 GenerateVarHandleGet(invoke, codegen_, std::memory_order_acquire); in VisitVarHandleGetAcquire()
4519 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetVolatile(HInvoke* invoke) { in VisitVarHandleGetVolatile() argument
4520 CreateVarHandleGetLocations(invoke); in VisitVarHandleGetVolatile()
4523 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetVolatile(HInvoke* invoke) { in VisitVarHandleGetVolatile() argument
4524 GenerateVarHandleGet(invoke, codegen_, std::memory_order_seq_cst); in VisitVarHandleGetVolatile()
4527 static void CreateVarHandleSetLocations(HInvoke* invoke) { in CreateVarHandleSetLocations() argument
4528 if (!HasVarHandleIntrinsicImplementation(invoke)) { in CreateVarHandleSetLocations()
4532 CreateVarHandleCommonLocations(invoke); in CreateVarHandleSetLocations()
4535 static void GenerateVarHandleSet(HInvoke* invoke, in GenerateVarHandleSet() argument
4539 uint32_t value_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleSet()
4540 DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index); in GenerateVarHandleSet()
4543 CPURegister value = InputCPURegisterOrZeroRegAt(invoke, value_index); in GenerateVarHandleSet()
4545 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleSet()
4548 slow_path = GenerateVarHandleChecks(invoke, codegen, order, value_type); in GenerateVarHandleSet()
4549 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleSet()
4585 codegen->StoreRelease(invoke, value_type, source, address, /*needs_null_check=*/ false); in GenerateVarHandleSet()
4591 if (CodeGenerator::StoreNeedsWriteBarrier(value_type, invoke->InputAt(value_index))) { in GenerateVarHandleSet()
4600 void IntrinsicLocationsBuilderARM64::VisitVarHandleSet(HInvoke* invoke) { in VisitVarHandleSet() argument
4601 CreateVarHandleSetLocations(invoke); in VisitVarHandleSet()
4604 void IntrinsicCodeGeneratorARM64::VisitVarHandleSet(HInvoke* invoke) { in VisitVarHandleSet() argument
4605 GenerateVarHandleSet(invoke, codegen_, std::memory_order_relaxed); in VisitVarHandleSet()
4608 void IntrinsicLocationsBuilderARM64::VisitVarHandleSetOpaque(HInvoke* invoke) { in VisitVarHandleSetOpaque() argument
4609 CreateVarHandleSetLocations(invoke); in VisitVarHandleSetOpaque()
4612 void IntrinsicCodeGeneratorARM64::VisitVarHandleSetOpaque(HInvoke* invoke) { in VisitVarHandleSetOpaque() argument
4613 GenerateVarHandleSet(invoke, codegen_, std::memory_order_relaxed); in VisitVarHandleSetOpaque()
4616 void IntrinsicLocationsBuilderARM64::VisitVarHandleSetRelease(HInvoke* invoke) { in VisitVarHandleSetRelease() argument
4617 CreateVarHandleSetLocations(invoke); in VisitVarHandleSetRelease()
4620 void IntrinsicCodeGeneratorARM64::VisitVarHandleSetRelease(HInvoke* invoke) { in VisitVarHandleSetRelease() argument
4621 GenerateVarHandleSet(invoke, codegen_, std::memory_order_release); in VisitVarHandleSetRelease()
4624 void IntrinsicLocationsBuilderARM64::VisitVarHandleSetVolatile(HInvoke* invoke) { in VisitVarHandleSetVolatile() argument
4625 CreateVarHandleSetLocations(invoke); in VisitVarHandleSetVolatile()
4628 void IntrinsicCodeGeneratorARM64::VisitVarHandleSetVolatile(HInvoke* invoke) { in VisitVarHandleSetVolatile() argument
4629 GenerateVarHandleSet(invoke, codegen_, std::memory_order_seq_cst); in VisitVarHandleSetVolatile()
4632 static void CreateVarHandleCompareAndSetOrExchangeLocations(HInvoke* invoke, bool return_success) { in CreateVarHandleCompareAndSetOrExchangeLocations() argument
4633 if (!HasVarHandleIntrinsicImplementation(invoke)) { in CreateVarHandleCompareAndSetOrExchangeLocations()
4637 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleCompareAndSetOrExchangeLocations()
4638 DataType::Type value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1u); in CreateVarHandleCompareAndSetOrExchangeLocations()
4651 LocationSummary* locations = CreateVarHandleCommonLocations(invoke); in CreateVarHandleCompareAndSetOrExchangeLocations()
4659 if (GetExpectedVarHandleCoordinatesCount(invoke) == 0u) { // For static fields. in CreateVarHandleCompareAndSetOrExchangeLocations()
4671 DCHECK_EQ(old_temp_count, (GetExpectedVarHandleCoordinatesCount(invoke) == 0) ? 2u : 1u); in CreateVarHandleCompareAndSetOrExchangeLocations()
4677 (IsConstantZeroBitPattern(invoke->InputAt(number_of_arguments - 1u)) ? 1u : 0u) + in CreateVarHandleCompareAndSetOrExchangeLocations()
4678 (IsConstantZeroBitPattern(invoke->InputAt(number_of_arguments - 2u)) ? 1u : 0u); in CreateVarHandleCompareAndSetOrExchangeLocations()
4685 !IsConstantZeroBitPattern(invoke->InputAt(number_of_arguments - 2u)) && in CreateVarHandleCompareAndSetOrExchangeLocations()
4686 !IsConstantZeroBitPattern(invoke->InputAt(number_of_arguments - 1u)) && in CreateVarHandleCompareAndSetOrExchangeLocations()
4687 GetExpectedVarHandleCoordinatesCount(invoke) == 2u) { in CreateVarHandleCompareAndSetOrExchangeLocations()
4719 static void GenerateVarHandleCompareAndSetOrExchange(HInvoke* invoke, in GenerateVarHandleCompareAndSetOrExchange() argument
4727 uint32_t expected_index = invoke->GetNumberOfArguments() - 2; in GenerateVarHandleCompareAndSetOrExchange()
4728 uint32_t new_value_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleCompareAndSetOrExchange()
4729 DataType::Type value_type = GetDataTypeFromShorty(invoke, new_value_index); in GenerateVarHandleCompareAndSetOrExchange()
4730 DCHECK_EQ(value_type, GetDataTypeFromShorty(invoke, expected_index)); in GenerateVarHandleCompareAndSetOrExchange()
4733 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleCompareAndSetOrExchange()
4734 CPURegister expected = InputCPURegisterOrZeroRegAt(invoke, expected_index); in GenerateVarHandleCompareAndSetOrExchange()
4735 CPURegister new_value = InputCPURegisterOrZeroRegAt(invoke, new_value_index); in GenerateVarHandleCompareAndSetOrExchange()
4736 CPURegister out = helpers::OutputCPURegister(invoke); in GenerateVarHandleCompareAndSetOrExchange()
4738 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleCompareAndSetOrExchange()
4741 slow_path = GenerateVarHandleChecks(invoke, codegen, order, value_type); in GenerateVarHandleCompareAndSetOrExchange()
4743 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleCompareAndSetOrExchange()
4748 if (CodeGenerator::StoreNeedsWriteBarrier(value_type, invoke->InputAt(new_value_index))) { in GenerateVarHandleCompareAndSetOrExchange()
4844 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleCompareAndSetOrExchange()
4852 invoke, in GenerateVarHandleCompareAndSetOrExchange()
4906 void IntrinsicLocationsBuilderARM64::VisitVarHandleCompareAndExchange(HInvoke* invoke) { in VisitVarHandleCompareAndExchange() argument
4907 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, /*return_success=*/ false); in VisitVarHandleCompareAndExchange()
4910 void IntrinsicCodeGeneratorARM64::VisitVarHandleCompareAndExchange(HInvoke* invoke) { in VisitVarHandleCompareAndExchange() argument
4912 invoke, codegen_, std::memory_order_seq_cst, /*return_success=*/ false, /*strong=*/ true); in VisitVarHandleCompareAndExchange()
4915 void IntrinsicLocationsBuilderARM64::VisitVarHandleCompareAndExchangeAcquire(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeAcquire() argument
4916 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, /*return_success=*/ false); in VisitVarHandleCompareAndExchangeAcquire()
4919 void IntrinsicCodeGeneratorARM64::VisitVarHandleCompareAndExchangeAcquire(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeAcquire() argument
4921 invoke, codegen_, std::memory_order_acquire, /*return_success=*/ false, /*strong=*/ true); in VisitVarHandleCompareAndExchangeAcquire()
4924 void IntrinsicLocationsBuilderARM64::VisitVarHandleCompareAndExchangeRelease(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeRelease() argument
4925 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, /*return_success=*/ false); in VisitVarHandleCompareAndExchangeRelease()
4928 void IntrinsicCodeGeneratorARM64::VisitVarHandleCompareAndExchangeRelease(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeRelease() argument
4930 invoke, codegen_, std::memory_order_release, /*return_success=*/ false, /*strong=*/ true); in VisitVarHandleCompareAndExchangeRelease()
4933 void IntrinsicLocationsBuilderARM64::VisitVarHandleCompareAndSet(HInvoke* invoke) { in VisitVarHandleCompareAndSet() argument
4934 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, /*return_success=*/ true); in VisitVarHandleCompareAndSet()
4937 void IntrinsicCodeGeneratorARM64::VisitVarHandleCompareAndSet(HInvoke* invoke) { in VisitVarHandleCompareAndSet() argument
4939 invoke, codegen_, std::memory_order_seq_cst, /*return_success=*/ true, /*strong=*/ true); in VisitVarHandleCompareAndSet()
4942 void IntrinsicLocationsBuilderARM64::VisitVarHandleWeakCompareAndSet(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSet() argument
4943 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, /*return_success=*/ true); in VisitVarHandleWeakCompareAndSet()
4946 void IntrinsicCodeGeneratorARM64::VisitVarHandleWeakCompareAndSet(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSet() argument
4948 invoke, codegen_, std::memory_order_seq_cst, /*return_success=*/ true, /*strong=*/ false); in VisitVarHandleWeakCompareAndSet()
4951 void IntrinsicLocationsBuilderARM64::VisitVarHandleWeakCompareAndSetAcquire(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetAcquire() argument
4952 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, /*return_success=*/ true); in VisitVarHandleWeakCompareAndSetAcquire()
4955 void IntrinsicCodeGeneratorARM64::VisitVarHandleWeakCompareAndSetAcquire(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetAcquire() argument
4957 invoke, codegen_, std::memory_order_acquire, /*return_success=*/ true, /*strong=*/ false); in VisitVarHandleWeakCompareAndSetAcquire()
4960 void IntrinsicLocationsBuilderARM64::VisitVarHandleWeakCompareAndSetPlain(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetPlain() argument
4961 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, /*return_success=*/ true); in VisitVarHandleWeakCompareAndSetPlain()
4964 void IntrinsicCodeGeneratorARM64::VisitVarHandleWeakCompareAndSetPlain(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetPlain() argument
4966 invoke, codegen_, std::memory_order_relaxed, /*return_success=*/ true, /*strong=*/ false); in VisitVarHandleWeakCompareAndSetPlain()
4969 void IntrinsicLocationsBuilderARM64::VisitVarHandleWeakCompareAndSetRelease(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetRelease() argument
4970 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, /*return_success=*/ true); in VisitVarHandleWeakCompareAndSetRelease()
4973 void IntrinsicCodeGeneratorARM64::VisitVarHandleWeakCompareAndSetRelease(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetRelease() argument
4975 invoke, codegen_, std::memory_order_release, /*return_success=*/ true, /*strong=*/ false); in VisitVarHandleWeakCompareAndSetRelease()
4978 static void CreateVarHandleGetAndUpdateLocations(HInvoke* invoke, in CreateVarHandleGetAndUpdateLocations() argument
4980 if (!HasVarHandleIntrinsicImplementation(invoke)) { in CreateVarHandleGetAndUpdateLocations()
4985 invoke->GetType() == DataType::Type::kReference) { in CreateVarHandleGetAndUpdateLocations()
4992 LocationSummary* locations = CreateVarHandleCommonLocations(invoke); in CreateVarHandleGetAndUpdateLocations()
4995 DCHECK_EQ(old_temp_count, (GetExpectedVarHandleCoordinatesCount(invoke) == 0) ? 2u : 1u); in CreateVarHandleGetAndUpdateLocations()
4996 if (DataType::IsFloatingPointType(invoke->GetType())) { in CreateVarHandleGetAndUpdateLocations()
4999 locations->SetInAt(invoke->GetNumberOfArguments() - 1u, Location::RequiresFpuRegister()); in CreateVarHandleGetAndUpdateLocations()
5004 !IsConstantZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { in CreateVarHandleGetAndUpdateLocations()
5014 GetExpectedVarHandleCoordinatesCount(invoke) == 2u && in CreateVarHandleGetAndUpdateLocations()
5015 !IsConstantZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { in CreateVarHandleGetAndUpdateLocations()
5017 GetVarHandleExpectedValueType(invoke, /*expected_coordinates_count=*/ 2u); in CreateVarHandleGetAndUpdateLocations()
5024 static void GenerateVarHandleGetAndUpdate(HInvoke* invoke, in GenerateVarHandleGetAndUpdate() argument
5029 uint32_t arg_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleGetAndUpdate()
5030 DataType::Type value_type = GetDataTypeFromShorty(invoke, arg_index); in GenerateVarHandleGetAndUpdate()
5033 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGetAndUpdate()
5034 CPURegister arg = InputCPURegisterOrZeroRegAt(invoke, arg_index); in GenerateVarHandleGetAndUpdate()
5035 CPURegister out = helpers::OutputCPURegister(invoke); in GenerateVarHandleGetAndUpdate()
5037 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleGetAndUpdate()
5040 slow_path = GenerateVarHandleChecks(invoke, codegen, order, value_type); in GenerateVarHandleGetAndUpdate()
5042 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleGetAndUpdate()
5047 if (CodeGenerator::StoreNeedsWriteBarrier(value_type, invoke->InputAt(arg_index))) { in GenerateVarHandleGetAndUpdate()
5141 invoke, in GenerateVarHandleGetAndUpdate()
5155 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndSet(HInvoke* invoke) { in VisitVarHandleGetAndSet() argument
5156 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kSet); in VisitVarHandleGetAndSet()
5159 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndSet(HInvoke* invoke) { in VisitVarHandleGetAndSet() argument
5160 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kSet, std::memory_order_seq_cst); in VisitVarHandleGetAndSet()
5163 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndSetAcquire(HInvoke* invoke) { in VisitVarHandleGetAndSetAcquire() argument
5164 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kSet); in VisitVarHandleGetAndSetAcquire()
5167 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndSetAcquire(HInvoke* invoke) { in VisitVarHandleGetAndSetAcquire() argument
5168 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kSet, std::memory_order_acquire); in VisitVarHandleGetAndSetAcquire()
5171 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndSetRelease(HInvoke* invoke) { in VisitVarHandleGetAndSetRelease() argument
5172 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kSet); in VisitVarHandleGetAndSetRelease()
5175 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndSetRelease(HInvoke* invoke) { in VisitVarHandleGetAndSetRelease() argument
5176 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kSet, std::memory_order_release); in VisitVarHandleGetAndSetRelease()
5179 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndAdd(HInvoke* invoke) { in VisitVarHandleGetAndAdd() argument
5180 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kAdd); in VisitVarHandleGetAndAdd()
5183 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndAdd(HInvoke* invoke) { in VisitVarHandleGetAndAdd() argument
5184 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAdd, std::memory_order_seq_cst); in VisitVarHandleGetAndAdd()
5187 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndAddAcquire(HInvoke* invoke) { in VisitVarHandleGetAndAddAcquire() argument
5188 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kAdd); in VisitVarHandleGetAndAddAcquire()
5191 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndAddAcquire(HInvoke* invoke) { in VisitVarHandleGetAndAddAcquire() argument
5192 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAdd, std::memory_order_acquire); in VisitVarHandleGetAndAddAcquire()
5195 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndAddRelease(HInvoke* invoke) { in VisitVarHandleGetAndAddRelease() argument
5196 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kAdd); in VisitVarHandleGetAndAddRelease()
5199 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndAddRelease(HInvoke* invoke) { in VisitVarHandleGetAndAddRelease() argument
5200 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAdd, std::memory_order_release); in VisitVarHandleGetAndAddRelease()
5203 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseAnd(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAnd() argument
5204 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kAnd); in VisitVarHandleGetAndBitwiseAnd()
5207 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseAnd(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAnd() argument
5208 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAnd, std::memory_order_seq_cst); in VisitVarHandleGetAndBitwiseAnd()
5211 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseAndAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndAcquire() argument
5212 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kAnd); in VisitVarHandleGetAndBitwiseAndAcquire()
5215 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseAndAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndAcquire() argument
5216 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAnd, std::memory_order_acquire); in VisitVarHandleGetAndBitwiseAndAcquire()
5219 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseAndRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndRelease() argument
5220 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kAnd); in VisitVarHandleGetAndBitwiseAndRelease()
5223 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseAndRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndRelease() argument
5224 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAnd, std::memory_order_release); in VisitVarHandleGetAndBitwiseAndRelease()
5227 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseOr(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOr() argument
5228 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kOr); in VisitVarHandleGetAndBitwiseOr()
5231 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseOr(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOr() argument
5232 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kOr, std::memory_order_seq_cst); in VisitVarHandleGetAndBitwiseOr()
5235 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseOrAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrAcquire() argument
5236 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kOr); in VisitVarHandleGetAndBitwiseOrAcquire()
5239 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseOrAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrAcquire() argument
5240 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kOr, std::memory_order_acquire); in VisitVarHandleGetAndBitwiseOrAcquire()
5243 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseOrRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrRelease() argument
5244 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kOr); in VisitVarHandleGetAndBitwiseOrRelease()
5247 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseOrRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrRelease() argument
5248 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kOr, std::memory_order_release); in VisitVarHandleGetAndBitwiseOrRelease()
5251 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseXor(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXor() argument
5252 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kXor); in VisitVarHandleGetAndBitwiseXor()
5255 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseXor(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXor() argument
5256 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kXor, std::memory_order_seq_cst); in VisitVarHandleGetAndBitwiseXor()
5259 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseXorAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorAcquire() argument
5260 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kXor); in VisitVarHandleGetAndBitwiseXorAcquire()
5263 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseXorAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorAcquire() argument
5264 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kXor, std::memory_order_acquire); in VisitVarHandleGetAndBitwiseXorAcquire()
5267 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseXorRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorRelease() argument
5268 CreateVarHandleGetAndUpdateLocations(invoke, GetAndUpdateOp::kXor); in VisitVarHandleGetAndBitwiseXorRelease()
5271 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseXorRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorRelease() argument
5272 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kXor, std::memory_order_release); in VisitVarHandleGetAndBitwiseXorRelease()
5279 HInvoke* invoke = GetInvoke(); in EmitByteArrayViewCode() local
5282 GetVarHandleExpectedValueType(invoke, /*expected_coordinates_count=*/ 2u); in EmitByteArrayViewCode()
5286 Register varhandle = InputRegisterAt(invoke, 0); in EmitByteArrayViewCode()
5287 Register object = InputRegisterAt(invoke, 1); in EmitByteArrayViewCode()
5288 Register index = InputRegisterAt(invoke, 2); in EmitByteArrayViewCode()
5297 VarHandleTarget target = GetVarHandleTarget(invoke); in EmitByteArrayViewCode()
5331 IsConstantZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { in EmitByteArrayViewCode()
5343 GenerateVarHandleGet(invoke, codegen, order_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5346 GenerateVarHandleSet(invoke, codegen, order_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5351 invoke, codegen, order_, return_success_, strong_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5355 invoke, codegen, get_and_update_op_, order_, /*byte_swap=*/ true); in EmitByteArrayViewCode()