1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "fast_compiler.h"
18
19 // TODO(VIXL): Make VIXL compile cleanly with -Wshadow, -Wdeprecated-declarations.
20 #pragma GCC diagnostic push
21 #pragma GCC diagnostic ignored "-Wshadow"
22 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
23 #include "aarch64/disasm-aarch64.h"
24 #include "aarch64/macro-assembler-aarch64.h"
25 #include "aarch64/disasm-aarch64.h"
26 #pragma GCC diagnostic pop
27
28 #include "code_generation_data.h"
29 #include "code_generator_arm64.h"
30 #include "data_type-inl.h"
31 #include "dex/code_item_accessors-inl.h"
32 #include "dex/dex_instruction-inl.h"
33 #include "driver/dex_compilation_unit.h"
34 #include "entrypoints/entrypoint_utils-inl.h"
35 #include "jit_patches_arm64.h"
36 #include "nodes.h"
37 #include "thread-inl.h"
38 #include "utils/arm64/assembler_arm64.h"
39
40
41 using namespace vixl::aarch64; // NOLINT(build/namespaces)
42 using vixl::ExactAssemblyScope;
43 using vixl::CodeBufferCheckScope;
44 using vixl::EmissionCheckScope;
45
46 #ifdef __
47 #error "ARM64 Codegen VIXL macro-assembler macro already defined."
48 #endif
49 #define __ GetVIXLAssembler()->
50
51 namespace art HIDDEN {
52 namespace arm64 {
53
54 using helpers::CPURegisterFrom;
55 using helpers::HeapOperand;
56 using helpers::LocationFrom;
57 using helpers::RegisterFrom;
58 using helpers::WRegisterFrom;
59 using helpers::DRegisterFrom;
60 using helpers::SRegisterFrom;
61
62 static const vixl::aarch64::Register kAvailableCalleeSaveRegisters[] = {
63 vixl::aarch64::x22,
64 vixl::aarch64::x23,
65 vixl::aarch64::x24,
66 vixl::aarch64::x25,
67 vixl::aarch64::x26,
68 vixl::aarch64::x27,
69 vixl::aarch64::x28,
70 vixl::aarch64::x29,
71 };
72
73 static const vixl::aarch64::Register kAvailableTempRegisters[] = {
74 vixl::aarch64::x8,
75 vixl::aarch64::x9,
76 vixl::aarch64::x10,
77 vixl::aarch64::x11,
78 vixl::aarch64::x12,
79 vixl::aarch64::x13,
80 vixl::aarch64::x14,
81 vixl::aarch64::x15,
82 };
83
84 static const vixl::aarch64::VRegister kAvailableCalleeSaveFpuRegisters[] = {
85 vixl::aarch64::d8,
86 vixl::aarch64::d9,
87 vixl::aarch64::d10,
88 vixl::aarch64::d11,
89 vixl::aarch64::d12,
90 vixl::aarch64::d13,
91 vixl::aarch64::d14,
92 vixl::aarch64::d15,
93 };
94
95 static const vixl::aarch64::VRegister kAvailableTempFpuRegisters[] = {
96 vixl::aarch64::d0,
97 vixl::aarch64::d1,
98 vixl::aarch64::d2,
99 vixl::aarch64::d3,
100 vixl::aarch64::d4,
101 vixl::aarch64::d5,
102 vixl::aarch64::d6,
103 vixl::aarch64::d7,
104 };
105
106 class FastCompilerARM64 : public FastCompiler {
107 public:
FastCompilerARM64(ArtMethod * method,ArenaAllocator * allocator,ArenaStack * arena_stack,VariableSizedHandleScope * handles,const CompilerOptions & compiler_options,const DexCompilationUnit & dex_compilation_unit)108 FastCompilerARM64(ArtMethod* method,
109 ArenaAllocator* allocator,
110 ArenaStack* arena_stack,
111 VariableSizedHandleScope* handles,
112 const CompilerOptions& compiler_options,
113 const DexCompilationUnit& dex_compilation_unit)
114 : method_(method),
115 allocator_(allocator),
116 handles_(handles),
117 assembler_(allocator,
118 compiler_options.GetInstructionSetFeatures()->AsArm64InstructionSetFeatures()),
119 jit_patches_(&assembler_, allocator),
120 compiler_options_(compiler_options),
121 dex_compilation_unit_(dex_compilation_unit),
122 code_generation_data_(CodeGenerationData::Create(arena_stack, InstructionSet::kArm64)),
123 vreg_locations_(dex_compilation_unit.GetCodeItemAccessor().RegistersSize(),
124 allocator->Adapter()),
125 branch_targets_(dex_compilation_unit.GetCodeItemAccessor().InsnsSizeInCodeUnits(),
126 allocator->Adapter()),
127 object_register_masks_(dex_compilation_unit.GetCodeItemAccessor().InsnsSizeInCodeUnits(),
128 allocator->Adapter()),
129 is_non_null_masks_(dex_compilation_unit.GetCodeItemAccessor().InsnsSizeInCodeUnits(),
130 allocator->Adapter()),
131 has_frame_(false),
132 core_spill_mask_(0u),
133 fpu_spill_mask_(0u),
134 object_register_mask_(0u),
135 is_non_null_mask_(0u) {
136 memset(is_non_null_masks_.data(), ~0, is_non_null_masks_.size() * sizeof(uint64_t));
137 memset(object_register_masks_.data(), ~0, object_register_masks_.size() * sizeof(uint64_t));
138 GetAssembler()->cfi().SetEnabled(compiler_options.GenerateAnyDebugInfo());
139 }
140
141 // Top-level method to generate code for `method_`.
142 bool Compile();
143
GetCode() const144 ArrayRef<const uint8_t> GetCode() const override {
145 return ArrayRef<const uint8_t>(assembler_.CodeBufferBaseAddress(), assembler_.CodeSize());
146 }
147
BuildStackMaps() const148 ScopedArenaVector<uint8_t> BuildStackMaps() const override {
149 return code_generation_data_->GetStackMapStream()->Encode();
150 }
151
GetCfiData() const152 ArrayRef<const uint8_t> GetCfiData() const override {
153 return ArrayRef<const uint8_t>(*assembler_.cfi().data());
154 }
155
GetFrameSize() const156 int32_t GetFrameSize() const override {
157 if (!has_frame_) {
158 return 0;
159 }
160 size_t size = FrameEntrySpillSize() +
161 /* method */ static_cast<size_t>(kArm64PointerSize) +
162 /* out registers */ GetCodeItemAccessor().OutsSize() * kVRegSize;
163 return RoundUp(size, kStackAlignment);
164 }
165
GetNumberOfJitRoots() const166 uint32_t GetNumberOfJitRoots() const override {
167 return code_generation_data_->GetNumberOfJitRoots();
168 }
169
EmitJitRoots(uint8_t * code,const uint8_t * roots_data,std::vector<Handle<mirror::Object>> * roots)170 void EmitJitRoots(uint8_t* code,
171 const uint8_t* roots_data,
172 /*out*/std::vector<Handle<mirror::Object>>* roots) override
173 REQUIRES_SHARED(Locks::mutator_lock_) {
174 code_generation_data_->EmitJitRoots(roots);
175 jit_patches_.EmitJitRootPatches(code, roots_data, *code_generation_data_);
176 }
177
~FastCompilerARM64()178 ~FastCompilerARM64() override {
179 GetVIXLAssembler()->Reset();
180 }
181
GetUnimplementedReason() const182 const char* GetUnimplementedReason() const {
183 return unimplemented_reason_;
184 }
185
186 private:
187 // Go over each instruction of the method, and generate code for them.
188 bool ProcessInstructions();
189
190 // Initialize the locations of parameters for this method.
191 bool InitializeParameters();
192
193 // Generate code for the frame entry. Only called when needed. If the frame
194 // entry has already been generated, do nothing.
195 bool EnsureHasFrame();
196
197 // Generate code for a frame exit.
198 void PopFrameAndReturn();
199
200 // Record a stack map at the given dex_pc.
201 void RecordPcInfo(uint32_t dex_pc);
202
203 // Generate code to move from one location to another.
204 bool MoveLocation(Location destination, Location source, DataType::Type dst_type);
205
206 // Get a register location for the dex register `reg`. Saves the location into
207 // `vreg_locations_` for next uses of `reg`.
208 // `next` should be the next dex instruction, to help choose the register.
209 Location CreateNewRegisterLocation(uint32_t reg, DataType::Type type, const Instruction* next);
210
211 // Return the existing register location for `reg`.
212 Location GetExistingRegisterLocation(uint32_t reg, DataType::Type type);
213
214 // Move dex registers holding constants into physical registers. Used when
215 // branching.
216 void MoveConstantsToRegisters();
217
218 // Update the masks associated to the given dex_pc. Used when dex_pc is a
219 // branch target.
220 void UpdateMasks(uint32_t dex_pc);
221
222 // Generate code for one instruction.
223 bool ProcessDexInstruction(const Instruction& instruction,
224 uint32_t dex_pc,
225 const Instruction* next);
226
227 // Setup the arguments for an invoke.
228 bool SetupArguments(InvokeType invoke_type,
229 const InstructionOperands& operands,
230 const char* shorty,
231 /* out */ uint32_t* obj_reg);
232
233 // Generate code for doing a Java invoke.
234 bool HandleInvoke(const Instruction& instruction, uint32_t dex_pc, InvokeType invoke_type);
235
236 // Generate code for IF_* instructions.
237 template<vixl::aarch64::Condition kCond, bool kCompareWithZero>
238 bool If_21_22t(const Instruction& instruction, uint32_t dex_pc);
239
240 // Generate code for doing a runtime invoke.
241 void InvokeRuntime(QuickEntrypointEnum entrypoint, uint32_t dex_pc);
242
243 bool BuildLoadString(uint32_t vreg, dex::StringIndex string_index, const Instruction* next);
244 bool BuildNewInstance(
245 uint32_t vreg, dex::TypeIndex string_index, uint32_t dex_pc, const Instruction* next);
246 bool BuildCheckCast(uint32_t vreg, dex::TypeIndex type_index, uint32_t dex_pc);
247 bool LoadMethod(Register reg, ArtMethod* method);
248 void DoReadBarrierOn(Register reg, vixl::aarch64::Label* exit = nullptr, bool do_mr_check = true);
249 bool CanGenerateCodeFor(ArtField* field, bool can_receiver_be_null)
250 REQUIRES_SHARED(Locks::mutator_lock_);
251
252 // Mark whether dex register `vreg_index` is an object.
UpdateRegisterMask(uint32_t vreg_index,bool is_object)253 void UpdateRegisterMask(uint32_t vreg_index, bool is_object) {
254 // Note that the register mask is only useful when there is a frame, so we
255 // use the callee save registers for the mask.
256 if (is_object) {
257 object_register_mask_ |= (1 << kAvailableCalleeSaveRegisters[vreg_index].GetCode());
258 } else {
259 object_register_mask_ &= ~(1 << kAvailableCalleeSaveRegisters[vreg_index].GetCode());
260 }
261 }
262
263 // Mark whether dex register `vreg_index` can be null.
UpdateNonNullMask(uint32_t vreg_index,bool can_be_null)264 void UpdateNonNullMask(uint32_t vreg_index, bool can_be_null) {
265 if (can_be_null) {
266 is_non_null_mask_ &= ~(1 << vreg_index);
267 } else {
268 is_non_null_mask_ |= (1 << vreg_index);
269 }
270 }
271
272 // Update information about dex register `vreg_index`.
UpdateLocal(uint32_t vreg_index,bool is_object,bool can_be_null=true)273 void UpdateLocal(uint32_t vreg_index, bool is_object, bool can_be_null = true) {
274 UpdateRegisterMask(vreg_index, is_object);
275 UpdateNonNullMask(vreg_index, can_be_null);
276 }
277
278 // Whether dex register `vreg_index` can be null.
CanBeNull(uint32_t vreg_index) const279 bool CanBeNull(uint32_t vreg_index) const {
280 return (is_non_null_mask_ & (1 << vreg_index)) == 0;
281 }
282
283 // Get the label associated with the given `dex_pc`.
GetLabelOf(uint32_t dex_pc)284 vixl::aarch64::Label* GetLabelOf(uint32_t dex_pc) {
285 return &branch_targets_[dex_pc];
286 }
287
288 // If we need to abort compilation, clear branch targets, required by vixl.
AbortCompilation()289 void AbortCompilation() {
290 for (vixl::aarch64::Label& label : branch_targets_) {
291 if (label.IsLinked()) {
292 __ Bind(&label);
293 }
294 }
295 }
296
297
298 // Compiler utilities.
299 //
GetAssembler()300 Arm64Assembler* GetAssembler() { return &assembler_; }
GetVIXLAssembler()301 vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
GetDexFile() const302 const DexFile& GetDexFile() const { return *dex_compilation_unit_.GetDexFile(); }
GetCodeItemAccessor() const303 const CodeItemDataAccessor& GetCodeItemAccessor() const {
304 return dex_compilation_unit_.GetCodeItemAccessor();
305 }
HitUnimplemented() const306 bool HitUnimplemented() const {
307 return unimplemented_reason_ != nullptr;
308 }
309
310 // Frame related utilities.
311 //
GetCoreSpillSize() const312 uint32_t GetCoreSpillSize() const {
313 return GetFramePreservedCoreRegisters().GetTotalSizeInBytes();
314 }
FrameEntrySpillSize() const315 uint32_t FrameEntrySpillSize() const {
316 return GetFramePreservedFPRegisters().GetTotalSizeInBytes() + GetCoreSpillSize();
317 }
GetFramePreservedCoreRegisters() const318 CPURegList GetFramePreservedCoreRegisters() const {
319 return CPURegList(CPURegister::kRegister, kXRegSize, core_spill_mask_);
320 }
GetFramePreservedFPRegisters() const321 CPURegList GetFramePreservedFPRegisters() const {
322 return CPURegList(CPURegister::kVRegister, kDRegSize, fpu_spill_mask_);
323 }
324
325 // Method being compiled.
326 ArtMethod* method_;
327
328 // Allocator for any allocation happening in the compiler.
329 ArenaAllocator* allocator_;
330
331 VariableSizedHandleScope* handles_;
332
333 // Compilation utilities.
334 Arm64Assembler assembler_;
335 JitPatchesARM64 jit_patches_;
336 const CompilerOptions& compiler_options_;
337 const DexCompilationUnit& dex_compilation_unit_;
338 std::unique_ptr<CodeGenerationData> code_generation_data_;
339
340 // The current location of each dex register.
341 ArenaVector<Location> vreg_locations_;
342
343 // A vector of size code units for dex pcs that are branch targets.
344 ArenaVector<vixl::aarch64::Label> branch_targets_;
345
346 // For dex pcs that are branch targets, the register mask that will be used at
347 // the point of that pc.
348 ArenaVector<uint64_t> object_register_masks_;
349
350 // For dex pcs that are branch targets, the mask for non-null objects that will
351 // be used at the point of that pc.
352 ArenaVector<uint64_t> is_non_null_masks_;
353
354 // Whether we've created a frame for this compiled method.
355 bool has_frame_;
356
357 // CPU registers that have been spilled in the frame.
358 uint32_t core_spill_mask_;
359
360 // FPU registers that have been spilled in the frame.
361 uint32_t fpu_spill_mask_;
362
363 // The current mask to know which physical register holds an object.
364 uint64_t object_register_mask_;
365
366 // The current mask to know if a dex register is known non-null.
367 uint64_t is_non_null_mask_;
368
369 // The return type of the compiled method. Saved to avoid re-computing it on
370 // the return instruction.
371 DataType::Type return_type_;
372
373 // The return type of the last invoke instruction.
374 DataType::Type previous_invoke_return_type_;
375
376 // If non-empty, the reason the compilation could not be finished.
377 const char* unimplemented_reason_ = nullptr;
378 };
379
InitializeParameters()380 bool FastCompilerARM64::InitializeParameters() {
381 if (GetCodeItemAccessor().TriesSize() != 0) {
382 // TODO: Support try/catch.
383 unimplemented_reason_ = "TryCatch";
384 return false;
385 }
386 const char* shorty = dex_compilation_unit_.GetShorty();
387 uint16_t number_of_vregs = GetCodeItemAccessor().RegistersSize();
388 uint16_t number_of_parameters = GetCodeItemAccessor().InsSize();
389 uint16_t vreg_parameter_index = number_of_vregs - number_of_parameters;
390
391 if (number_of_vregs > arraysize(kAvailableTempRegisters) ||
392 number_of_vregs > arraysize(kAvailableCalleeSaveRegisters) ||
393 number_of_vregs > arraysize(kAvailableTempFpuRegisters) ||
394 number_of_vregs > arraysize(kAvailableCalleeSaveFpuRegisters)) {
395 // Too many registers for this compiler.
396 unimplemented_reason_ = "TooManyRegisters";
397 return false;
398 }
399
400 InvokeDexCallingConventionVisitorARM64 convention;
401 if (!dex_compilation_unit_.IsStatic()) {
402 // Add the implicit 'this' argument, not expressed in the signature.
403 vreg_locations_[vreg_parameter_index] = convention.GetNextLocation(DataType::Type::kReference);
404 UpdateLocal(vreg_parameter_index, /* is_object= */ true, /* can_be_null= */ false);
405 ++vreg_parameter_index;
406 --number_of_parameters;
407 }
408
409 for (int i = 0, shorty_pos = 1;
410 i < number_of_parameters;
411 i++, shorty_pos++, vreg_parameter_index++) {
412 DataType::Type type = DataType::FromShorty(shorty[shorty_pos]);
413 vreg_locations_[vreg_parameter_index] = convention.GetNextLocation(type);
414 UpdateLocal(vreg_parameter_index,
415 /* is_object= */ (type == DataType::Type::kReference),
416 /* can_be_null= */ true);
417 if (DataType::Is64BitType(type)) {
418 ++i;
419 ++vreg_parameter_index;
420 }
421 }
422 return_type_ = DataType::FromShorty(shorty[0]);
423 return true;
424 }
425
MoveConstantsToRegisters()426 void FastCompilerARM64::MoveConstantsToRegisters() {
427 for (uint32_t i = 0; i < vreg_locations_.size(); ++i) {
428 Location location = vreg_locations_[i];
429 if (location.IsConstant()) {
430 vreg_locations_[i] =
431 CreateNewRegisterLocation(i, DataType::Type::kInt32, /* next= */ nullptr);
432 MoveLocation(vreg_locations_[i], location, DataType::Type::kInt32);
433 DCHECK(!HitUnimplemented());
434 }
435 }
436 }
437
UpdateMasks(uint32_t dex_pc)438 void FastCompilerARM64::UpdateMasks(uint32_t dex_pc) {
439 object_register_masks_[dex_pc] &= object_register_mask_;
440 is_non_null_masks_[dex_pc] &= is_non_null_mask_;
441 }
442
ProcessInstructions()443 bool FastCompilerARM64::ProcessInstructions() {
444 DCHECK(GetCodeItemAccessor().HasCodeItem());
445
446 DexInstructionIterator it = GetCodeItemAccessor().begin();
447 DexInstructionIterator end = GetCodeItemAccessor().end();
448 DCHECK(it != end);
449 do {
450 DexInstructionPcPair pair = *it;
451 ++it;
452
453 // Fetch the next instruction as a micro-optimization currently only used
454 // for optimizing returns.
455 const Instruction* next = nullptr;
456 if (it != end) {
457 const DexInstructionPcPair& next_pair = *it;
458 next = &next_pair.Inst();
459 if (GetLabelOf(next_pair.DexPc())->IsLinked()) {
460 // Disable the micro-optimization, as the next instruction is a branch
461 // target.
462 next = nullptr;
463 }
464 }
465 vixl::aarch64::Label* label = GetLabelOf(pair.DexPc());
466 if (label->IsLinked()) {
467 // Emulate a branch to this pc.
468 MoveConstantsToRegisters();
469 UpdateMasks(pair.DexPc());
470 // Set new masks based on all incoming edges.
471 is_non_null_mask_ = is_non_null_masks_[pair.DexPc()];
472 object_register_mask_ = object_register_masks_[pair.DexPc()];
473 __ Bind(label);
474 }
475
476 if (!ProcessDexInstruction(pair.Inst(), pair.DexPc(), next)) {
477 DCHECK(HitUnimplemented());
478 return false;
479 }
480 // Note: There may be no Thread for gtests.
481 DCHECK(Thread::Current() == nullptr || !Thread::Current()->IsExceptionPending())
482 << GetDexFile().PrettyMethod(dex_compilation_unit_.GetDexMethodIndex())
483 << " " << pair.Inst().Name() << "@" << pair.DexPc();
484
485 DCHECK(!HitUnimplemented()) << GetUnimplementedReason();
486 } while (it != end);
487 return true;
488 }
489
MoveLocation(Location destination,Location source,DataType::Type dst_type)490 bool FastCompilerARM64::MoveLocation(Location destination,
491 Location source,
492 DataType::Type dst_type) {
493 if (source.Equals(destination)) {
494 return true;
495 }
496 if (source.IsRegister() && destination.IsRegister()) {
497 CPURegister dst = CPURegisterFrom(destination, dst_type);
498 __ Mov(Register(dst), RegisterFrom(source, dst_type));
499 return true;
500 }
501 if (source.IsConstant() && destination.IsRegister()) {
502 if (source.GetConstant()->IsIntConstant()) {
503 __ Mov(RegisterFrom(destination, DataType::Type::kInt32),
504 source.GetConstant()->AsIntConstant()->GetValue());
505 return true;
506 }
507 }
508 unimplemented_reason_ = "MoveLocation";
509 return false;
510 }
511
CreateNewRegisterLocation(uint32_t reg,DataType::Type type,const Instruction * next)512 Location FastCompilerARM64::CreateNewRegisterLocation(uint32_t reg,
513 DataType::Type type,
514 const Instruction* next) {
515 if (next != nullptr &&
516 (next->Opcode() == Instruction::RETURN_OBJECT || next->Opcode() == Instruction::RETURN) &&
517 (next->VRegA_11x() == reg)) {
518 // If the next instruction is a return, use the return register from the calling
519 // convention.
520 InvokeDexCallingConventionVisitorARM64 convention;
521 vreg_locations_[reg] = convention.GetReturnLocation(return_type_);
522 return vreg_locations_[reg];
523 } else if (vreg_locations_[reg].IsStackSlot() ||
524 vreg_locations_[reg].IsDoubleStackSlot()) {
525 unimplemented_reason_ = "MoveStackSlot";
526 // Return a phony location.
527 return DataType::IsFloatingPointType(type)
528 ? Location::FpuRegisterLocation(1)
529 : Location::RegisterLocation(1);
530 } else if (DataType::IsFloatingPointType(type)) {
531 if (vreg_locations_[reg].IsFpuRegister()) {
532 // Re-use existing register.
533 return vreg_locations_[reg];
534 } else if (has_frame_) {
535 // TODO: Regenerate the method with floating point support.
536 unimplemented_reason_ = "FpuRegisterAllocation";
537 vreg_locations_[reg] = Location::FpuRegisterLocation(1);
538 return vreg_locations_[reg];
539 } else {
540 vreg_locations_[reg] =
541 Location::FpuRegisterLocation(kAvailableTempFpuRegisters[reg].GetCode());
542 return vreg_locations_[reg];
543 }
544 } else if (vreg_locations_[reg].IsRegister()) {
545 // Re-use existing register.
546 return vreg_locations_[reg];
547 } else {
548 // Get the associated register with `reg`.
549 uint32_t register_code = has_frame_
550 ? kAvailableCalleeSaveRegisters[reg].GetCode()
551 : kAvailableTempRegisters[reg].GetCode();
552 vreg_locations_[reg] = Location::RegisterLocation(register_code);
553 return vreg_locations_[reg];
554 }
555 }
556
GetExistingRegisterLocation(uint32_t reg,DataType::Type type)557 Location FastCompilerARM64::GetExistingRegisterLocation(uint32_t reg, DataType::Type type) {
558 if (vreg_locations_[reg].IsStackSlot() || vreg_locations_[reg].IsDoubleStackSlot()) {
559 unimplemented_reason_ = "MoveStackSlot";
560 // Return a phony location.
561 return DataType::IsFloatingPointType(type)
562 ? Location::FpuRegisterLocation(1)
563 : Location::RegisterLocation(1);
564 } else if (DataType::IsFloatingPointType(type)) {
565 if (vreg_locations_[reg].IsFpuRegister()) {
566 return vreg_locations_[reg];
567 } else {
568 // TODO: Regenerate the method with floating point support.
569 unimplemented_reason_ = "FpuRegisterAllocation";
570 vreg_locations_[reg] = Location::FpuRegisterLocation(1);
571 return vreg_locations_[reg];
572 }
573 } else if (vreg_locations_[reg].IsRegister()) {
574 return vreg_locations_[reg];
575 } else {
576 unimplemented_reason_ = "UnknownLocation";
577 vreg_locations_[reg] = Location::RegisterLocation(1);
578 return Location::RegisterLocation(1);
579 }
580 }
581
RecordPcInfo(uint32_t dex_pc)582 void FastCompilerARM64::RecordPcInfo(uint32_t dex_pc) {
583 DCHECK(has_frame_);
584 uint32_t native_pc = GetAssembler()->CodePosition();
585 StackMapStream* stack_map_stream = code_generation_data_->GetStackMapStream();
586 CHECK_EQ(object_register_mask_ & callee_saved_core_registers.GetList(), object_register_mask_);
587 stack_map_stream->BeginStackMapEntry(dex_pc, native_pc, object_register_mask_);
588 stack_map_stream->EndStackMapEntry();
589 }
590
PopFrameAndReturn()591 void FastCompilerARM64::PopFrameAndReturn() {
592 if (has_frame_) {
593 CodeGeneratorARM64::PopFrameAndReturn(GetAssembler(),
594 GetFrameSize(),
595 GetFramePreservedCoreRegisters(),
596 GetFramePreservedFPRegisters());
597 } else {
598 DCHECK_EQ(GetFrameSize(), 0);
599 __ Ret();
600 }
601 }
602
EnsureHasFrame()603 bool FastCompilerARM64::EnsureHasFrame() {
604 if (has_frame_) {
605 // Frame entry has already been generated.
606 return true;
607 }
608 has_frame_ = true;
609 uint16_t number_of_vregs = GetCodeItemAccessor().RegistersSize();
610 for (int i = 0; i < number_of_vregs; ++i) {
611 // Assume any vreg will be held in a callee-save register.
612 core_spill_mask_ |= (1 << kAvailableCalleeSaveRegisters[i].GetCode());
613 if (vreg_locations_[i].IsFpuRegister()) {
614 // TODO: Re-generate method with floating points.
615 unimplemented_reason_ = "FloatingPoint";
616 return false;
617 }
618 }
619 core_spill_mask_ |= (1 << lr.GetCode());
620
621 code_generation_data_->GetStackMapStream()->BeginMethod(GetFrameSize(),
622 core_spill_mask_,
623 fpu_spill_mask_,
624 GetCodeItemAccessor().RegistersSize(),
625 /* is_compiling_baseline= */ true,
626 /* is_debuggable= */ false);
627 MacroAssembler* masm = GetVIXLAssembler();
628 {
629 UseScratchRegisterScope temps(masm);
630 Register temp = temps.AcquireX();
631 __ Sub(temp, sp, static_cast<int32_t>(GetStackOverflowReservedBytes(InstructionSet::kArm64)));
632 // Ensure that between load and RecordPcInfo there are no pools emitted.
633 ExactAssemblyScope eas(GetVIXLAssembler(),
634 kInstructionSize,
635 CodeBufferCheckScope::kExactSize);
636 __ ldr(wzr, MemOperand(temp, 0));
637 RecordPcInfo(0);
638 }
639
640 // Stack layout:
641 // sp[frame_size - 8] : lr.
642 // ... : other preserved core registers.
643 // ... : other preserved fp registers.
644 // ... : reserved frame space.
645 // sp[0] : current method.
646 int32_t frame_size = GetFrameSize();
647 uint32_t core_spills_offset = frame_size - GetCoreSpillSize();
648 CPURegList preserved_core_registers = GetFramePreservedCoreRegisters();
649 DCHECK(!preserved_core_registers.IsEmpty());
650 uint32_t fp_spills_offset = frame_size - FrameEntrySpillSize();
651 CPURegList preserved_fp_registers = GetFramePreservedFPRegisters();
652
653 // Save the current method if we need it, or if using STP reduces code
654 // size. Note that we do not do this in HCurrentMethod, as the
655 // instruction might have been removed in the SSA graph.
656 CPURegister lowest_spill;
657 if (core_spills_offset == kXRegSizeInBytes) {
658 // If there is no gap between the method and the lowest core spill, use
659 // aligned STP pre-index to store both. Max difference is 512. We do
660 // that to reduce code size even if we do not have to save the method.
661 DCHECK_LE(frame_size, 512); // 32 core registers are only 256 bytes.
662 lowest_spill = preserved_core_registers.PopLowestIndex();
663 __ Stp(kArtMethodRegister, lowest_spill, MemOperand(sp, -frame_size, PreIndex));
664 } else {
665 __ Str(kArtMethodRegister, MemOperand(sp, -frame_size, PreIndex));
666 }
667 GetAssembler()->cfi().AdjustCFAOffset(frame_size);
668 if (lowest_spill.IsValid()) {
669 GetAssembler()->cfi().RelOffset(DWARFReg(lowest_spill), core_spills_offset);
670 core_spills_offset += kXRegSizeInBytes;
671 }
672 GetAssembler()->SpillRegisters(preserved_core_registers, core_spills_offset);
673 GetAssembler()->SpillRegisters(preserved_fp_registers, fp_spills_offset);
674
675 // Move registers which are currently allocated from caller-saves to callee-saves.
676 for (int i = 0; i < number_of_vregs; ++i) {
677 if (vreg_locations_[i].IsRegister()) {
678 Location new_location =
679 Location::RegisterLocation(kAvailableCalleeSaveRegisters[i].GetCode());
680 if (!MoveLocation(new_location, vreg_locations_[i], DataType::Type::kInt64)) {
681 return false;
682 }
683 vreg_locations_[i] = new_location;
684 } else if (vreg_locations_[i].IsFpuRegister()) {
685 Location new_location =
686 Location::FpuRegisterLocation(kAvailableCalleeSaveFpuRegisters[i].GetCode());
687 if (!MoveLocation(new_location, vreg_locations_[i], DataType::Type::kFloat64)) {
688 return false;
689 }
690 vreg_locations_[i] = new_location;
691 }
692 }
693
694 // Increment hotness. We use the ArtMethod's counter as we're not allocating a
695 // `ProfilingInfo` object in the fast baseline compiler.
696 if (!Runtime::Current()->IsAotCompiler()) {
697 uint64_t address = reinterpret_cast64<uint64_t>(method_);
698 UseScratchRegisterScope temps(masm);
699 Register counter = temps.AcquireW();
700 vixl::aarch64::Label increment, done;
701 uint32_t entrypoint_offset =
702 GetThreadOffset<kArm64PointerSize>(kQuickCompileOptimized).Int32Value();
703
704 __ Ldrh(counter, MemOperand(kArtMethodRegister, ArtMethod::HotnessCountOffset().Int32Value()));
705 __ Cbnz(counter, &increment);
706 __ Ldr(lr, MemOperand(tr, entrypoint_offset));
707 // Note: we don't record the call here (and therefore don't generate a stack
708 // map), as the entrypoint should never be suspended.
709 __ Blr(lr);
710 __ Bind(&increment);
711 __ Add(counter, counter, -1);
712 __ Strh(counter, MemOperand(kArtMethodRegister, ArtMethod::HotnessCountOffset().Int32Value()));
713 __ Bind(&done);
714 }
715
716 // Do the suspend check.
717 if (compiler_options_.GetImplicitSuspendChecks()) {
718 ExactAssemblyScope eas(GetVIXLAssembler(),
719 kInstructionSize,
720 CodeBufferCheckScope::kExactSize);
721 __ ldr(kImplicitSuspendCheckRegister, MemOperand(kImplicitSuspendCheckRegister));
722 RecordPcInfo(0);
723 } else {
724 UseScratchRegisterScope temps(masm);
725 Register temp = temps.AcquireW();
726 vixl::aarch64::Label continue_label;
727 __ Ldr(temp, MemOperand(tr, Thread::ThreadFlagsOffset<kArm64PointerSize>().SizeValue()));
728 __ Tst(temp, Thread::SuspendOrCheckpointRequestFlags());
729 __ B(eq, &continue_label);
730 uint32_t entrypoint_offset =
731 GetThreadOffset<kArm64PointerSize>(kQuickTestSuspend).Int32Value();
732 __ Ldr(lr, MemOperand(tr, entrypoint_offset));
733 {
734 ExactAssemblyScope eas(GetVIXLAssembler(),
735 kInstructionSize,
736 CodeBufferCheckScope::kExactSize);
737 __ blr(lr);
738 RecordPcInfo(0);
739 }
740 __ Bind(&continue_label);
741 }
742 return true;
743 }
744
745
SetupArguments(InvokeType invoke_type,const InstructionOperands & operands,const char * shorty,uint32_t * obj_reg)746 bool FastCompilerARM64::SetupArguments(InvokeType invoke_type,
747 const InstructionOperands& operands,
748 const char* shorty,
749 /* out */ uint32_t* obj_reg) {
750 const size_t number_of_operands = operands.GetNumberOfOperands();
751
752 size_t start_index = 0u;
753 size_t argument_index = 0u;
754 InvokeDexCallingConventionVisitorARM64 convention;
755
756 // Handle 'this' parameter.
757 if (invoke_type != kStatic) {
758 if (number_of_operands == 0u) {
759 unimplemented_reason_ = "BogusSignature";
760 return false;
761 }
762 start_index = 1u;
763 *obj_reg = operands.GetOperand(0u);
764 if (!MoveLocation(convention.GetNextLocation(DataType::Type::kReference),
765 vreg_locations_[*obj_reg],
766 DataType::Type::kReference)) {
767 return false;
768 }
769 }
770
771 uint32_t shorty_index = 1; // Skip the return type.
772 // Handle all parameters except 'this'.
773 for (size_t i = start_index; i < number_of_operands; ++i, ++argument_index, ++shorty_index) {
774 // Make sure we don't go over the expected arguments or over the number of
775 // dex registers given. If the instruction was seen as dead by the verifier,
776 // it hasn't been properly checked.
777 char c = shorty[shorty_index];
778 if (UNLIKELY(c == 0)) {
779 unimplemented_reason_ = "BogusSignature";
780 return false;
781 }
782 DataType::Type type = DataType::FromShorty(c);
783 bool is_wide = (type == DataType::Type::kInt64) || (type == DataType::Type::kFloat64);
784 if (is_wide && ((i + 1 == number_of_operands) ||
785 (operands.GetOperand(i) + 1 != operands.GetOperand(i + 1)))) {
786 unimplemented_reason_ = "BogusSignature";
787 return false;
788 }
789 if (!MoveLocation(convention.GetNextLocation(type),
790 vreg_locations_[operands.GetOperand(i)],
791 type)) {
792 return false;
793 }
794 if (is_wide) {
795 ++i;
796 }
797 }
798 return true;
799 }
800
LoadMethod(Register reg,ArtMethod * method)801 bool FastCompilerARM64::LoadMethod(Register reg, ArtMethod* method) {
802 if (Runtime::Current()->IsAotCompiler()) {
803 unimplemented_reason_ = "AOTLoadMethod";
804 return false;
805 }
806 __ Ldr(reg, jit_patches_.DeduplicateUint64Literal(reinterpret_cast<uint64_t>(method)));
807 return true;
808 }
809
HandleInvoke(const Instruction & instruction,uint32_t dex_pc,InvokeType invoke_type)810 bool FastCompilerARM64::HandleInvoke(const Instruction& instruction,
811 uint32_t dex_pc,
812 InvokeType invoke_type) {
813 Instruction::Code opcode = instruction.Opcode();
814 uint16_t method_index = (opcode >= Instruction::INVOKE_VIRTUAL_RANGE)
815 ? instruction.VRegB_3rc()
816 : instruction.VRegB_35c();
817 ArtMethod* resolved_method = nullptr;
818 size_t offset = 0u;
819 {
820 Thread* self = Thread::Current();
821 ScopedObjectAccess soa(self);
822 ClassLinker* const class_linker = dex_compilation_unit_.GetClassLinker();
823 resolved_method = method_->SkipAccessChecks()
824 ? class_linker->ResolveMethodId(method_index, method_)
825 : class_linker->ResolveMethodWithChecks(
826 method_index, method_, invoke_type);
827 if (resolved_method == nullptr) {
828 DCHECK(self->IsExceptionPending());
829 self->ClearException();
830 unimplemented_reason_ = "UnresolvedInvoke";
831 return false;
832 }
833
834 if (resolved_method->IsConstructor() && resolved_method->GetDeclaringClass()->IsObjectClass()) {
835 // Object.<init> is always empty. Return early to not generate a frame.
836 if (kIsDebugBuild) {
837 CHECK(resolved_method->GetDeclaringClass()->IsVerified());
838 CodeItemDataAccessor accessor(*resolved_method->GetDexFile(),
839 resolved_method->GetCodeItem());
840 CHECK_EQ(accessor.InsnsSizeInCodeUnits(), 1u);
841 CHECK_EQ(accessor.begin().Inst().Opcode(), Instruction::RETURN_VOID);
842 }
843 // No need to update `previous_invoke_return_type_`, we know it is not going the
844 // be used.
845 return true;
846 }
847
848 if (invoke_type == kSuper) {
849 resolved_method = method_->SkipAccessChecks()
850 ? FindSuperMethodToCall</*access_check=*/false>(method_index,
851 resolved_method,
852 method_,
853 self)
854 : FindSuperMethodToCall</*access_check=*/true>(method_index,
855 resolved_method,
856 method_,
857 self);
858 if (resolved_method == nullptr) {
859 DCHECK(self->IsExceptionPending()) << method_->PrettyMethod();
860 self->ClearException();
861 unimplemented_reason_ = "UnresolvedInvokeSuper";
862 return false;
863 }
864 } else if (invoke_type == kVirtual) {
865 offset = resolved_method->GetVtableIndex();
866 } else if (invoke_type == kInterface) {
867 offset = resolved_method->GetImtIndex();
868 }
869
870 if (resolved_method->IsStringConstructor()) {
871 unimplemented_reason_ = "StringConstructor";
872 return false;
873 }
874 }
875
876 // Given we are calling a method, generate a frame.
877 if (!EnsureHasFrame()) {
878 return false;
879 }
880
881 // Setup the arguments for the call.
882 uint32_t obj_reg = -1;
883 const char* shorty = dex_compilation_unit_.GetDexFile()->GetMethodShorty(method_index);
884 if (opcode >= Instruction::INVOKE_VIRTUAL_RANGE) {
885 RangeInstructionOperands operands(instruction.VRegC(), instruction.VRegA_3rc());
886 if (!SetupArguments(invoke_type, operands, shorty, &obj_reg)) {
887 return false;
888 }
889 } else {
890 uint32_t args[5];
891 uint32_t number_of_vreg_arguments = instruction.GetVarArgs(args);
892 VarArgsInstructionOperands operands(args, number_of_vreg_arguments);
893 if (!SetupArguments(invoke_type, operands, shorty, &obj_reg)) {
894 return false;
895 }
896 }
897 // Save the invoke return type for the next move-result instruction.
898 previous_invoke_return_type_ = DataType::FromShorty(shorty[0]);
899
900 if (invoke_type != kStatic) {
901 bool can_be_null = CanBeNull(obj_reg);
902 // Load the class of the instance. For kDirect and kSuper, this acts as a
903 // null check.
904 if (can_be_null || invoke_type == kVirtual || invoke_type == kInterface) {
905 InvokeDexCallingConvention calling_convention;
906 Register receiver = calling_convention.GetRegisterAt(0);
907 Offset class_offset = mirror::Object::ClassOffset();
908 EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
909 __ Ldr(kArtMethodRegister.W(), HeapOperand(receiver.W(), class_offset));
910 if (can_be_null) {
911 RecordPcInfo(dex_pc);
912 }
913 }
914 }
915
916 if (invoke_type == kVirtual) {
917 size_t method_offset =
918 mirror::Class::EmbeddedVTableEntryOffset(offset, kArm64PointerSize).SizeValue();
919 __ Ldr(kArtMethodRegister, MemOperand(kArtMethodRegister, method_offset));
920 } else if (invoke_type == kInterface) {
921 __ Ldr(kArtMethodRegister,
922 MemOperand(kArtMethodRegister,
923 mirror::Class::ImtPtrOffset(kArm64PointerSize).Uint32Value()));
924 uint32_t method_offset =
925 static_cast<uint32_t>(ImTable::OffsetOfElement(offset, kArm64PointerSize));
926 __ Ldr(kArtMethodRegister, MemOperand(kArtMethodRegister, method_offset));
927 if (!LoadMethod(ip1, resolved_method)) {
928 return false;
929 }
930 } else {
931 DCHECK(invoke_type == kDirect || invoke_type == kSuper || invoke_type == kStatic);
932 if (!LoadMethod(kArtMethodRegister, resolved_method)) {
933 return false;
934 }
935 }
936
937 Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64PointerSize);
938 __ Ldr(lr, MemOperand(kArtMethodRegister, entry_point.SizeValue()));
939 {
940 // Use a scope to help guarantee that `RecordPcInfo()` records the correct pc.
941 ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize);
942 __ blr(lr);
943 RecordPcInfo(dex_pc);
944 }
945 return true;
946 }
947
InvokeRuntime(QuickEntrypointEnum entrypoint,uint32_t dex_pc)948 void FastCompilerARM64::InvokeRuntime(QuickEntrypointEnum entrypoint, uint32_t dex_pc) {
949 ThreadOffset64 entrypoint_offset = GetThreadOffset<kArm64PointerSize>(entrypoint);
950 __ Ldr(lr, MemOperand(tr, entrypoint_offset.Int32Value()));
951 // Ensure the pc position is recorded immediately after the `blr` instruction.
952 ExactAssemblyScope eas(GetVIXLAssembler(), kInstructionSize, CodeBufferCheckScope::kExactSize);
953 __ blr(lr);
954 if (EntrypointRequiresStackMap(entrypoint)) {
955 RecordPcInfo(dex_pc);
956 }
957 }
958
BuildLoadString(uint32_t vreg,dex::StringIndex string_index,const Instruction * next)959 bool FastCompilerARM64::BuildLoadString(uint32_t vreg,
960 dex::StringIndex string_index,
961 const Instruction* next) {
962 // Generate a frame because of the read barrier.
963 if (!EnsureHasFrame()) {
964 return false;
965 }
966 Location loc = CreateNewRegisterLocation(vreg, DataType::Type::kReference, next);
967 if (HitUnimplemented()) {
968 return false;
969 }
970 if (Runtime::Current()->IsAotCompiler()) {
971 unimplemented_reason_ = "AOTLoadString";
972 return false;
973 }
974
975 ScopedObjectAccess soa(Thread::Current());
976 ClassLinker* const class_linker = dex_compilation_unit_.GetClassLinker();
977 ObjPtr<mirror::String> str = class_linker->ResolveString(string_index, method_);
978 if (str == nullptr) {
979 soa.Self()->ClearException();
980 unimplemented_reason_ = "NullString";
981 return false;
982 }
983
984 Handle<mirror::String> h_str = handles_->NewHandle(str);
985 Register dst = RegisterFrom(loc, DataType::Type::kReference);
986 __ Ldr(dst.W(), jit_patches_.DeduplicateJitStringLiteral(GetDexFile(),
987 string_index,
988 h_str,
989 code_generation_data_.get()));
990 __ Ldr(dst.W(), MemOperand(dst.X()));
991 DoReadBarrierOn(dst);
992 UpdateLocal(vreg, /* is_object= */ true, /* can_be_null= */ false);
993 return true;
994 }
995
BuildNewInstance(uint32_t vreg,dex::TypeIndex type_index,uint32_t dex_pc,const Instruction * next)996 bool FastCompilerARM64::BuildNewInstance(uint32_t vreg,
997 dex::TypeIndex type_index,
998 uint32_t dex_pc,
999 const Instruction* next) {
1000 if (!EnsureHasFrame()) {
1001 return false;
1002 }
1003 if (Runtime::Current()->IsAotCompiler()) {
1004 unimplemented_reason_ = "AOTNewInstance";
1005 return false;
1006 }
1007
1008 ScopedObjectAccess soa(Thread::Current());
1009 ObjPtr<mirror::Class> klass = dex_compilation_unit_.GetClassLinker()->ResolveType(
1010 type_index, dex_compilation_unit_.GetDexCache(), dex_compilation_unit_.GetClassLoader());
1011 if (klass == nullptr ||
1012 !method_->GetDeclaringClass()->CanAccess(klass) ||
1013 klass->IsStringClass()) {
1014 soa.Self()->ClearException();
1015 unimplemented_reason_ = "UnsupportedClassForNewInstance";
1016 return false;
1017 }
1018
1019 InvokeRuntimeCallingConvention calling_convention;
1020 Register cls_reg = calling_convention.GetRegisterAt(0);
1021 Handle<mirror::Class> h_klass = handles_->NewHandle(klass);
1022 __ Ldr(cls_reg.W(), jit_patches_.DeduplicateJitClassLiteral(GetDexFile(),
1023 type_index,
1024 h_klass ,
1025 code_generation_data_.get()));
1026 __ Ldr(cls_reg.W(), MemOperand(cls_reg.X()));
1027 DoReadBarrierOn(cls_reg);
1028
1029 QuickEntrypointEnum entrypoint = kQuickAllocObjectInitialized;
1030 if (h_klass->IsFinalizable() ||
1031 !h_klass->IsVisiblyInitialized() ||
1032 h_klass->IsClassClass() || // Classes cannot be allocated in code
1033 !klass->IsInstantiable()) {
1034 entrypoint = kQuickAllocObjectWithChecks;
1035 }
1036 InvokeRuntime(entrypoint, dex_pc);
1037 __ Dmb(InnerShareable, BarrierWrites);
1038 if (!MoveLocation(CreateNewRegisterLocation(vreg, DataType::Type::kReference, next),
1039 calling_convention.GetReturnLocation(DataType::Type::kReference),
1040 DataType::Type::kReference)) {
1041 return false;
1042 }
1043 if (HitUnimplemented()) {
1044 return false;
1045 }
1046 UpdateLocal(vreg, /* is_object= */ true, /* can_be_null= */ false);
1047 return true;
1048 }
1049
BuildCheckCast(uint32_t vreg,dex::TypeIndex type_index,uint32_t dex_pc)1050 bool FastCompilerARM64::BuildCheckCast(uint32_t vreg, dex::TypeIndex type_index, uint32_t dex_pc) {
1051 if (!EnsureHasFrame()) {
1052 return false;
1053 }
1054
1055 InvokeRuntimeCallingConvention calling_convention;
1056 UseScratchRegisterScope temps(GetVIXLAssembler());
1057 Register cls = calling_convention.GetRegisterAt(1);
1058 Register obj_cls = calling_convention.GetRegisterAt(2);
1059 Register obj = WRegisterFrom(GetExistingRegisterLocation(vreg, DataType::Type::kReference));
1060 if (HitUnimplemented()) {
1061 return false;
1062 }
1063
1064 ScopedObjectAccess soa(Thread::Current());
1065 ObjPtr<mirror::Class> klass = dex_compilation_unit_.GetClassLinker()->ResolveType(
1066 type_index, dex_compilation_unit_.GetDexCache(), dex_compilation_unit_.GetClassLoader());
1067 if (klass == nullptr || !method_->GetDeclaringClass()->CanAccess(klass)) {
1068 soa.Self()->ClearException();
1069 unimplemented_reason_ = "UnsupportedCheckCast";
1070 return false;
1071 }
1072 Handle<mirror::Class> h_klass = handles_->NewHandle(klass);
1073
1074 vixl::aarch64::Label exit, read_barrier_exit;
1075 __ Cbz(obj, &exit);
1076 __ Ldr(cls.W(), jit_patches_.DeduplicateJitClassLiteral(GetDexFile(),
1077 type_index,
1078 h_klass ,
1079 code_generation_data_.get()));
1080 __ Ldr(cls.W(), MemOperand(cls.X()));
1081 __ Ldr(obj_cls.W(), MemOperand(obj.X()));
1082 __ Cmp(cls.W(), obj_cls.W());
1083 __ B(eq, &exit);
1084
1085 // Read barrier on the GC Root.
1086 DoReadBarrierOn(cls, &read_barrier_exit);
1087 // Read barrier on the object's class.
1088 DoReadBarrierOn(obj_cls, &read_barrier_exit, /* do_mr_check= */ false);
1089
1090 __ Bind(&read_barrier_exit);
1091 __ Cmp(cls.W(), obj_cls.W());
1092 __ B(eq, &exit);
1093 if (!MoveLocation(LocationFrom(calling_convention.GetRegisterAt(0)),
1094 LocationFrom(obj),
1095 DataType::Type::kReference)) {
1096 return false;
1097 }
1098 InvokeRuntime(kQuickCheckInstanceOf, dex_pc);
1099
1100 __ Bind(&exit);
1101 return true;
1102 }
1103
DoReadBarrierOn(Register reg,vixl::aarch64::Label * exit,bool do_mr_check)1104 void FastCompilerARM64::DoReadBarrierOn(Register reg,
1105 vixl::aarch64::Label* exit,
1106 bool do_mr_check) {
1107 DCHECK(has_frame_);
1108 vixl::aarch64::Label local_exit;
1109 if (do_mr_check) {
1110 __ Cbz(mr, (exit != nullptr) ? exit : &local_exit);
1111 }
1112 int32_t entry_point_offset =
1113 Thread::ReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(reg.GetCode());
1114 __ Ldr(lr, MemOperand(tr, entry_point_offset));
1115 __ Blr(lr);
1116 if (exit == nullptr && do_mr_check) {
1117 __ Bind(&local_exit);
1118 }
1119 }
1120
CanGenerateCodeFor(ArtField * field,bool can_receiver_be_null)1121 bool FastCompilerARM64::CanGenerateCodeFor(ArtField* field, bool can_receiver_be_null) {
1122 if (field == nullptr) {
1123 // Clear potential resolution exception.
1124 Thread::Current()->ClearException();
1125 unimplemented_reason_ = "UnresolvedField";
1126 return false;
1127 }
1128 if (field->IsVolatile()) {
1129 unimplemented_reason_ = "VolatileField";
1130 return false;
1131 }
1132
1133 if (can_receiver_be_null) {
1134 if (!CanDoImplicitNullCheckOn(field->GetOffset().Uint32Value())) {
1135 unimplemented_reason_ = "TooLargeFieldOffset";
1136 return false;
1137 }
1138 }
1139 return true;
1140 }
1141
1142 #define DO_CASE(arm_op, op, other) \
1143 case arm_op: { \
1144 if (constant op other) { \
1145 __ B(label); \
1146 } \
1147 return true; \
1148 } \
1149
1150 template<vixl::aarch64::Condition kCond, bool kCompareWithZero>
If_21_22t(const Instruction & instruction,uint32_t dex_pc)1151 bool FastCompilerARM64::If_21_22t(const Instruction& instruction, uint32_t dex_pc) {
1152 DCHECK_EQ(kCompareWithZero ? Instruction::Format::k21t : Instruction::Format::k22t,
1153 Instruction::FormatOf(instruction.Opcode()));
1154 if (!EnsureHasFrame()) {
1155 return false;
1156 }
1157 int32_t target_offset = kCompareWithZero ? instruction.VRegB_21t() : instruction.VRegC_22t();
1158 DCHECK_EQ(target_offset, instruction.GetTargetOffset());
1159 if (target_offset < 0) {
1160 // TODO: Support for negative branches requires two passes.
1161 unimplemented_reason_ = "NegativeBranch";
1162 return false;
1163 }
1164 int32_t register_index = kCompareWithZero ? instruction.VRegA_21t() : instruction.VRegA_22t();
1165 vixl::aarch64::Label* label = GetLabelOf(dex_pc + target_offset);
1166 Location location = vreg_locations_[register_index];
1167
1168 if (kCompareWithZero) {
1169 // We are going to branch, move all constants to registers to make the merge
1170 // point use the same locations.
1171 MoveConstantsToRegisters();
1172 UpdateMasks(dex_pc + target_offset);
1173 if (location.IsConstant()) {
1174 DCHECK(location.GetConstant()->IsIntConstant());
1175 int32_t constant = location.GetConstant()->AsIntConstant()->GetValue();
1176 switch (kCond) {
1177 DO_CASE(vixl::aarch64::eq, ==, 0);
1178 DO_CASE(vixl::aarch64::ne, !=, 0);
1179 DO_CASE(vixl::aarch64::lt, <, 0);
1180 DO_CASE(vixl::aarch64::le, <=, 0);
1181 DO_CASE(vixl::aarch64::gt, >, 0);
1182 DO_CASE(vixl::aarch64::ge, >=, 0);
1183 }
1184 return true;
1185 } else if (location.IsRegister()) {
1186 CPURegister reg = CPURegisterFrom(location, DataType::Type::kInt32);
1187 switch (kCond) {
1188 case vixl::aarch64::eq: {
1189 __ Cbz(Register(reg), label);
1190 return true;
1191 }
1192 case vixl::aarch64::ne: {
1193 __ Cbnz(Register(reg), label);
1194 return true;
1195 }
1196 default: {
1197 __ Cmp(Register(reg), 0);
1198 __ B(kCond, label);
1199 return true;
1200 }
1201 }
1202 } else {
1203 DCHECK(location.IsStackSlot());
1204 unimplemented_reason_ = "CompareWithZeroOnStackSlot";
1205 }
1206 return false;
1207 }
1208
1209 // !kCompareWithZero
1210 Location other_location = vreg_locations_[instruction.VRegB_22t()];
1211 // We are going to branch, move all constants to registers to make the merge
1212 // point use the same locations.
1213 MoveConstantsToRegisters();
1214 UpdateMasks(dex_pc + target_offset);
1215 if (location.IsConstant() && other_location.IsConstant()) {
1216 int32_t constant = location.GetConstant()->AsIntConstant()->GetValue();
1217 int32_t other_constant = other_location.GetConstant()->AsIntConstant()->GetValue();
1218 switch (kCond) {
1219 DO_CASE(vixl::aarch64::eq, ==, other_constant);
1220 DO_CASE(vixl::aarch64::ne, !=, other_constant);
1221 DO_CASE(vixl::aarch64::lt, <, other_constant);
1222 DO_CASE(vixl::aarch64::le, <=, other_constant);
1223 DO_CASE(vixl::aarch64::gt, >, other_constant);
1224 DO_CASE(vixl::aarch64::ge, >=, other_constant);
1225 }
1226 return true;
1227 }
1228 // Reload the locations, which can now be registers.
1229 location = vreg_locations_[register_index];
1230 other_location = vreg_locations_[instruction.VRegB_22t()];
1231 if (location.IsRegister() && other_location.IsRegister()) {
1232 CPURegister reg = CPURegisterFrom(location, DataType::Type::kInt32);
1233 CPURegister other_reg = CPURegisterFrom(other_location, DataType::Type::kInt32);
1234 __ Cmp(Register(reg), Register(other_reg));
1235 __ B(kCond, label);
1236 return true;
1237 }
1238
1239 unimplemented_reason_ = "UnimplementedCompare";
1240 return false;
1241 }
1242 #undef DO_CASE
1243
ProcessDexInstruction(const Instruction & instruction,uint32_t dex_pc,const Instruction * next)1244 bool FastCompilerARM64::ProcessDexInstruction(const Instruction& instruction,
1245 uint32_t dex_pc,
1246 const Instruction* next) {
1247 bool is_object = false;
1248 switch (instruction.Opcode()) {
1249 case Instruction::CONST_4: {
1250 int32_t register_index = instruction.VRegA_11n();
1251 int32_t constant = instruction.VRegB_11n();
1252 vreg_locations_[register_index] =
1253 Location::ConstantLocation(new (allocator_) HIntConstant(constant));
1254 UpdateLocal(register_index, /* is_object= */ false);
1255 return true;
1256 }
1257
1258 case Instruction::CONST_16: {
1259 int32_t register_index = instruction.VRegA_21s();
1260 int32_t constant = instruction.VRegB_21s();
1261 vreg_locations_[register_index] =
1262 Location::ConstantLocation(new (allocator_) HIntConstant(constant));
1263 UpdateLocal(register_index, /* is_object= */ false);
1264 return true;
1265 }
1266
1267 case Instruction::CONST: {
1268 break;
1269 }
1270
1271 case Instruction::CONST_HIGH16: {
1272 break;
1273 }
1274
1275 case Instruction::CONST_WIDE_16: {
1276 break;
1277 }
1278
1279 case Instruction::CONST_WIDE_32: {
1280 break;
1281 }
1282
1283 case Instruction::CONST_WIDE: {
1284 break;
1285 }
1286
1287 case Instruction::CONST_WIDE_HIGH16: {
1288 break;
1289 }
1290
1291 case Instruction::MOVE:
1292 case Instruction::MOVE_FROM16:
1293 case Instruction::MOVE_16: {
1294 break;
1295 }
1296
1297 case Instruction::MOVE_WIDE:
1298 case Instruction::MOVE_WIDE_FROM16:
1299 case Instruction::MOVE_WIDE_16: {
1300 break;
1301 }
1302
1303 case Instruction::MOVE_OBJECT:
1304 case Instruction::MOVE_OBJECT_16:
1305 case Instruction::MOVE_OBJECT_FROM16: {
1306 break;
1307 }
1308
1309 case Instruction::RETURN_VOID: {
1310 if (method_->IsConstructor() &&
1311 !method_->IsStatic() &&
1312 dex_compilation_unit_.RequiresConstructorBarrier()) {
1313 __ Dmb(InnerShareable, BarrierWrites);
1314 }
1315 PopFrameAndReturn();
1316 return true;
1317 }
1318
1319 #define IF_XX(comparison, cond) \
1320 case Instruction::IF_##cond: \
1321 return If_21_22t<comparison, /* kCompareWithZero= */ false>(instruction, dex_pc); \
1322 case Instruction::IF_##cond##Z: \
1323 return If_21_22t<comparison, /* kCompareWithZero= */ true>(instruction, dex_pc);
1324
1325 IF_XX(vixl::aarch64::eq, EQ);
1326 IF_XX(vixl::aarch64::ne, NE);
1327 IF_XX(vixl::aarch64::lt, LT);
1328 IF_XX(vixl::aarch64::le, LE);
1329 IF_XX(vixl::aarch64::gt, GT);
1330 IF_XX(vixl::aarch64::ge, GE);
1331
1332 case Instruction::GOTO:
1333 case Instruction::GOTO_16:
1334 case Instruction::GOTO_32: {
1335 break;
1336 }
1337
1338 case Instruction::RETURN:
1339 case Instruction::RETURN_OBJECT: {
1340 int32_t register_index = instruction.VRegA_11x();
1341 InvokeDexCallingConventionVisitorARM64 convention;
1342 if (!MoveLocation(convention.GetReturnLocation(return_type_),
1343 vreg_locations_[register_index],
1344 return_type_)) {
1345 return false;
1346 }
1347 if (has_frame_) {
1348 // We may have used the "record last instruction before return in return
1349 // register" optimization (see `CreateNewRegisterLocation`),
1350 // so set the returned register back to a callee save location in case the
1351 // method has a frame and there are instructions after this return that
1352 // may use this register.
1353 uint32_t register_code = kAvailableCalleeSaveRegisters[register_index].GetCode();
1354 vreg_locations_[register_index] = Location::RegisterLocation(register_code);
1355 }
1356 PopFrameAndReturn();
1357 return true;
1358 }
1359
1360 case Instruction::RETURN_WIDE: {
1361 break;
1362 }
1363
1364 case Instruction::INVOKE_DIRECT:
1365 case Instruction::INVOKE_DIRECT_RANGE:
1366 return HandleInvoke(instruction, dex_pc, kDirect);
1367 case Instruction::INVOKE_INTERFACE:
1368 case Instruction::INVOKE_INTERFACE_RANGE:
1369 return HandleInvoke(instruction, dex_pc, kInterface);
1370 case Instruction::INVOKE_STATIC:
1371 case Instruction::INVOKE_STATIC_RANGE:
1372 return HandleInvoke(instruction, dex_pc, kStatic);
1373 case Instruction::INVOKE_SUPER:
1374 case Instruction::INVOKE_SUPER_RANGE:
1375 return HandleInvoke(instruction, dex_pc, kSuper);
1376 case Instruction::INVOKE_VIRTUAL:
1377 case Instruction::INVOKE_VIRTUAL_RANGE: {
1378 return HandleInvoke(instruction, dex_pc, kVirtual);
1379 }
1380
1381 case Instruction::INVOKE_POLYMORPHIC: {
1382 break;
1383 }
1384
1385 case Instruction::INVOKE_POLYMORPHIC_RANGE: {
1386 break;
1387 }
1388
1389 case Instruction::INVOKE_CUSTOM: {
1390 break;
1391 }
1392
1393 case Instruction::INVOKE_CUSTOM_RANGE: {
1394 break;
1395 }
1396
1397 case Instruction::NEG_INT: {
1398 break;
1399 }
1400
1401 case Instruction::NEG_LONG: {
1402 break;
1403 }
1404
1405 case Instruction::NEG_FLOAT: {
1406 break;
1407 }
1408
1409 case Instruction::NEG_DOUBLE: {
1410 break;
1411 }
1412
1413 case Instruction::NOT_INT: {
1414 break;
1415 }
1416
1417 case Instruction::NOT_LONG: {
1418 break;
1419 }
1420
1421 case Instruction::INT_TO_LONG: {
1422 break;
1423 }
1424
1425 case Instruction::INT_TO_FLOAT: {
1426 break;
1427 }
1428
1429 case Instruction::INT_TO_DOUBLE: {
1430 break;
1431 }
1432
1433 case Instruction::LONG_TO_INT: {
1434 break;
1435 }
1436
1437 case Instruction::LONG_TO_FLOAT: {
1438 break;
1439 }
1440
1441 case Instruction::LONG_TO_DOUBLE: {
1442 break;
1443 }
1444
1445 case Instruction::FLOAT_TO_INT: {
1446 break;
1447 }
1448
1449 case Instruction::FLOAT_TO_LONG: {
1450 break;
1451 }
1452
1453 case Instruction::FLOAT_TO_DOUBLE: {
1454 break;
1455 }
1456
1457 case Instruction::DOUBLE_TO_INT: {
1458 break;
1459 }
1460
1461 case Instruction::DOUBLE_TO_LONG: {
1462 break;
1463 }
1464
1465 case Instruction::DOUBLE_TO_FLOAT: {
1466 break;
1467 }
1468
1469 case Instruction::INT_TO_BYTE: {
1470 break;
1471 }
1472
1473 case Instruction::INT_TO_SHORT: {
1474 break;
1475 }
1476
1477 case Instruction::INT_TO_CHAR: {
1478 break;
1479 }
1480
1481 case Instruction::ADD_INT: {
1482 break;
1483 }
1484
1485 case Instruction::ADD_LONG: {
1486 break;
1487 }
1488
1489 case Instruction::ADD_DOUBLE: {
1490 break;
1491 }
1492
1493 case Instruction::ADD_FLOAT: {
1494 break;
1495 }
1496
1497 case Instruction::SUB_INT: {
1498 break;
1499 }
1500
1501 case Instruction::SUB_LONG: {
1502 break;
1503 }
1504
1505 case Instruction::SUB_FLOAT: {
1506 break;
1507 }
1508
1509 case Instruction::SUB_DOUBLE: {
1510 break;
1511 }
1512
1513 case Instruction::ADD_INT_2ADDR: {
1514 break;
1515 }
1516
1517 case Instruction::MUL_INT: {
1518 break;
1519 }
1520
1521 case Instruction::MUL_LONG: {
1522 break;
1523 }
1524
1525 case Instruction::MUL_FLOAT: {
1526 break;
1527 }
1528
1529 case Instruction::MUL_DOUBLE: {
1530 break;
1531 }
1532
1533 case Instruction::DIV_INT: {
1534 break;
1535 }
1536
1537 case Instruction::DIV_LONG: {
1538 break;
1539 }
1540
1541 case Instruction::DIV_FLOAT: {
1542 break;
1543 }
1544
1545 case Instruction::DIV_DOUBLE: {
1546 break;
1547 }
1548
1549 case Instruction::REM_INT: {
1550 break;
1551 }
1552
1553 case Instruction::REM_LONG: {
1554 break;
1555 }
1556
1557 case Instruction::REM_FLOAT: {
1558 break;
1559 }
1560
1561 case Instruction::REM_DOUBLE: {
1562 break;
1563 }
1564
1565 case Instruction::AND_INT: {
1566 break;
1567 }
1568
1569 case Instruction::AND_LONG: {
1570 break;
1571 }
1572
1573 case Instruction::SHL_INT: {
1574 break;
1575 }
1576
1577 case Instruction::SHL_LONG: {
1578 break;
1579 }
1580
1581 case Instruction::SHR_INT: {
1582 break;
1583 }
1584
1585 case Instruction::SHR_LONG: {
1586 break;
1587 }
1588
1589 case Instruction::USHR_INT: {
1590 break;
1591 }
1592
1593 case Instruction::USHR_LONG: {
1594 break;
1595 }
1596
1597 case Instruction::OR_INT: {
1598 break;
1599 }
1600
1601 case Instruction::OR_LONG: {
1602 break;
1603 }
1604
1605 case Instruction::XOR_INT: {
1606 break;
1607 }
1608
1609 case Instruction::XOR_LONG: {
1610 break;
1611 }
1612
1613 case Instruction::ADD_LONG_2ADDR: {
1614 break;
1615 }
1616
1617 case Instruction::ADD_DOUBLE_2ADDR: {
1618 break;
1619 }
1620
1621 case Instruction::ADD_FLOAT_2ADDR: {
1622 break;
1623 }
1624
1625 case Instruction::SUB_INT_2ADDR: {
1626 break;
1627 }
1628
1629 case Instruction::SUB_LONG_2ADDR: {
1630 break;
1631 }
1632
1633 case Instruction::SUB_FLOAT_2ADDR: {
1634 break;
1635 }
1636
1637 case Instruction::SUB_DOUBLE_2ADDR: {
1638 break;
1639 }
1640
1641 case Instruction::MUL_INT_2ADDR: {
1642 break;
1643 }
1644
1645 case Instruction::MUL_LONG_2ADDR: {
1646 break;
1647 }
1648
1649 case Instruction::MUL_FLOAT_2ADDR: {
1650 break;
1651 }
1652
1653 case Instruction::MUL_DOUBLE_2ADDR: {
1654 break;
1655 }
1656
1657 case Instruction::DIV_INT_2ADDR: {
1658 break;
1659 }
1660
1661 case Instruction::DIV_LONG_2ADDR: {
1662 break;
1663 }
1664
1665 case Instruction::REM_INT_2ADDR: {
1666 break;
1667 }
1668
1669 case Instruction::REM_LONG_2ADDR: {
1670 break;
1671 }
1672
1673 case Instruction::REM_FLOAT_2ADDR: {
1674 break;
1675 }
1676
1677 case Instruction::REM_DOUBLE_2ADDR: {
1678 break;
1679 }
1680
1681 case Instruction::SHL_INT_2ADDR: {
1682 break;
1683 }
1684
1685 case Instruction::SHL_LONG_2ADDR: {
1686 break;
1687 }
1688
1689 case Instruction::SHR_INT_2ADDR: {
1690 break;
1691 }
1692
1693 case Instruction::SHR_LONG_2ADDR: {
1694 break;
1695 }
1696
1697 case Instruction::USHR_INT_2ADDR: {
1698 break;
1699 }
1700
1701 case Instruction::USHR_LONG_2ADDR: {
1702 break;
1703 }
1704
1705 case Instruction::DIV_FLOAT_2ADDR: {
1706 break;
1707 }
1708
1709 case Instruction::DIV_DOUBLE_2ADDR: {
1710 break;
1711 }
1712
1713 case Instruction::AND_INT_2ADDR: {
1714 break;
1715 }
1716
1717 case Instruction::AND_LONG_2ADDR: {
1718 break;
1719 }
1720
1721 case Instruction::OR_INT_2ADDR: {
1722 break;
1723 }
1724
1725 case Instruction::OR_LONG_2ADDR: {
1726 break;
1727 }
1728
1729 case Instruction::XOR_INT_2ADDR: {
1730 break;
1731 }
1732
1733 case Instruction::XOR_LONG_2ADDR: {
1734 break;
1735 }
1736
1737 case Instruction::ADD_INT_LIT16: {
1738 break;
1739 }
1740
1741 case Instruction::AND_INT_LIT16: {
1742 break;
1743 }
1744
1745 case Instruction::OR_INT_LIT16: {
1746 break;
1747 }
1748
1749 case Instruction::XOR_INT_LIT16: {
1750 break;
1751 }
1752
1753 case Instruction::RSUB_INT: {
1754 break;
1755 }
1756
1757 case Instruction::MUL_INT_LIT16: {
1758 break;
1759 }
1760
1761 case Instruction::ADD_INT_LIT8: {
1762 break;
1763 }
1764
1765 case Instruction::AND_INT_LIT8: {
1766 break;
1767 }
1768
1769 case Instruction::OR_INT_LIT8: {
1770 break;
1771 }
1772
1773 case Instruction::XOR_INT_LIT8: {
1774 break;
1775 }
1776
1777 case Instruction::RSUB_INT_LIT8: {
1778 break;
1779 }
1780
1781 case Instruction::MUL_INT_LIT8: {
1782 break;
1783 }
1784
1785 case Instruction::DIV_INT_LIT16:
1786 case Instruction::DIV_INT_LIT8: {
1787 break;
1788 }
1789
1790 case Instruction::REM_INT_LIT16:
1791 case Instruction::REM_INT_LIT8: {
1792 break;
1793 }
1794
1795 case Instruction::SHL_INT_LIT8: {
1796 break;
1797 }
1798
1799 case Instruction::SHR_INT_LIT8: {
1800 break;
1801 }
1802
1803 case Instruction::USHR_INT_LIT8: {
1804 break;
1805 }
1806
1807 case Instruction::NEW_INSTANCE: {
1808 dex::TypeIndex type_index(instruction.VRegB_21c());
1809 return BuildNewInstance(instruction.VRegA_21c(), type_index, dex_pc, next);
1810 }
1811
1812 case Instruction::NEW_ARRAY: {
1813 break;
1814 }
1815
1816 case Instruction::FILLED_NEW_ARRAY: {
1817 break;
1818 }
1819
1820 case Instruction::FILLED_NEW_ARRAY_RANGE: {
1821 break;
1822 }
1823
1824 case Instruction::FILL_ARRAY_DATA: {
1825 break;
1826 }
1827
1828 case Instruction::MOVE_RESULT_OBJECT:
1829 is_object = true;
1830 FALLTHROUGH_INTENDED;
1831 case Instruction::MOVE_RESULT: {
1832 int32_t register_index = instruction.VRegA_11x();
1833 InvokeDexCallingConventionVisitorARM64 convention;
1834 if (!MoveLocation(
1835 CreateNewRegisterLocation(register_index, previous_invoke_return_type_, next),
1836 convention.GetReturnLocation(previous_invoke_return_type_),
1837 previous_invoke_return_type_)) {
1838 return false;
1839 }
1840 if (HitUnimplemented()) {
1841 return false;
1842 }
1843 UpdateLocal(register_index, is_object);
1844 return true;
1845 }
1846
1847 case Instruction::MOVE_RESULT_WIDE: {
1848 break;
1849 }
1850
1851 case Instruction::CMP_LONG: {
1852 break;
1853 }
1854
1855 case Instruction::CMPG_FLOAT: {
1856 break;
1857 }
1858
1859 case Instruction::CMPG_DOUBLE: {
1860 break;
1861 }
1862
1863 case Instruction::CMPL_FLOAT: {
1864 break;
1865 }
1866
1867 case Instruction::CMPL_DOUBLE: {
1868 break;
1869 }
1870
1871 case Instruction::NOP:
1872 return true;
1873
1874 case Instruction::IGET_OBJECT:
1875 is_object = true;
1876 FALLTHROUGH_INTENDED;
1877 case Instruction::IGET:
1878 case Instruction::IGET_WIDE:
1879 case Instruction::IGET_BOOLEAN:
1880 case Instruction::IGET_BYTE:
1881 case Instruction::IGET_CHAR:
1882 case Instruction::IGET_SHORT: {
1883 uint32_t source_or_dest_reg = instruction.VRegA_22c();
1884 uint32_t obj_reg = instruction.VRegB_22c();
1885 uint16_t field_index = instruction.VRegC_22c();
1886 bool can_receiver_be_null = CanBeNull(obj_reg);
1887 ArtField* field = nullptr;
1888 {
1889 ScopedObjectAccess soa(Thread::Current());
1890 field = ResolveFieldWithAccessChecks(soa.Self(),
1891 dex_compilation_unit_.GetClassLinker(),
1892 field_index,
1893 method_,
1894 /* is_static= */ false,
1895 /* is_put= */ false,
1896 /* resolve_field_type= */ 0u);
1897 if (!CanGenerateCodeFor(field, can_receiver_be_null)) {
1898 return false;
1899 }
1900 }
1901
1902 if (can_receiver_be_null || is_object) {
1903 // We need a frame in case the null check throws or there is a read
1904 // barrier.
1905 if (!EnsureHasFrame()) {
1906 return false;
1907 }
1908 }
1909
1910 MemOperand mem = HeapOperand(
1911 RegisterFrom(GetExistingRegisterLocation(obj_reg, DataType::Type::kReference),
1912 DataType::Type::kReference),
1913 field->GetOffset());
1914 if (HitUnimplemented()) {
1915 return false;
1916 }
1917 if (is_object) {
1918 Register dst = WRegisterFrom(
1919 CreateNewRegisterLocation(source_or_dest_reg, DataType::Type::kReference, next));
1920 if (HitUnimplemented()) {
1921 return false;
1922 }
1923 {
1924 // Ensure the pc position is recorded immediately after the load instruction.
1925 EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
1926 __ Ldr(dst, mem);
1927 if (can_receiver_be_null) {
1928 RecordPcInfo(dex_pc);
1929 }
1930 }
1931 UpdateLocal(source_or_dest_reg, /* is_object= */ true);
1932 DoReadBarrierOn(dst);
1933 return true;
1934 }
1935 // Ensure the pc position is recorded immediately after the load instruction.
1936 EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
1937 switch (instruction.Opcode()) {
1938 case Instruction::IGET_BOOLEAN: {
1939 Register dst = WRegisterFrom(
1940 CreateNewRegisterLocation(source_or_dest_reg, DataType::Type::kInt32, next));
1941 __ Ldrb(Register(dst), mem);
1942 break;
1943 }
1944 case Instruction::IGET_BYTE: {
1945 Register dst = WRegisterFrom(
1946 CreateNewRegisterLocation(source_or_dest_reg, DataType::Type::kInt32, next));
1947 __ Ldrsb(Register(dst), mem);
1948 break;
1949 }
1950 case Instruction::IGET_CHAR: {
1951 Register dst = WRegisterFrom(
1952 CreateNewRegisterLocation(source_or_dest_reg, DataType::Type::kInt32, next));
1953 __ Ldrh(Register(dst), mem);
1954 break;
1955 }
1956 case Instruction::IGET_SHORT: {
1957 Register dst = WRegisterFrom(
1958 CreateNewRegisterLocation(source_or_dest_reg, DataType::Type::kInt32, next));
1959 __ Ldrsh(Register(dst), mem);
1960 break;
1961 }
1962 case Instruction::IGET: {
1963 const dex::FieldId& field_id = GetDexFile().GetFieldId(field_index);
1964 const char* type = GetDexFile().GetFieldTypeDescriptor(field_id);
1965 DataType::Type field_type = DataType::FromShorty(type[0]);
1966 if (DataType::IsFloatingPointType(field_type)) {
1967 VRegister dst = SRegisterFrom(
1968 CreateNewRegisterLocation(source_or_dest_reg, field_type, next));
1969 __ Ldr(dst, mem);
1970 } else {
1971 Register dst = WRegisterFrom(
1972 CreateNewRegisterLocation(source_or_dest_reg, DataType::Type::kInt32, next));
1973 __ Ldr(dst, mem);
1974 }
1975 if (HitUnimplemented()) {
1976 return false;
1977 }
1978 break;
1979 }
1980 default:
1981 unimplemented_reason_ = "UnimplementedIGet";
1982 return false;
1983 }
1984 UpdateLocal(source_or_dest_reg, /* is_object= */ false);
1985 if (can_receiver_be_null) {
1986 RecordPcInfo(dex_pc);
1987 }
1988 return true;
1989 }
1990
1991 case Instruction::IPUT_OBJECT:
1992 is_object = true;
1993 FALLTHROUGH_INTENDED;
1994 case Instruction::IPUT:
1995 case Instruction::IPUT_WIDE:
1996 case Instruction::IPUT_BOOLEAN:
1997 case Instruction::IPUT_BYTE:
1998 case Instruction::IPUT_CHAR:
1999 case Instruction::IPUT_SHORT: {
2000 uint32_t source_reg = instruction.VRegA_22c();
2001 uint32_t obj_reg = instruction.VRegB_22c();
2002 uint16_t field_index = instruction.VRegC_22c();
2003 bool can_receiver_be_null = CanBeNull(obj_reg);
2004 ArtField* field = nullptr;
2005 {
2006 ScopedObjectAccess soa(Thread::Current());
2007 field = ResolveFieldWithAccessChecks(soa.Self(),
2008 dex_compilation_unit_.GetClassLinker(),
2009 field_index,
2010 method_,
2011 /* is_static= */ false,
2012 /* is_put= */ true,
2013 /* resolve_field_type= */ is_object);
2014 if (!CanGenerateCodeFor(field, can_receiver_be_null)) {
2015 return false;
2016 }
2017 }
2018
2019 if (can_receiver_be_null) {
2020 // We need a frame in case the null check throws.
2021 if (!EnsureHasFrame()) {
2022 return false;
2023 }
2024 }
2025
2026 Register holder = RegisterFrom(
2027 GetExistingRegisterLocation(obj_reg, DataType::Type::kReference),
2028 DataType::Type::kReference);
2029 if (HitUnimplemented()) {
2030 return false;
2031 }
2032 MemOperand mem = HeapOperand(holder, field->GetOffset());
2033
2034 // Need one temp if the stored value is a constant.
2035 UseScratchRegisterScope temps(GetVIXLAssembler());
2036 Location src = vreg_locations_[source_reg];
2037 bool assigning_constant = false;
2038 if (src.IsConstant()) {
2039 assigning_constant = true;
2040 if (src.GetConstant()->IsIntConstant() &&
2041 src.GetConstant()->AsIntConstant()->GetValue() == 0) {
2042 src = Location::RegisterLocation(XZR);
2043 } else {
2044 src = Location::RegisterLocation(temps.AcquireW().GetCode());
2045 if (!MoveLocation(src, vreg_locations_[source_reg], DataType::Type::kInt32)) {
2046 return false;
2047 }
2048 }
2049 } else if (src.IsStackSlot() || src.IsDoubleStackSlot()) {
2050 unimplemented_reason_ = "IPUTOnStackSlot";
2051 return false;
2052 }
2053 if (instruction.Opcode() == Instruction::IPUT_OBJECT) {
2054 Register reg = WRegisterFrom(src);
2055 {
2056 // Ensure the pc position is recorded immediately after the store instruction.
2057 EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
2058 __ Str(reg, mem);
2059 if (can_receiver_be_null) {
2060 RecordPcInfo(dex_pc);
2061 }
2062 }
2063 // If we assign a constant (only null for iput-object), no need for the write
2064 // barrier.
2065 if (!assigning_constant) {
2066 vixl::aarch64::Label exit;
2067 __ Cbz(reg, &exit);
2068 Register card = temps.AcquireX();
2069 Register temp = temps.AcquireW();
2070 __ Ldr(card, MemOperand(tr, Thread::CardTableOffset<kArm64PointerSize>().Int32Value()));
2071 __ Lsr(temp, holder, gc::accounting::CardTable::kCardShift);
2072 __ Strb(card, MemOperand(card, temp.X()));
2073 __ Bind(&exit);
2074 }
2075 return true;
2076 }
2077 // Ensure the pc position is recorded immediately after the store instruction.
2078 EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
2079 switch (instruction.Opcode()) {
2080 case Instruction::IPUT_BOOLEAN:
2081 case Instruction::IPUT_BYTE: {
2082 __ Strb(WRegisterFrom(src), mem);
2083 break;
2084 }
2085 case Instruction::IPUT_CHAR:
2086 case Instruction::IPUT_SHORT: {
2087 __ Strh(WRegisterFrom(src), mem);
2088 break;
2089 }
2090 case Instruction::IPUT: {
2091 if (src.IsFpuRegister()) {
2092 __ Str(SRegisterFrom(src), mem);
2093 } else {
2094 __ Str(WRegisterFrom(src), mem);
2095 }
2096 break;
2097 }
2098 default:
2099 unimplemented_reason_ = "UnimplementedIPut";
2100 return false;
2101 }
2102 if (can_receiver_be_null) {
2103 RecordPcInfo(dex_pc);
2104 }
2105 return true;
2106 }
2107
2108 case Instruction::SGET:
2109 case Instruction::SGET_WIDE:
2110 case Instruction::SGET_OBJECT:
2111 case Instruction::SGET_BOOLEAN:
2112 case Instruction::SGET_BYTE:
2113 case Instruction::SGET_CHAR:
2114 case Instruction::SGET_SHORT: {
2115 break;
2116 }
2117
2118 case Instruction::SPUT:
2119 case Instruction::SPUT_WIDE:
2120 case Instruction::SPUT_OBJECT:
2121 case Instruction::SPUT_BOOLEAN:
2122 case Instruction::SPUT_BYTE:
2123 case Instruction::SPUT_CHAR:
2124 case Instruction::SPUT_SHORT: {
2125 break;
2126 }
2127
2128 #define ARRAY_XX(kind, anticipated_type) \
2129 case Instruction::AGET##kind: { \
2130 break; \
2131 } \
2132 case Instruction::APUT##kind: { \
2133 break; \
2134 }
2135
2136 ARRAY_XX(, DataType::Type::kInt32);
2137 ARRAY_XX(_WIDE, DataType::Type::kInt64);
2138 ARRAY_XX(_OBJECT, DataType::Type::kReference);
2139 ARRAY_XX(_BOOLEAN, DataType::Type::kBool);
2140 ARRAY_XX(_BYTE, DataType::Type::kInt8);
2141 ARRAY_XX(_CHAR, DataType::Type::kUint16);
2142 ARRAY_XX(_SHORT, DataType::Type::kInt16);
2143
2144 case Instruction::ARRAY_LENGTH: {
2145 break;
2146 }
2147
2148 case Instruction::CONST_STRING: {
2149 dex::StringIndex string_index(instruction.VRegB_21c());
2150 return BuildLoadString(instruction.VRegA_21c(), string_index, next);
2151 }
2152
2153 case Instruction::CONST_STRING_JUMBO: {
2154 dex::StringIndex string_index(instruction.VRegB_31c());
2155 return BuildLoadString(instruction.VRegA_31c(), string_index, next);
2156 }
2157
2158 case Instruction::CONST_CLASS: {
2159 break;
2160 }
2161
2162 case Instruction::CONST_METHOD_HANDLE: {
2163 break;
2164 }
2165
2166 case Instruction::CONST_METHOD_TYPE: {
2167 break;
2168 }
2169
2170 case Instruction::MOVE_EXCEPTION: {
2171 break;
2172 }
2173
2174 case Instruction::THROW: {
2175 if (!EnsureHasFrame()) {
2176 return false;
2177 }
2178 int32_t reg = instruction.VRegA_11x();
2179 InvokeRuntimeCallingConvention calling_convention;
2180 if (!MoveLocation(LocationFrom(calling_convention.GetRegisterAt(0)),
2181 vreg_locations_[reg],
2182 DataType::Type::kReference)) {
2183 return false;
2184 }
2185 InvokeRuntime(kQuickDeliverException, dex_pc);
2186 return true;
2187 }
2188
2189 case Instruction::INSTANCE_OF: {
2190 break;
2191 }
2192
2193 case Instruction::CHECK_CAST: {
2194 uint8_t reference = instruction.VRegA_21c();
2195 dex::TypeIndex type_index(instruction.VRegB_21c());
2196 return BuildCheckCast(reference, type_index, dex_pc);
2197 }
2198
2199 case Instruction::MONITOR_ENTER: {
2200 break;
2201 }
2202
2203 case Instruction::MONITOR_EXIT: {
2204 break;
2205 }
2206
2207 case Instruction::SPARSE_SWITCH:
2208 case Instruction::PACKED_SWITCH: {
2209 break;
2210 }
2211
2212 case Instruction::UNUSED_3E ... Instruction::UNUSED_43:
2213 case Instruction::UNUSED_73:
2214 case Instruction::UNUSED_79:
2215 case Instruction::UNUSED_7A:
2216 case Instruction::UNUSED_E3 ... Instruction::UNUSED_F9: {
2217 break;
2218 }
2219 }
2220 unimplemented_reason_ = instruction.Name();
2221 return false;
2222 } // NOLINT(readability/fn_size)
2223
Compile()2224 bool FastCompilerARM64::Compile() {
2225 if (!InitializeParameters()) {
2226 DCHECK(HitUnimplemented());
2227 AbortCompilation();
2228 return false;
2229 }
2230 if (!ProcessInstructions()) {
2231 DCHECK(HitUnimplemented());
2232 AbortCompilation();
2233 return false;
2234 }
2235 DCHECK(!HitUnimplemented()) << GetUnimplementedReason();
2236 if (!has_frame_) {
2237 code_generation_data_->GetStackMapStream()->BeginMethod(/* frame_size= */ 0u,
2238 /* core_spill_mask= */ 0u,
2239 /* fp_spill_mask= */ 0u,
2240 GetCodeItemAccessor().RegistersSize(),
2241 /* is_compiling_baseline= */ true,
2242 /* is_debuggable= */ false);
2243 }
2244 code_generation_data_->GetStackMapStream()->EndMethod(assembler_.CodeSize());
2245 assembler_.FinalizeCode();
2246
2247 if (VLOG_IS_ON(jit)) {
2248 // Dump the generated code
2249 {
2250 ScopedObjectAccess soa(Thread::Current());
2251 VLOG(jit) << "Dumping generated fast baseline code for " << method_->PrettyMethod();
2252 }
2253 FILE* file = tmpfile();
2254 MacroAssembler* masm = GetVIXLAssembler();
2255 PrintDisassembler print_disasm(file);
2256 vixl::aarch64::Instruction* dis_start =
2257 masm->GetBuffer()->GetStartAddress<vixl::aarch64::Instruction*>();
2258 vixl::aarch64::Instruction* dis_end =
2259 masm->GetBuffer()->GetEndAddress<vixl::aarch64::Instruction*>();
2260 print_disasm.DisassembleBuffer(dis_start, dis_end);
2261 fseek(file, 0L, SEEK_SET);
2262 char buffer[1024];
2263 const char* line;
2264 while ((line = fgets(buffer, sizeof(buffer), file)) != nullptr) {
2265 VLOG(jit) << std::string(line);
2266 }
2267 fclose(file);
2268 }
2269 return true;
2270 }
2271
2272 } // namespace arm64
2273
CompileARM64(ArtMethod * method,ArenaAllocator * allocator,ArenaStack * arena_stack,VariableSizedHandleScope * handles,const CompilerOptions & compiler_options,const DexCompilationUnit & dex_compilation_unit)2274 std::unique_ptr<FastCompiler> FastCompiler::CompileARM64(
2275 ArtMethod* method,
2276 ArenaAllocator* allocator,
2277 ArenaStack* arena_stack,
2278 VariableSizedHandleScope* handles,
2279 const CompilerOptions& compiler_options,
2280 const DexCompilationUnit& dex_compilation_unit) {
2281 if (!compiler_options.GetImplicitNullChecks() ||
2282 !compiler_options.GetImplicitStackOverflowChecks() ||
2283 kUseTableLookupReadBarrier ||
2284 !kReserveMarkingRegister ||
2285 kPoisonHeapReferences) {
2286 // Configurations we don't support.
2287 return nullptr;
2288 }
2289 std::unique_ptr<arm64::FastCompilerARM64> compiler(new arm64::FastCompilerARM64(
2290 method,
2291 allocator,
2292 arena_stack,
2293 handles,
2294 compiler_options,
2295 dex_compilation_unit));
2296 if (compiler->Compile()) {
2297 return compiler;
2298 }
2299 VLOG(jit) << "Did not fast compile because of " << compiler->GetUnimplementedReason();
2300 return nullptr;
2301 }
2302
2303 } // namespace art
2304