1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
18 #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
19
20 #include "base/bit_field.h"
21 #include "base/macros.h"
22 #include "class_root.h"
23 #include "code_generator.h"
24 #include "common_arm64.h"
25 #include "dex/dex_file_types.h"
26 #include "dex/string_reference.h"
27 #include "dex/type_reference.h"
28 #include "driver/compiler_options.h"
29 #include "nodes.h"
30 #include "parallel_move_resolver.h"
31 #include "utils/arm64/assembler_arm64.h"
32
33 // TODO(VIXL): Make VIXL compile with -Wshadow.
34 #pragma GCC diagnostic push
35 #pragma GCC diagnostic ignored "-Wshadow"
36 #include "aarch64/disasm-aarch64.h"
37 #include "aarch64/macro-assembler-aarch64.h"
38 #pragma GCC diagnostic pop
39
40 namespace art HIDDEN {
41
42 namespace linker {
43 class Arm64RelativePatcherTest;
44 } // namespace linker
45
46 namespace arm64 {
47
48 class CodeGeneratorARM64;
49
50 // Use a local definition to prevent copying mistakes.
51 static constexpr size_t kArm64WordSize = static_cast<size_t>(kArm64PointerSize);
52
53 // These constants are used as an approximate margin when emission of veneer and literal pools
54 // must be blocked.
55 static constexpr int kMaxMacroInstructionSizeInBytes = 15 * vixl::aarch64::kInstructionSize;
56 static constexpr int kInvokeCodeMarginSizeInBytes = 6 * kMaxMacroInstructionSizeInBytes;
57
58 static const vixl::aarch64::Register kParameterCoreRegisters[] = {
59 vixl::aarch64::x1,
60 vixl::aarch64::x2,
61 vixl::aarch64::x3,
62 vixl::aarch64::x4,
63 vixl::aarch64::x5,
64 vixl::aarch64::x6,
65 vixl::aarch64::x7
66 };
67 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters);
68 static const vixl::aarch64::VRegister kParameterFPRegisters[] = {
69 vixl::aarch64::d0,
70 vixl::aarch64::d1,
71 vixl::aarch64::d2,
72 vixl::aarch64::d3,
73 vixl::aarch64::d4,
74 vixl::aarch64::d5,
75 vixl::aarch64::d6,
76 vixl::aarch64::d7
77 };
78 static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters);
79
80 // Thread Register.
81 const vixl::aarch64::Register tr = vixl::aarch64::x19;
82 // Marking Register.
83 const vixl::aarch64::Register mr = vixl::aarch64::x20;
84 // Implicit suspend check register.
85 const vixl::aarch64::Register kImplicitSuspendCheckRegister = vixl::aarch64::x21;
86 // Method register on invoke.
87 static const vixl::aarch64::Register kArtMethodRegister = vixl::aarch64::x0;
88 const vixl::aarch64::CPURegList vixl_reserved_core_registers(vixl::aarch64::ip0,
89 vixl::aarch64::ip1);
90 const vixl::aarch64::CPURegList vixl_reserved_fp_registers(vixl::aarch64::d31);
91
92 const vixl::aarch64::CPURegList runtime_reserved_core_registers =
93 vixl::aarch64::CPURegList(
94 tr,
95 // Reserve X20 as Marking Register when emitting Baker read barriers.
96 // TODO: We don't need to reserve marking-register for userfaultfd GC. But
97 // that would require some work in the assembler code as the right GC is
98 // chosen at load-time and not compile time.
99 (kReserveMarkingRegister ? mr : vixl::aarch64::NoCPUReg),
100 kImplicitSuspendCheckRegister,
101 vixl::aarch64::lr);
102
103 // Some instructions have special requirements for a temporary, for example
104 // LoadClass/kBssEntry and LoadString/kBssEntry for Baker read barrier require
105 // temp that's not an R0 (to avoid an extra move) and Baker read barrier field
106 // loads with large offsets need a fixed register to limit the number of link-time
107 // thunks we generate. For these and similar cases, we want to reserve a specific
108 // register that's neither callee-save nor an argument register. We choose x15.
FixedTempLocation()109 inline Location FixedTempLocation() {
110 return Location::RegisterLocation(vixl::aarch64::x15.GetCode());
111 }
112
113 // Callee-save registers AAPCS64, without x19 (Thread Register) (nor
114 // x20 (Marking Register) when emitting Baker read barriers).
115 const vixl::aarch64::CPURegList callee_saved_core_registers(
116 vixl::aarch64::CPURegister::kRegister,
117 vixl::aarch64::kXRegSize,
118 (kReserveMarkingRegister ? vixl::aarch64::x21.GetCode() : vixl::aarch64::x20.GetCode()),
119 vixl::aarch64::x30.GetCode());
120 const vixl::aarch64::CPURegList callee_saved_fp_registers(vixl::aarch64::CPURegister::kVRegister,
121 vixl::aarch64::kDRegSize,
122 vixl::aarch64::d8.GetCode(),
123 vixl::aarch64::d15.GetCode());
124 Location ARM64ReturnLocation(DataType::Type return_type);
125
126 #define UNIMPLEMENTED_INTRINSIC_LIST_ARM64(V) \
127 V(StringStringIndexOf) \
128 V(StringStringIndexOfAfter) \
129 V(StringBufferAppend) \
130 V(StringBufferLength) \
131 V(StringBufferToString) \
132 V(StringBuilderAppendObject) \
133 V(StringBuilderAppendString) \
134 V(StringBuilderAppendCharSequence) \
135 V(StringBuilderAppendCharArray) \
136 V(StringBuilderAppendBoolean) \
137 V(StringBuilderAppendChar) \
138 V(StringBuilderAppendInt) \
139 V(StringBuilderAppendLong) \
140 V(StringBuilderAppendFloat) \
141 V(StringBuilderAppendDouble) \
142 V(StringBuilderLength) \
143 V(StringBuilderToString) \
144 V(SystemArrayCopyByte) \
145 V(SystemArrayCopyInt) \
146 /* 1.8 */ \
147 V(UnsafeGetAndAddInt) \
148 V(UnsafeGetAndAddLong) \
149 V(UnsafeGetAndSetInt) \
150 V(UnsafeGetAndSetLong) \
151 V(UnsafeGetAndSetObject) \
152 V(MethodHandleInvokeExact) \
153 V(MethodHandleInvoke) \
154 /* OpenJDK 11 */ \
155 V(JdkUnsafeGetAndAddInt) \
156 V(JdkUnsafeGetAndAddLong) \
157 V(JdkUnsafeGetAndSetInt) \
158 V(JdkUnsafeGetAndSetLong) \
159 V(JdkUnsafeGetAndSetObject)
160
161 class SlowPathCodeARM64 : public SlowPathCode {
162 public:
SlowPathCodeARM64(HInstruction * instruction)163 explicit SlowPathCodeARM64(HInstruction* instruction)
164 : SlowPathCode(instruction), entry_label_(), exit_label_() {}
165
GetEntryLabel()166 vixl::aarch64::Label* GetEntryLabel() { return &entry_label_; }
GetExitLabel()167 vixl::aarch64::Label* GetExitLabel() { return &exit_label_; }
168
169 void SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override;
170 void RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override;
171
172 private:
173 vixl::aarch64::Label entry_label_;
174 vixl::aarch64::Label exit_label_;
175
176 DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64);
177 };
178
179 class JumpTableARM64 : public DeletableArenaObject<kArenaAllocSwitchTable> {
180 public:
JumpTableARM64(HPackedSwitch * switch_instr)181 explicit JumpTableARM64(HPackedSwitch* switch_instr)
182 : switch_instr_(switch_instr), table_start_() {}
183
GetTableStartLabel()184 vixl::aarch64::Label* GetTableStartLabel() { return &table_start_; }
185
186 void EmitTable(CodeGeneratorARM64* codegen);
187
188 private:
189 HPackedSwitch* const switch_instr_;
190 vixl::aarch64::Label table_start_;
191
192 DISALLOW_COPY_AND_ASSIGN(JumpTableARM64);
193 };
194
195 static const vixl::aarch64::Register kRuntimeParameterCoreRegisters[] =
196 { vixl::aarch64::x0,
197 vixl::aarch64::x1,
198 vixl::aarch64::x2,
199 vixl::aarch64::x3,
200 vixl::aarch64::x4,
201 vixl::aarch64::x5,
202 vixl::aarch64::x6,
203 vixl::aarch64::x7 };
204 static constexpr size_t kRuntimeParameterCoreRegistersLength =
205 arraysize(kRuntimeParameterCoreRegisters);
206 static const vixl::aarch64::VRegister kRuntimeParameterFpuRegisters[] =
207 { vixl::aarch64::d0,
208 vixl::aarch64::d1,
209 vixl::aarch64::d2,
210 vixl::aarch64::d3,
211 vixl::aarch64::d4,
212 vixl::aarch64::d5,
213 vixl::aarch64::d6,
214 vixl::aarch64::d7 };
215 static constexpr size_t kRuntimeParameterFpuRegistersLength =
216 arraysize(kRuntimeParameterCoreRegisters);
217
218 class InvokeRuntimeCallingConvention : public CallingConvention<vixl::aarch64::Register,
219 vixl::aarch64::VRegister> {
220 public:
221 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters);
222
InvokeRuntimeCallingConvention()223 InvokeRuntimeCallingConvention()
224 : CallingConvention(kRuntimeParameterCoreRegisters,
225 kRuntimeParameterCoreRegistersLength,
226 kRuntimeParameterFpuRegisters,
227 kRuntimeParameterFpuRegistersLength,
228 kArm64PointerSize) {}
229
230 Location GetReturnLocation(DataType::Type return_type);
231
232 private:
233 DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
234 };
235
236 class InvokeDexCallingConvention : public CallingConvention<vixl::aarch64::Register,
237 vixl::aarch64::VRegister> {
238 public:
InvokeDexCallingConvention()239 InvokeDexCallingConvention()
240 : CallingConvention(kParameterCoreRegisters,
241 kParameterCoreRegistersLength,
242 kParameterFPRegisters,
243 kParameterFPRegistersLength,
244 kArm64PointerSize) {}
245
GetReturnLocation(DataType::Type return_type)246 Location GetReturnLocation(DataType::Type return_type) const {
247 return ARM64ReturnLocation(return_type);
248 }
249
250
251 private:
252 DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention);
253 };
254
255 class InvokeDexCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor {
256 public:
InvokeDexCallingConventionVisitorARM64()257 InvokeDexCallingConventionVisitorARM64() {}
~InvokeDexCallingConventionVisitorARM64()258 virtual ~InvokeDexCallingConventionVisitorARM64() {}
259
260 Location GetNextLocation(DataType::Type type) override;
GetReturnLocation(DataType::Type return_type)261 Location GetReturnLocation(DataType::Type return_type) const override {
262 return calling_convention.GetReturnLocation(return_type);
263 }
264 Location GetMethodLocation() const override;
265
266 private:
267 InvokeDexCallingConvention calling_convention;
268
269 DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorARM64);
270 };
271
272 class CriticalNativeCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor {
273 public:
CriticalNativeCallingConventionVisitorARM64(bool for_register_allocation)274 explicit CriticalNativeCallingConventionVisitorARM64(bool for_register_allocation)
275 : for_register_allocation_(for_register_allocation) {}
276
~CriticalNativeCallingConventionVisitorARM64()277 virtual ~CriticalNativeCallingConventionVisitorARM64() {}
278
279 Location GetNextLocation(DataType::Type type) override;
280 Location GetReturnLocation(DataType::Type type) const override;
281 Location GetMethodLocation() const override;
282
GetStackOffset()283 size_t GetStackOffset() const { return stack_offset_; }
284
285 private:
286 // Register allocator does not support adjusting frame size, so we cannot provide final locations
287 // of stack arguments for register allocation. We ask the register allocator for any location and
288 // move these arguments to the right place after adjusting the SP when generating the call.
289 const bool for_register_allocation_;
290 size_t gpr_index_ = 0u;
291 size_t fpr_index_ = 0u;
292 size_t stack_offset_ = 0u;
293
294 DISALLOW_COPY_AND_ASSIGN(CriticalNativeCallingConventionVisitorARM64);
295 };
296
297 class FieldAccessCallingConventionARM64 : public FieldAccessCallingConvention {
298 public:
FieldAccessCallingConventionARM64()299 FieldAccessCallingConventionARM64() {}
300
GetObjectLocation()301 Location GetObjectLocation() const override {
302 return helpers::LocationFrom(vixl::aarch64::x1);
303 }
GetFieldIndexLocation()304 Location GetFieldIndexLocation() const override {
305 return helpers::LocationFrom(vixl::aarch64::x0);
306 }
GetReturnLocation(DataType::Type type ATTRIBUTE_UNUSED)307 Location GetReturnLocation(DataType::Type type ATTRIBUTE_UNUSED) const override {
308 return helpers::LocationFrom(vixl::aarch64::x0);
309 }
GetSetValueLocation(DataType::Type type ATTRIBUTE_UNUSED,bool is_instance)310 Location GetSetValueLocation(DataType::Type type ATTRIBUTE_UNUSED,
311 bool is_instance) const override {
312 return is_instance
313 ? helpers::LocationFrom(vixl::aarch64::x2)
314 : helpers::LocationFrom(vixl::aarch64::x1);
315 }
GetFpuLocation(DataType::Type type ATTRIBUTE_UNUSED)316 Location GetFpuLocation(DataType::Type type ATTRIBUTE_UNUSED) const override {
317 return helpers::LocationFrom(vixl::aarch64::d0);
318 }
319
320 private:
321 DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionARM64);
322 };
323
324 class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator {
325 public:
326 InstructionCodeGeneratorARM64(HGraph* graph, CodeGeneratorARM64* codegen);
327
328 #define DECLARE_VISIT_INSTRUCTION(name, super) \
329 void Visit##name(H##name* instr) override;
330
331 FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)332 FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)
333 FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
334
335 #undef DECLARE_VISIT_INSTRUCTION
336
337 void VisitInstruction(HInstruction* instruction) override {
338 LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
339 << " (id " << instruction->GetId() << ")";
340 }
341
GetAssembler()342 Arm64Assembler* GetAssembler() const { return assembler_; }
GetVIXLAssembler()343 vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
344
345 // SIMD helpers.
346 virtual Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) = 0;
347 virtual void FreeSIMDScratchLocation(Location loc,
348 vixl::aarch64::UseScratchRegisterScope* scope) = 0;
349 virtual void LoadSIMDRegFromStack(Location destination, Location source) = 0;
350 virtual void MoveSIMDRegToSIMDReg(Location destination, Location source) = 0;
351 virtual void MoveToSIMDStackSlot(Location destination, Location source) = 0;
352 virtual void SaveLiveRegistersHelper(LocationSummary* locations,
353 int64_t spill_offset) = 0;
354 virtual void RestoreLiveRegistersHelper(LocationSummary* locations,
355 int64_t spill_offset) = 0;
356
357 protected:
358 void GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path,
359 vixl::aarch64::Register class_reg);
360 void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check,
361 vixl::aarch64::Register temp);
362 void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor);
363 void HandleBinaryOp(HBinaryOperation* instr);
364
365 void HandleFieldSet(HInstruction* instruction,
366 const FieldInfo& field_info,
367 bool value_can_be_null,
368 WriteBarrierKind write_barrier_kind);
369 void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
370 void HandleCondition(HCondition* instruction);
371
372 // Generate a heap reference load using one register `out`:
373 //
374 // out <- *(out + offset)
375 //
376 // while honoring heap poisoning and/or read barriers (if any).
377 //
378 // Location `maybe_temp` is used when generating a read barrier and
379 // shall be a register in that case; it may be an invalid location
380 // otherwise.
381 void GenerateReferenceLoadOneRegister(HInstruction* instruction,
382 Location out,
383 uint32_t offset,
384 Location maybe_temp,
385 ReadBarrierOption read_barrier_option);
386 // Generate a heap reference load using two different registers
387 // `out` and `obj`:
388 //
389 // out <- *(obj + offset)
390 //
391 // while honoring heap poisoning and/or read barriers (if any).
392 //
393 // Location `maybe_temp` is used when generating a Baker's (fast
394 // path) read barrier and shall be a register in that case; it may
395 // be an invalid location otherwise.
396 void GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
397 Location out,
398 Location obj,
399 uint32_t offset,
400 Location maybe_temp,
401 ReadBarrierOption read_barrier_option);
402
403 // Generate a floating-point comparison.
404 void GenerateFcmp(HInstruction* instruction);
405
406 void HandleShift(HBinaryOperation* instr);
407 void GenerateTestAndBranch(HInstruction* instruction,
408 size_t condition_input_index,
409 vixl::aarch64::Label* true_target,
410 vixl::aarch64::Label* false_target);
411 void DivRemOneOrMinusOne(HBinaryOperation* instruction);
412 void DivRemByPowerOfTwo(HBinaryOperation* instruction);
413 void GenerateIncrementNegativeByOne(vixl::aarch64::Register out,
414 vixl::aarch64::Register in, bool use_cond_inc);
415 void GenerateResultRemWithAnyConstant(vixl::aarch64::Register out,
416 vixl::aarch64::Register dividend,
417 vixl::aarch64::Register quotient,
418 int64_t divisor,
419 // This function may acquire a scratch register.
420 vixl::aarch64::UseScratchRegisterScope* temps_scope);
421 void GenerateInt64UnsignedDivRemWithAnyPositiveConstant(HBinaryOperation* instruction);
422 void GenerateInt64DivRemWithAnyConstant(HBinaryOperation* instruction);
423 void GenerateInt32DivRemWithAnyConstant(HBinaryOperation* instruction);
424 void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction, int64_t divisor);
425 void GenerateIntDiv(HDiv* instruction);
426 void GenerateIntDivForConstDenom(HDiv *instruction);
427 void GenerateIntDivForPower2Denom(HDiv *instruction);
428 void GenerateIntRem(HRem* instruction);
429 void GenerateIntRemForConstDenom(HRem *instruction);
430 void GenerateIntRemForPower2Denom(HRem *instruction);
431 void HandleGoto(HInstruction* got, HBasicBlock* successor);
432 void GenerateMethodEntryExitHook(HInstruction* instruction);
433
434 // Helpers to set up locations for vector memory operations. Returns the memory operand and,
435 // if used, sets the output parameter scratch to a temporary register used in this operand,
436 // so that the client can release it right after the memory operand use.
437 // Neon version.
438 vixl::aarch64::MemOperand VecNEONAddress(
439 HVecMemoryOperation* instruction,
440 // This function may acquire a scratch register.
441 vixl::aarch64::UseScratchRegisterScope* temps_scope,
442 size_t size,
443 bool is_string_char_at,
444 /*out*/ vixl::aarch64::Register* scratch);
445 // SVE version.
446 vixl::aarch64::SVEMemOperand VecSVEAddress(
447 HVecMemoryOperation* instruction,
448 // This function may acquire a scratch register.
449 vixl::aarch64::UseScratchRegisterScope* temps_scope,
450 size_t size,
451 bool is_string_char_at,
452 /*out*/ vixl::aarch64::Register* scratch);
453
454 Arm64Assembler* const assembler_;
455 CodeGeneratorARM64* const codegen_;
456
457 DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARM64);
458 };
459
460 class LocationsBuilderARM64 : public HGraphVisitor {
461 public:
LocationsBuilderARM64(HGraph * graph,CodeGeneratorARM64 * codegen)462 LocationsBuilderARM64(HGraph* graph, CodeGeneratorARM64* codegen)
463 : HGraphVisitor(graph), codegen_(codegen) {}
464
465 #define DECLARE_VISIT_INSTRUCTION(name, super) \
466 void Visit##name(H##name* instr) override;
467
468 FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)469 FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)
470 FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
471
472 #undef DECLARE_VISIT_INSTRUCTION
473
474 void VisitInstruction(HInstruction* instruction) override {
475 LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
476 << " (id " << instruction->GetId() << ")";
477 }
478
479 protected:
480 void HandleBinaryOp(HBinaryOperation* instr);
481 void HandleFieldSet(HInstruction* instruction);
482 void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
483 void HandleInvoke(HInvoke* instr);
484 void HandleCondition(HCondition* instruction);
485 void HandleShift(HBinaryOperation* instr);
486
487 CodeGeneratorARM64* const codegen_;
488 InvokeDexCallingConventionVisitorARM64 parameter_visitor_;
489
490 DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARM64);
491 };
492
493 class InstructionCodeGeneratorARM64Neon : public InstructionCodeGeneratorARM64 {
494 public:
InstructionCodeGeneratorARM64Neon(HGraph * graph,CodeGeneratorARM64 * codegen)495 InstructionCodeGeneratorARM64Neon(HGraph* graph, CodeGeneratorARM64* codegen) :
496 InstructionCodeGeneratorARM64(graph, codegen) {}
497
498 #define DECLARE_VISIT_INSTRUCTION(name, super) \
499 void Visit##name(H##name* instr) override;
500
501 FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
502
503 #undef DECLARE_VISIT_INSTRUCTION
504
505 Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) override;
506 void FreeSIMDScratchLocation(Location loc,
507 vixl::aarch64::UseScratchRegisterScope* scope) override;
508 void LoadSIMDRegFromStack(Location destination, Location source) override;
509 void MoveSIMDRegToSIMDReg(Location destination, Location source) override;
510 void MoveToSIMDStackSlot(Location destination, Location source) override;
511 void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
512 void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
513 };
514
515 class LocationsBuilderARM64Neon : public LocationsBuilderARM64 {
516 public:
LocationsBuilderARM64Neon(HGraph * graph,CodeGeneratorARM64 * codegen)517 LocationsBuilderARM64Neon(HGraph* graph, CodeGeneratorARM64* codegen) :
518 LocationsBuilderARM64(graph, codegen) {}
519
520 #define DECLARE_VISIT_INSTRUCTION(name, super) \
521 void Visit##name(H##name* instr) override;
522
523 FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
524
525 #undef DECLARE_VISIT_INSTRUCTION
526 };
527
528 class InstructionCodeGeneratorARM64Sve : public InstructionCodeGeneratorARM64 {
529 public:
InstructionCodeGeneratorARM64Sve(HGraph * graph,CodeGeneratorARM64 * codegen)530 InstructionCodeGeneratorARM64Sve(HGraph* graph, CodeGeneratorARM64* codegen) :
531 InstructionCodeGeneratorARM64(graph, codegen) {}
532
533 #define DECLARE_VISIT_INSTRUCTION(name, super) \
534 void Visit##name(H##name* instr) override;
535
536 FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
537
538 #undef DECLARE_VISIT_INSTRUCTION
539
540 Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) override;
541 void FreeSIMDScratchLocation(Location loc,
542 vixl::aarch64::UseScratchRegisterScope* scope) override;
543 void LoadSIMDRegFromStack(Location destination, Location source) override;
544 void MoveSIMDRegToSIMDReg(Location destination, Location source) override;
545 void MoveToSIMDStackSlot(Location destination, Location source) override;
546 void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
547 void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
548
549 private:
550 // Validate that instruction vector length and packed type are compliant with the SIMD
551 // register size (full SIMD register is used).
552 void ValidateVectorLength(HVecOperation* instr) const;
553
554 // Returns default predicate register which is used as governing vector predicate
555 // to implement predicated loop execution.
556 //
557 // TODO: This is a hack to be addressed when register allocator supports SIMD types.
LoopPReg()558 static vixl::aarch64::PRegister LoopPReg() {
559 return vixl::aarch64::p0;
560 }
561 };
562
563 class LocationsBuilderARM64Sve : public LocationsBuilderARM64 {
564 public:
LocationsBuilderARM64Sve(HGraph * graph,CodeGeneratorARM64 * codegen)565 LocationsBuilderARM64Sve(HGraph* graph, CodeGeneratorARM64* codegen) :
566 LocationsBuilderARM64(graph, codegen) {}
567
568 #define DECLARE_VISIT_INSTRUCTION(name, super) \
569 void Visit##name(H##name* instr) override;
570
571 FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
572
573 #undef DECLARE_VISIT_INSTRUCTION
574 };
575
576 class ParallelMoveResolverARM64 : public ParallelMoveResolverNoSwap {
577 public:
ParallelMoveResolverARM64(ArenaAllocator * allocator,CodeGeneratorARM64 * codegen)578 ParallelMoveResolverARM64(ArenaAllocator* allocator, CodeGeneratorARM64* codegen)
579 : ParallelMoveResolverNoSwap(allocator), codegen_(codegen), vixl_temps_() {}
580
581 protected:
582 void PrepareForEmitNativeCode() override;
583 void FinishEmitNativeCode() override;
584 Location AllocateScratchLocationFor(Location::Kind kind) override;
585 void FreeScratchLocation(Location loc) override;
586 void EmitMove(size_t index) override;
587
588 private:
589 Arm64Assembler* GetAssembler() const;
GetVIXLAssembler()590 vixl::aarch64::MacroAssembler* GetVIXLAssembler() const {
591 return GetAssembler()->GetVIXLAssembler();
592 }
593
594 CodeGeneratorARM64* const codegen_;
595 vixl::aarch64::UseScratchRegisterScope vixl_temps_;
596
597 DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverARM64);
598 };
599
600 class CodeGeneratorARM64 : public CodeGenerator {
601 public:
602 CodeGeneratorARM64(HGraph* graph,
603 const CompilerOptions& compiler_options,
604 OptimizingCompilerStats* stats = nullptr);
~CodeGeneratorARM64()605 virtual ~CodeGeneratorARM64() {}
606
607 void GenerateFrameEntry() override;
608 void GenerateFrameExit() override;
609
610 vixl::aarch64::CPURegList GetFramePreservedCoreRegisters() const;
611 vixl::aarch64::CPURegList GetFramePreservedFPRegisters() const;
612
613 void Bind(HBasicBlock* block) override;
614
GetLabelOf(HBasicBlock * block)615 vixl::aarch64::Label* GetLabelOf(HBasicBlock* block) {
616 block = FirstNonEmptyBlock(block);
617 return &(block_labels_[block->GetBlockId()]);
618 }
619
GetWordSize()620 size_t GetWordSize() const override {
621 return kArm64WordSize;
622 }
623
SupportsPredicatedSIMD()624 bool SupportsPredicatedSIMD() const override { return ShouldUseSVE(); }
625
GetSlowPathFPWidth()626 size_t GetSlowPathFPWidth() const override {
627 return GetGraph()->HasSIMD()
628 ? GetSIMDRegisterWidth()
629 : vixl::aarch64::kDRegSizeInBytes;
630 }
631
GetCalleePreservedFPWidth()632 size_t GetCalleePreservedFPWidth() const override {
633 return vixl::aarch64::kDRegSizeInBytes;
634 }
635
636 size_t GetSIMDRegisterWidth() const override;
637
GetAddressOf(HBasicBlock * block)638 uintptr_t GetAddressOf(HBasicBlock* block) override {
639 vixl::aarch64::Label* block_entry_label = GetLabelOf(block);
640 DCHECK(block_entry_label->IsBound());
641 return block_entry_label->GetLocation();
642 }
643
GetLocationBuilder()644 HGraphVisitor* GetLocationBuilder() override { return location_builder_; }
GetInstructionCodeGeneratorArm64()645 InstructionCodeGeneratorARM64* GetInstructionCodeGeneratorArm64() {
646 return instruction_visitor_;
647 }
GetInstructionVisitor()648 HGraphVisitor* GetInstructionVisitor() override { return GetInstructionCodeGeneratorArm64(); }
GetAssembler()649 Arm64Assembler* GetAssembler() override { return &assembler_; }
GetAssembler()650 const Arm64Assembler& GetAssembler() const override { return assembler_; }
GetVIXLAssembler()651 vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
652
653 // Emit a write barrier.
654 void MarkGCCard(vixl::aarch64::Register object,
655 vixl::aarch64::Register value,
656 bool emit_null_check);
657
658 void GenerateMemoryBarrier(MemBarrierKind kind);
659
660 // Register allocation.
661
662 void SetupBlockedRegisters() const override;
663
664 size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) override;
665 size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) override;
666 size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
667 size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
668
669 // The number of registers that can be allocated. The register allocator may
670 // decide to reserve and not use a few of them.
671 // We do not consider registers sp, xzr, wzr. They are either not allocatable
672 // (xzr, wzr), or make for poor allocatable registers (sp alignment
673 // requirements, etc.). This also facilitates our task as all other registers
674 // can easily be mapped via to or from their type and index or code.
675 static const int kNumberOfAllocatableRegisters = vixl::aarch64::kNumberOfRegisters - 1;
676 static const int kNumberOfAllocatableFPRegisters = vixl::aarch64::kNumberOfVRegisters;
677 static constexpr int kNumberOfAllocatableRegisterPairs = 0;
678
679 void DumpCoreRegister(std::ostream& stream, int reg) const override;
680 void DumpFloatingPointRegister(std::ostream& stream, int reg) const override;
681
GetInstructionSet()682 InstructionSet GetInstructionSet() const override {
683 return InstructionSet::kArm64;
684 }
685
686 const Arm64InstructionSetFeatures& GetInstructionSetFeatures() const;
687
Initialize()688 void Initialize() override {
689 block_labels_.resize(GetGraph()->GetBlocks().size());
690 }
691
692 // We want to use the STP and LDP instructions to spill and restore registers for slow paths.
693 // These instructions can only encode offsets that are multiples of the register size accessed.
GetPreferredSlotsAlignment()694 uint32_t GetPreferredSlotsAlignment() const override { return vixl::aarch64::kXRegSizeInBytes; }
695
CreateJumpTable(HPackedSwitch * switch_instr)696 JumpTableARM64* CreateJumpTable(HPackedSwitch* switch_instr) {
697 jump_tables_.emplace_back(new (GetGraph()->GetAllocator()) JumpTableARM64(switch_instr));
698 return jump_tables_.back().get();
699 }
700
701 void Finalize(CodeAllocator* allocator) override;
702
703 // Code generation helpers.
704 void MoveConstant(vixl::aarch64::CPURegister destination, HConstant* constant);
705 void MoveConstant(Location destination, int32_t value) override;
706 void MoveLocation(Location dst, Location src, DataType::Type dst_type) override;
707 void AddLocationAsTemp(Location location, LocationSummary* locations) override;
708
709 void Load(DataType::Type type,
710 vixl::aarch64::CPURegister dst,
711 const vixl::aarch64::MemOperand& src);
712 void Store(DataType::Type type,
713 vixl::aarch64::CPURegister src,
714 const vixl::aarch64::MemOperand& dst);
715 void LoadAcquire(HInstruction* instruction,
716 DataType::Type type,
717 vixl::aarch64::CPURegister dst,
718 const vixl::aarch64::MemOperand& src,
719 bool needs_null_check);
720 void StoreRelease(HInstruction* instruction,
721 DataType::Type type,
722 vixl::aarch64::CPURegister src,
723 const vixl::aarch64::MemOperand& dst,
724 bool needs_null_check);
725
726 // Generate code to invoke a runtime entry point.
727 void InvokeRuntime(QuickEntrypointEnum entrypoint,
728 HInstruction* instruction,
729 uint32_t dex_pc,
730 SlowPathCode* slow_path = nullptr) override;
731
732 // Generate code to invoke a runtime entry point, but do not record
733 // PC-related information in a stack map.
734 void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset,
735 HInstruction* instruction,
736 SlowPathCode* slow_path);
737
GetMoveResolver()738 ParallelMoveResolverARM64* GetMoveResolver() override { return &move_resolver_; }
739
NeedsTwoRegisters(DataType::Type type ATTRIBUTE_UNUSED)740 bool NeedsTwoRegisters(DataType::Type type ATTRIBUTE_UNUSED) const override {
741 return false;
742 }
743
744 // Check if the desired_string_load_kind is supported. If it is, return it,
745 // otherwise return a fall-back kind that should be used instead.
746 HLoadString::LoadKind GetSupportedLoadStringKind(
747 HLoadString::LoadKind desired_string_load_kind) override;
748
749 // Check if the desired_class_load_kind is supported. If it is, return it,
750 // otherwise return a fall-back kind that should be used instead.
751 HLoadClass::LoadKind GetSupportedLoadClassKind(
752 HLoadClass::LoadKind desired_class_load_kind) override;
753
754 // Check if the desired_dispatch_info is supported. If it is, return it,
755 // otherwise return a fall-back info that should be used instead.
756 HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
757 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
758 ArtMethod* method) override;
759
760 void LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke);
761 void GenerateStaticOrDirectCall(
762 HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
763 void GenerateVirtualCall(
764 HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
765
766 void MoveFromReturnRegister(Location trg, DataType::Type type) override;
767
768 // Add a new boot image intrinsic patch for an instruction and return the label
769 // to be bound before the instruction. The instruction will be either the
770 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
771 // to the associated ADRP patch label).
772 vixl::aarch64::Label* NewBootImageIntrinsicPatch(uint32_t intrinsic_data,
773 vixl::aarch64::Label* adrp_label = nullptr);
774
775 // Add a new boot image relocation patch for an instruction and return the label
776 // to be bound before the instruction. The instruction will be either the
777 // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing
778 // to the associated ADRP patch label).
779 vixl::aarch64::Label* NewBootImageRelRoPatch(uint32_t boot_image_offset,
780 vixl::aarch64::Label* adrp_label = nullptr);
781
782 // Add a new boot image method patch for an instruction and return the label
783 // to be bound before the instruction. The instruction will be either the
784 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
785 // to the associated ADRP patch label).
786 vixl::aarch64::Label* NewBootImageMethodPatch(MethodReference target_method,
787 vixl::aarch64::Label* adrp_label = nullptr);
788
789 // Add a new .bss entry method patch for an instruction and return
790 // the label to be bound before the instruction. The instruction will be
791 // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label`
792 // pointing to the associated ADRP patch label).
793 vixl::aarch64::Label* NewMethodBssEntryPatch(MethodReference target_method,
794 vixl::aarch64::Label* adrp_label = nullptr);
795
796 // Add a new boot image type patch for an instruction and return the label
797 // to be bound before the instruction. The instruction will be either the
798 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
799 // to the associated ADRP patch label).
800 vixl::aarch64::Label* NewBootImageTypePatch(const DexFile& dex_file,
801 dex::TypeIndex type_index,
802 vixl::aarch64::Label* adrp_label = nullptr);
803
804 // Add a new .bss entry type patch for an instruction and return the label
805 // to be bound before the instruction. The instruction will be either the
806 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
807 // to the associated ADRP patch label).
808 vixl::aarch64::Label* NewBssEntryTypePatch(HLoadClass* load_class,
809 vixl::aarch64::Label* adrp_label = nullptr);
810
811 // Add a new boot image string patch for an instruction and return the label
812 // to be bound before the instruction. The instruction will be either the
813 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
814 // to the associated ADRP patch label).
815 vixl::aarch64::Label* NewBootImageStringPatch(const DexFile& dex_file,
816 dex::StringIndex string_index,
817 vixl::aarch64::Label* adrp_label = nullptr);
818
819 // Add a new .bss entry string patch for an instruction and return the label
820 // to be bound before the instruction. The instruction will be either the
821 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
822 // to the associated ADRP patch label).
823 vixl::aarch64::Label* NewStringBssEntryPatch(const DexFile& dex_file,
824 dex::StringIndex string_index,
825 vixl::aarch64::Label* adrp_label = nullptr);
826
827 // Add a new boot image JNI entrypoint patch for an instruction and return the label
828 // to be bound before the instruction. The instruction will be either the
829 // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing
830 // to the associated ADRP patch label).
831 vixl::aarch64::Label* NewBootImageJniEntrypointPatch(MethodReference target_method,
832 vixl::aarch64::Label* adrp_label = nullptr);
833
834 // Emit the BL instruction for entrypoint thunk call and record the associated patch for AOT.
835 void EmitEntrypointThunkCall(ThreadOffset64 entrypoint_offset);
836
837 // Emit the CBNZ instruction for baker read barrier and record
838 // the associated patch for AOT or slow path for JIT.
839 void EmitBakerReadBarrierCbnz(uint32_t custom_data);
840
841 vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address);
842 vixl::aarch64::Literal<uint32_t>* DeduplicateJitStringLiteral(const DexFile& dex_file,
843 dex::StringIndex string_index,
844 Handle<mirror::String> handle);
845 vixl::aarch64::Literal<uint32_t>* DeduplicateJitClassLiteral(const DexFile& dex_file,
846 dex::TypeIndex string_index,
847 Handle<mirror::Class> handle);
848
849 void EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg);
850 void EmitAddPlaceholder(vixl::aarch64::Label* fixup_label,
851 vixl::aarch64::Register out,
852 vixl::aarch64::Register base);
853 void EmitLdrOffsetPlaceholder(vixl::aarch64::Label* fixup_label,
854 vixl::aarch64::Register out,
855 vixl::aarch64::Register base);
856
857 void LoadBootImageRelRoEntry(vixl::aarch64::Register reg, uint32_t boot_image_offset);
858 void LoadBootImageAddress(vixl::aarch64::Register reg, uint32_t boot_image_reference);
859 void LoadTypeForBootImageIntrinsic(vixl::aarch64::Register reg, TypeReference type_reference);
860 void LoadIntrinsicDeclaringClass(vixl::aarch64::Register reg, HInvoke* invoke);
861 void LoadClassRootForIntrinsic(vixl::aarch64::Register reg, ClassRoot class_root);
862
863 void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) override;
864 bool NeedsThunkCode(const linker::LinkerPatch& patch) const override;
865 void EmitThunkCode(const linker::LinkerPatch& patch,
866 /*out*/ ArenaVector<uint8_t>* code,
867 /*out*/ std::string* debug_name) override;
868
869 void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) override;
870
871 // Generate a GC root reference load:
872 //
873 // root <- *(obj + offset)
874 //
875 // while honoring read barriers based on read_barrier_option.
876 void GenerateGcRootFieldLoad(HInstruction* instruction,
877 Location root,
878 vixl::aarch64::Register obj,
879 uint32_t offset,
880 vixl::aarch64::Label* fixup_label,
881 ReadBarrierOption read_barrier_option);
882 // Generate MOV for the `old_value` in intrinsic CAS and mark it with Baker read barrier.
883 void GenerateIntrinsicCasMoveWithBakerReadBarrier(vixl::aarch64::Register marked_old_value,
884 vixl::aarch64::Register old_value);
885 // Fast path implementation of ReadBarrier::Barrier for a heap
886 // reference field load when Baker's read barriers are used.
887 // Overload suitable for Unsafe.getObject/-Volatile() intrinsic.
888 void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
889 Location ref,
890 vixl::aarch64::Register obj,
891 const vixl::aarch64::MemOperand& src,
892 bool needs_null_check,
893 bool use_load_acquire);
894 // Fast path implementation of ReadBarrier::Barrier for a heap
895 // reference field load when Baker's read barriers are used.
896 void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
897 Location ref,
898 vixl::aarch64::Register obj,
899 uint32_t offset,
900 Location maybe_temp,
901 bool needs_null_check,
902 bool use_load_acquire);
903 // Fast path implementation of ReadBarrier::Barrier for a heap
904 // reference array load when Baker's read barriers are used.
905 void GenerateArrayLoadWithBakerReadBarrier(HArrayGet* instruction,
906 Location ref,
907 vixl::aarch64::Register obj,
908 uint32_t data_offset,
909 Location index,
910 bool needs_null_check);
911
912 // Emit code checking the status of the Marking Register, and
913 // aborting the program if MR does not match the value stored in the
914 // art::Thread object. Code is only emitted in debug mode and if
915 // CompilerOptions::EmitRunTimeChecksInDebugMode returns true.
916 //
917 // Argument `code` is used to identify the different occurrences of
918 // MaybeGenerateMarkingRegisterCheck in the code generator, and is
919 // passed to the BRK instruction.
920 //
921 // If `temp_loc` is a valid location, it is expected to be a
922 // register and will be used as a temporary to generate code;
923 // otherwise, a temporary will be fetched from the core register
924 // scratch pool.
925 virtual void MaybeGenerateMarkingRegisterCheck(int code,
926 Location temp_loc = Location::NoLocation());
927
928 // Create slow path for a read barrier for a heap reference within `instruction`.
929 //
930 // This is a helper function for GenerateReadBarrierSlow() that has the same
931 // arguments. The creation and adding of the slow path is exposed for intrinsics
932 // that cannot use GenerateReadBarrierSlow() from their own slow paths.
933 SlowPathCodeARM64* AddReadBarrierSlowPath(HInstruction* instruction,
934 Location out,
935 Location ref,
936 Location obj,
937 uint32_t offset,
938 Location index);
939
940 // Generate a read barrier for a heap reference within `instruction`
941 // using a slow path.
942 //
943 // A read barrier for an object reference read from the heap is
944 // implemented as a call to the artReadBarrierSlow runtime entry
945 // point, which is passed the values in locations `ref`, `obj`, and
946 // `offset`:
947 //
948 // mirror::Object* artReadBarrierSlow(mirror::Object* ref,
949 // mirror::Object* obj,
950 // uint32_t offset);
951 //
952 // The `out` location contains the value returned by
953 // artReadBarrierSlow.
954 //
955 // When `index` is provided (i.e. for array accesses), the offset
956 // value passed to artReadBarrierSlow is adjusted to take `index`
957 // into account.
958 void GenerateReadBarrierSlow(HInstruction* instruction,
959 Location out,
960 Location ref,
961 Location obj,
962 uint32_t offset,
963 Location index = Location::NoLocation());
964
965 // If read barriers are enabled, generate a read barrier for a heap
966 // reference using a slow path. If heap poisoning is enabled, also
967 // unpoison the reference in `out`.
968 void MaybeGenerateReadBarrierSlow(HInstruction* instruction,
969 Location out,
970 Location ref,
971 Location obj,
972 uint32_t offset,
973 Location index = Location::NoLocation());
974
975 // Generate a read barrier for a GC root within `instruction` using
976 // a slow path.
977 //
978 // A read barrier for an object reference GC root is implemented as
979 // a call to the artReadBarrierForRootSlow runtime entry point,
980 // which is passed the value in location `root`:
981 //
982 // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root);
983 //
984 // The `out` location contains the value returned by
985 // artReadBarrierForRootSlow.
986 void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root);
987
988 void IncreaseFrame(size_t adjustment) override;
989 void DecreaseFrame(size_t adjustment) override;
990
991 void GenerateNop() override;
992
993 void GenerateImplicitNullCheck(HNullCheck* instruction) override;
994 void GenerateExplicitNullCheck(HNullCheck* instruction) override;
995
MaybeRecordImplicitNullCheck(HInstruction * instr)996 void MaybeRecordImplicitNullCheck(HInstruction* instr) final {
997 // The function must be only called within special scopes
998 // (EmissionCheckScope, ExactAssemblyScope) which prevent generation of
999 // veneer/literal pools by VIXL assembler.
1000 CHECK_EQ(GetVIXLAssembler()->ArePoolsBlocked(), true)
1001 << "The function must only be called within EmissionCheckScope or ExactAssemblyScope";
1002 CodeGenerator::MaybeRecordImplicitNullCheck(instr);
1003 }
1004
1005 void MaybeGenerateInlineCacheCheck(HInstruction* instruction, vixl::aarch64::Register klass);
1006 void MaybeIncrementHotness(bool is_frame_entry);
1007
1008 bool CanUseImplicitSuspendCheck() const;
1009
1010 private:
1011 // Encoding of thunk type and data for link-time generated thunks for Baker read barriers.
1012
1013 enum class BakerReadBarrierKind : uint8_t {
1014 kField, // Field get or array get with constant offset (i.e. constant index).
1015 kAcquire, // Volatile field get.
1016 kArray, // Array get with index in register.
1017 kGcRoot, // GC root load.
1018 kLast = kGcRoot
1019 };
1020
1021 static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* sp/zr is invalid */ 31u;
1022
1023 static constexpr size_t kBitsForBakerReadBarrierKind =
1024 MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
1025 static constexpr size_t kBakerReadBarrierBitsForRegister =
1026 MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg);
1027 using BakerReadBarrierKindField =
1028 BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
1029 using BakerReadBarrierFirstRegField =
1030 BitField<uint32_t, kBitsForBakerReadBarrierKind, kBakerReadBarrierBitsForRegister>;
1031 using BakerReadBarrierSecondRegField =
1032 BitField<uint32_t,
1033 kBitsForBakerReadBarrierKind + kBakerReadBarrierBitsForRegister,
1034 kBakerReadBarrierBitsForRegister>;
1035
CheckValidReg(uint32_t reg)1036 static void CheckValidReg(uint32_t reg) {
1037 DCHECK(reg < vixl::aarch64::lr.GetCode() &&
1038 reg != vixl::aarch64::ip0.GetCode() &&
1039 reg != vixl::aarch64::ip1.GetCode()) << reg;
1040 }
1041
EncodeBakerReadBarrierFieldData(uint32_t base_reg,uint32_t holder_reg)1042 static inline uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
1043 CheckValidReg(base_reg);
1044 CheckValidReg(holder_reg);
1045 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
1046 BakerReadBarrierFirstRegField::Encode(base_reg) |
1047 BakerReadBarrierSecondRegField::Encode(holder_reg);
1048 }
1049
EncodeBakerReadBarrierAcquireData(uint32_t base_reg,uint32_t holder_reg)1050 static inline uint32_t EncodeBakerReadBarrierAcquireData(uint32_t base_reg, uint32_t holder_reg) {
1051 CheckValidReg(base_reg);
1052 CheckValidReg(holder_reg);
1053 DCHECK_NE(base_reg, holder_reg);
1054 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kAcquire) |
1055 BakerReadBarrierFirstRegField::Encode(base_reg) |
1056 BakerReadBarrierSecondRegField::Encode(holder_reg);
1057 }
1058
EncodeBakerReadBarrierArrayData(uint32_t base_reg)1059 static inline uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
1060 CheckValidReg(base_reg);
1061 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
1062 BakerReadBarrierFirstRegField::Encode(base_reg) |
1063 BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg);
1064 }
1065
EncodeBakerReadBarrierGcRootData(uint32_t root_reg)1066 static inline uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
1067 CheckValidReg(root_reg);
1068 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
1069 BakerReadBarrierFirstRegField::Encode(root_reg) |
1070 BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg);
1071 }
1072
1073 void CompileBakerReadBarrierThunk(Arm64Assembler& assembler,
1074 uint32_t encoded_data,
1075 /*out*/ std::string* debug_name);
1076
1077 using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::aarch64::Literal<uint64_t>*>;
1078 using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::aarch64::Literal<uint32_t>*>;
1079 using StringToLiteralMap = ArenaSafeMap<StringReference,
1080 vixl::aarch64::Literal<uint32_t>*,
1081 StringReferenceValueComparator>;
1082 using TypeToLiteralMap = ArenaSafeMap<TypeReference,
1083 vixl::aarch64::Literal<uint32_t>*,
1084 TypeReferenceValueComparator>;
1085
1086 vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value);
1087 vixl::aarch64::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value);
1088
1089 // The PcRelativePatchInfo is used for PC-relative addressing of methods/strings/types,
1090 // whether through .data.bimg.rel.ro, .bss, or directly in the boot image.
1091 struct PcRelativePatchInfo : PatchInfo<vixl::aarch64::Label> {
PcRelativePatchInfoPcRelativePatchInfo1092 PcRelativePatchInfo(const DexFile* dex_file, uint32_t off_or_idx)
1093 : PatchInfo<vixl::aarch64::Label>(dex_file, off_or_idx), pc_insn_label() { }
1094
1095 vixl::aarch64::Label* pc_insn_label;
1096 };
1097
1098 struct BakerReadBarrierPatchInfo {
BakerReadBarrierPatchInfoBakerReadBarrierPatchInfo1099 explicit BakerReadBarrierPatchInfo(uint32_t data) : label(), custom_data(data) { }
1100
1101 vixl::aarch64::Label label;
1102 uint32_t custom_data;
1103 };
1104
1105 vixl::aarch64::Label* NewPcRelativePatch(const DexFile* dex_file,
1106 uint32_t offset_or_index,
1107 vixl::aarch64::Label* adrp_label,
1108 ArenaDeque<PcRelativePatchInfo>* patches);
1109
1110 void EmitJumpTables();
1111
1112 template <linker::LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)>
1113 static void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos,
1114 ArenaVector<linker::LinkerPatch>* linker_patches);
1115
1116 // Returns whether SVE features are supported and should be used.
1117 bool ShouldUseSVE() const;
1118
1119 // Labels for each block that will be compiled.
1120 // We use a deque so that the `vixl::aarch64::Label` objects do not move in memory.
1121 ArenaDeque<vixl::aarch64::Label> block_labels_; // Indexed by block id.
1122 vixl::aarch64::Label frame_entry_label_;
1123 ArenaVector<std::unique_ptr<JumpTableARM64>> jump_tables_;
1124
1125 LocationsBuilderARM64Neon location_builder_neon_;
1126 InstructionCodeGeneratorARM64Neon instruction_visitor_neon_;
1127 LocationsBuilderARM64Sve location_builder_sve_;
1128 InstructionCodeGeneratorARM64Sve instruction_visitor_sve_;
1129
1130 LocationsBuilderARM64* location_builder_;
1131 InstructionCodeGeneratorARM64* instruction_visitor_;
1132 ParallelMoveResolverARM64 move_resolver_;
1133 Arm64Assembler assembler_;
1134
1135 // PC-relative method patch info for kBootImageLinkTimePcRelative.
1136 ArenaDeque<PcRelativePatchInfo> boot_image_method_patches_;
1137 // PC-relative method patch info for kBssEntry.
1138 ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_;
1139 // PC-relative type patch info for kBootImageLinkTimePcRelative.
1140 ArenaDeque<PcRelativePatchInfo> boot_image_type_patches_;
1141 // PC-relative type patch info for kBssEntry.
1142 ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
1143 // PC-relative public type patch info for kBssEntryPublic.
1144 ArenaDeque<PcRelativePatchInfo> public_type_bss_entry_patches_;
1145 // PC-relative package type patch info for kBssEntryPackage.
1146 ArenaDeque<PcRelativePatchInfo> package_type_bss_entry_patches_;
1147 // PC-relative String patch info for kBootImageLinkTimePcRelative.
1148 ArenaDeque<PcRelativePatchInfo> boot_image_string_patches_;
1149 // PC-relative String patch info for kBssEntry.
1150 ArenaDeque<PcRelativePatchInfo> string_bss_entry_patches_;
1151 // PC-relative method patch info for kBootImageLinkTimePcRelative+kCallCriticalNative.
1152 ArenaDeque<PcRelativePatchInfo> boot_image_jni_entrypoint_patches_;
1153 // PC-relative patch info for IntrinsicObjects for the boot image,
1154 // and for method/type/string patches for kBootImageRelRo otherwise.
1155 ArenaDeque<PcRelativePatchInfo> boot_image_other_patches_;
1156 // Patch info for calls to entrypoint dispatch thunks. Used for slow paths.
1157 ArenaDeque<PatchInfo<vixl::aarch64::Label>> call_entrypoint_patches_;
1158 // Baker read barrier patch info.
1159 ArenaDeque<BakerReadBarrierPatchInfo> baker_read_barrier_patches_;
1160
1161 // Deduplication map for 32-bit literals, used for JIT for boot image addresses.
1162 Uint32ToLiteralMap uint32_literals_;
1163 // Deduplication map for 64-bit literals, used for JIT for method address or method code.
1164 Uint64ToLiteralMap uint64_literals_;
1165 // Patches for string literals in JIT compiled code.
1166 StringToLiteralMap jit_string_patches_;
1167 // Patches for class literals in JIT compiled code.
1168 TypeToLiteralMap jit_class_patches_;
1169
1170 // Baker read barrier slow paths, mapping custom data (uint32_t) to label.
1171 // Wrap the label to work around vixl::aarch64::Label being non-copyable
1172 // and non-moveable and as such unusable in ArenaSafeMap<>.
1173 struct LabelWrapper {
LabelWrapperLabelWrapper1174 LabelWrapper(const LabelWrapper& src)
1175 : label() {
1176 DCHECK(!src.label.IsLinked() && !src.label.IsBound());
1177 }
1178 LabelWrapper() = default;
1179 vixl::aarch64::Label label;
1180 };
1181 ArenaSafeMap<uint32_t, LabelWrapper> jit_baker_read_barrier_slow_paths_;
1182
1183 friend class linker::Arm64RelativePatcherTest;
1184 DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64);
1185 };
1186
GetAssembler()1187 inline Arm64Assembler* ParallelMoveResolverARM64::GetAssembler() const {
1188 return codegen_->GetAssembler();
1189 }
1190
1191 } // namespace arm64
1192 } // namespace art
1193
1194 #endif // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
1195