1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "asm_support_arm64.S" 18#include "interpreter/cfi_asm_support.h" 19 20#include "arch/quick_alloc_entrypoints.S" 21#include "arch/quick_field_entrypoints.S" 22 23.macro SAVE_REG_INCREASE_FRAME reg, frame_adjustment 24 str \reg, [sp, #-(\frame_adjustment)]! 25 .cfi_adjust_cfa_offset (\frame_adjustment) 26 .cfi_rel_offset \reg, 0 27.endm 28 29.macro RESTORE_REG_DECREASE_FRAME reg, frame_adjustment 30 ldr \reg, [sp], #(\frame_adjustment) 31 .cfi_restore \reg 32 .cfi_adjust_cfa_offset -(\frame_adjustment) 33.endm 34 35.macro POP_SAVE_REFS_ONLY_FRAME 36 DECREASE_FRAME 96 37.endm 38 39 /* 40 * Macro that sets up the callee save frame to conform with 41 * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs). 42 * 43 * TODO This is probably too conservative - saving FP & LR. 44 */ 45.macro SETUP_SAVE_REFS_AND_ARGS_FRAME 46 // art::Runtime* xIP0 = art::Runtime::instance_; 47 // Our registers aren't intermixed - just spill in order. 48 LOAD_RUNTIME_INSTANCE xIP0 49 50 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveRefAndArgs]; 51 ldr xIP0, [xIP0, RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET] 52 53 INCREASE_FRAME FRAME_SIZE_SAVE_REFS_AND_ARGS 54 SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL sp 55 56 str xIP0, [sp] // Store ArtMethod* Runtime::callee_save_methods_[kSaveRefsAndArgs]. 57 // Place sp in Thread::Current()->top_quick_frame. 58 mov xIP0, sp 59 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 60.endm 61 62.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 63 INCREASE_FRAME FRAME_SIZE_SAVE_REFS_AND_ARGS 64 SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL sp 65 str x0, [sp, #0] // Store ArtMethod* to bottom of stack. 66 // Place sp in Thread::Current()->top_quick_frame. 67 mov xIP0, sp 68 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 69.endm 70 71 /* 72 * Macro that sets up the callee save frame to conform with 73 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 74 * when the SP has already been decremented by FRAME_SIZE_SAVE_EVERYTHING 75 * and saving registers x29 and LR is handled elsewhere. 76 */ 77.macro SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR \ 78 runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 79 // Ugly compile-time check, but we only have the preprocessor. 80#if (FRAME_SIZE_SAVE_EVERYTHING != 512) 81#error "FRAME_SIZE_SAVE_EVERYTHING(ARM64) size not as expected." 82#endif 83 84 // Save FP registers. 85 stp d0, d1, [sp, #16] 86 stp d2, d3, [sp, #32] 87 stp d4, d5, [sp, #48] 88 stp d6, d7, [sp, #64] 89 stp d8, d9, [sp, #80] 90 stp d10, d11, [sp, #96] 91 stp d12, d13, [sp, #112] 92 stp d14, d15, [sp, #128] 93 stp d16, d17, [sp, #144] 94 stp d18, d19, [sp, #160] 95 stp d20, d21, [sp, #176] 96 stp d22, d23, [sp, #192] 97 stp d24, d25, [sp, #208] 98 stp d26, d27, [sp, #224] 99 stp d28, d29, [sp, #240] 100 stp d30, d31, [sp, #256] 101 102 // Save core registers. 103 SAVE_TWO_REGS x0, x1, 272 104 SAVE_TWO_REGS x2, x3, 288 105 SAVE_TWO_REGS x4, x5, 304 106 SAVE_TWO_REGS x6, x7, 320 107 SAVE_TWO_REGS x8, x9, 336 108 SAVE_TWO_REGS x10, x11, 352 109 SAVE_TWO_REGS x12, x13, 368 110 SAVE_TWO_REGS x14, x15, 384 111 SAVE_TWO_REGS x16, x17, 400 // Do not save the platform register. 112 SAVE_TWO_REGS x19, x20, 416 113 SAVE_TWO_REGS x21, x22, 432 114 SAVE_TWO_REGS x23, x24, 448 115 SAVE_TWO_REGS x25, x26, 464 116 SAVE_TWO_REGS x27, x28, 480 117 118 // art::Runtime* xIP0 = art::Runtime::instance_; 119 LOAD_RUNTIME_INSTANCE xIP0 120 121 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveEverything]; 122 ldr xIP0, [xIP0, \runtime_method_offset] 123 124 // Store ArtMethod* Runtime::callee_save_methods_[kSaveEverything]. 125 str xIP0, [sp] 126 // Place sp in Thread::Current()->top_quick_frame. 127 mov xIP0, sp 128 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 129.endm 130 131 /* 132 * Macro that sets up the callee save frame to conform with 133 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 134 */ 135.macro SETUP_SAVE_EVERYTHING_FRAME runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 136 INCREASE_FRAME 512 137 SAVE_TWO_REGS x29, xLR, 496 138 SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR \runtime_method_offset 139.endm 140 141.macro RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 142 // Restore FP registers. 143 ldp d0, d1, [sp, #16] 144 ldp d2, d3, [sp, #32] 145 ldp d4, d5, [sp, #48] 146 ldp d6, d7, [sp, #64] 147 ldp d8, d9, [sp, #80] 148 ldp d10, d11, [sp, #96] 149 ldp d12, d13, [sp, #112] 150 ldp d14, d15, [sp, #128] 151 ldp d16, d17, [sp, #144] 152 ldp d18, d19, [sp, #160] 153 ldp d20, d21, [sp, #176] 154 ldp d22, d23, [sp, #192] 155 ldp d24, d25, [sp, #208] 156 ldp d26, d27, [sp, #224] 157 ldp d28, d29, [sp, #240] 158 ldp d30, d31, [sp, #256] 159 160 // Restore core registers, except x0. 161 RESTORE_REG x1, 280 162 RESTORE_TWO_REGS x2, x3, 288 163 RESTORE_TWO_REGS x4, x5, 304 164 RESTORE_TWO_REGS x6, x7, 320 165 RESTORE_TWO_REGS x8, x9, 336 166 RESTORE_TWO_REGS x10, x11, 352 167 RESTORE_TWO_REGS x12, x13, 368 168 RESTORE_TWO_REGS x14, x15, 384 169 RESTORE_TWO_REGS x16, x17, 400 // Do not restore the platform register. 170 RESTORE_TWO_REGS x19, x20, 416 171 RESTORE_TWO_REGS x21, x22, 432 172 RESTORE_TWO_REGS x23, x24, 448 173 RESTORE_TWO_REGS x25, x26, 464 174 RESTORE_TWO_REGS x27, x28, 480 175 RESTORE_TWO_REGS x29, xLR, 496 176 177 DECREASE_FRAME 512 178.endm 179 180.macro RESTORE_SAVE_EVERYTHING_FRAME 181 RESTORE_REG x0, 272 182 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 183.endm 184 185.macro RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION is_ref = 0 186 ldr x1, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 187 CFI_REMEMBER_STATE 188 cbnz x1, 1f 189 DEOPT_OR_RETURN x1, \is_ref // Check if deopt is required 1901: // deliver exception on current thread 191 CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 192 DELIVER_PENDING_EXCEPTION 193.endm 194 195.macro RETURN_REF_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 196 RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION /* is_ref= */ 1 197.endm 198 199.macro DEOPT_OR_RETURN temp, is_ref = 0 200 ldr \temp, [xSELF, #THREAD_DEOPT_CHECK_REQUIRED_OFFSET] 201 cbnz \temp, 2f 202 ret 2032: 204 SETUP_SAVE_EVERYTHING_FRAME 205 mov x2, \is_ref // pass if result is a reference 206 mov x1, x0 // pass the result 207 mov x0, xSELF // Thread::Current 208 CALL_SYMBOL artDeoptimizeIfNeeded 209 210 CFI_REMEMBER_STATE 211 cbnz x0, 3f 212 213 RESTORE_SAVE_EVERYTHING_FRAME 214 REFRESH_MARKING_REGISTER 215 ret 216 2173: 218 // Deoptimize. 219 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 220 CALL_SYMBOL art_quick_do_long_jump // (Context*) 221 brk 0 // Unreached 222.endm 223 224.macro DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_X0 temp, is_ref 225 ldr \temp, [xSELF, #THREAD_DEOPT_CHECK_REQUIRED_OFFSET] 226 CFI_REMEMBER_STATE 227 cbnz \temp, 2f 228 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_X0 229 REFRESH_MARKING_REGISTER 230 ret 2312: 232 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 233 str x0, [sp, #SAVE_EVERYTHING_FRAME_X0_OFFSET] // update result in the frame 234 mov x2, \is_ref // pass if result is a reference 235 mov x1, x0 // pass the result 236 mov x0, xSELF // Thread::Current 237 CALL_SYMBOL artDeoptimizeIfNeeded 238 239 CFI_REMEMBER_STATE 240 cbnz x0, 3f 241 242 RESTORE_SAVE_EVERYTHING_FRAME 243 REFRESH_MARKING_REGISTER 244 ret 245 2463: 247 // Deoptimize. 248 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 249 CALL_SYMBOL art_quick_do_long_jump // (Context*) 250 brk 0 // Unreached 251.endm 252 253 254.macro RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER 255 CFI_REMEMBER_STATE 256 cbnz w0, 1f // result non-zero branch over 257 DEOPT_OR_RETURN x1 2581: 259 CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 260 DELIVER_PENDING_EXCEPTION 261.endm 262 263.macro NO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 264 .extern \cxx_name 265ENTRY \c_name 266 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 267 mov x0, xSELF // pass Thread::Current 268 CALL_SYMBOL \cxx_name // \cxx_name(Thread*) 269 CALL_SYMBOL art_quick_do_long_jump // (Context*) 270 brk 0 // Unreached 271END \c_name 272.endm 273 274.macro NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name 275 .extern \cxx_name 276ENTRY \c_name 277 SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context 278 mov x0, xSELF // pass Thread::Current 279 CALL_SYMBOL \cxx_name // \cxx_name(Thread*) 280 CALL_SYMBOL art_quick_do_long_jump // (Context*) 281 brk 0 // Unreached 282END \c_name 283.endm 284 285.macro ONE_ARG_RUNTIME_EXCEPTION c_name, cxx_name 286 .extern \cxx_name 287ENTRY \c_name 288 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context. 289 mov x1, xSELF // pass Thread::Current. 290 CALL_SYMBOL \cxx_name // \cxx_name(arg, Thread*). 291 CALL_SYMBOL art_quick_do_long_jump // (Context*) 292 brk 0 // Unreached 293END \c_name 294.endm 295 296.macro TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name 297 .extern \cxx_name 298ENTRY \c_name 299 SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context 300 mov x2, xSELF // pass Thread::Current 301 CALL_SYMBOL \cxx_name // \cxx_name(arg1, arg2, Thread*) 302 CALL_SYMBOL art_quick_do_long_jump // (Context*) 303 brk 0 // Unreached 304END \c_name 305.endm 306 307 /* 308 * Called by managed code, saves callee saves and then calls artThrowException 309 * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception. 310 */ 311ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode 312 313 /* 314 * Called by managed code to create and deliver a NullPointerException. 315 */ 316NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING \ 317 art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode 318 319 /* 320 * Call installed by a signal handler to create and deliver a NullPointerException. 321 */ 322 .extern art_quick_throw_null_pointer_exception_from_signal 323ENTRY art_quick_throw_null_pointer_exception_from_signal 324 // The fault handler pushes the gc map address, i.e. "return address", to stack 325 // and passes the fault address in LR. So we need to set up the CFI info accordingly. 326 .cfi_def_cfa_offset __SIZEOF_POINTER__ 327 .cfi_rel_offset lr, 0 328 // Save all registers as basis for long jump context. 329 INCREASE_FRAME (FRAME_SIZE_SAVE_EVERYTHING - __SIZEOF_POINTER__) 330 SAVE_REG x29, (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__) // LR already saved. 331 SETUP_SAVE_EVERYTHING_FRAME_DECREMENTED_SP_SKIP_X29_LR 332 mov x0, lr // pass the fault address stored in LR by the fault handler. 333 mov x1, xSELF // pass Thread::Current. 334 CALL_SYMBOL artThrowNullPointerExceptionFromSignal // (arg, Thread*). 335 CALL_SYMBOL art_quick_do_long_jump // (Context*) 336 brk 0 // Unreached 337END art_quick_throw_null_pointer_exception_from_signal 338 339 /* 340 * Called by managed code to create and deliver an ArithmeticException. 341 */ 342NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_div_zero, artThrowDivZeroFromCode 343 344 /* 345 * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds 346 * index, arg2 holds limit. 347 */ 348TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_array_bounds, artThrowArrayBoundsFromCode 349 350 /* 351 * Called by managed code to create and deliver a StringIndexOutOfBoundsException 352 * as if thrown from a call to String.charAt(). Arg1 holds index, arg2 holds limit. 353 */ 354TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING \ 355 art_quick_throw_string_bounds, artThrowStringBoundsFromCode 356 357 /* 358 * Called by managed code to create and deliver a StackOverflowError. 359 */ 360NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode 361 362 /* 363 * All generated callsites for interface invokes and invocation slow paths will load arguments 364 * as usual - except instead of loading arg0/x0 with the target Method*, arg0/x0 will contain 365 * the method_idx. This wrapper will save arg1-arg3, and call the appropriate C helper. 366 * NOTE: "this" is first visible argument of the target, and so can be found in arg1/x1. 367 * 368 * The helper will attempt to locate the target and return a 128-bit result in x0/x1 consisting 369 * of the target Method* in x0 and method->code_ in x1. 370 * 371 * If unsuccessful, the helper will return null/????. There will be a pending exception in the 372 * thread and we branch to another stub to deliver it. 373 * 374 * On success this wrapper will restore arguments and *jump* to the target, leaving the lr 375 * pointing back to the original caller. 376 * 377 * Adapted from ARM32 code. 378 * 379 * Clobbers xIP0. 380 */ 381.macro INVOKE_TRAMPOLINE_BODY cxx_name 382 .extern \cxx_name 383 SETUP_SAVE_REFS_AND_ARGS_FRAME // save callee saves in case allocation triggers GC 384 // Helper signature is always 385 // (method_idx, *this_object, *caller_method, *self, sp) 386 387 mov x2, xSELF // pass Thread::Current 388 mov x3, sp 389 CALL_SYMBOL \cxx_name // (method_idx, this, Thread*, SP) 390 mov xIP0, x1 // save Method*->code_ 391 RESTORE_SAVE_REFS_AND_ARGS_FRAME 392 REFRESH_MARKING_REGISTER 393 cbz x0, 1f // did we find the target? if not go to exception delivery 394 br xIP0 // tail call to target 3951: 396 DELIVER_PENDING_EXCEPTION 397.endm 398.macro INVOKE_TRAMPOLINE c_name, cxx_name 399ENTRY \c_name 400 INVOKE_TRAMPOLINE_BODY \cxx_name 401END \c_name 402.endm 403 404INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, \ 405 artInvokeInterfaceTrampolineWithAccessCheck 406 407INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, \ 408 artInvokeStaticTrampolineWithAccessCheck 409INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, \ 410 artInvokeDirectTrampolineWithAccessCheck 411INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, \ 412 artInvokeSuperTrampolineWithAccessCheck 413INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, \ 414 artInvokeVirtualTrampolineWithAccessCheck 415 416 417.macro INVOKE_STUB_CREATE_FRAME 418SAVE_SIZE=8*8 // x4, x5, <padding>, x19, x20, x21, FP, LR saved. 419 SAVE_TWO_REGS_INCREASE_FRAME x4, x5, SAVE_SIZE 420 SAVE_REG x19, 24 421 SAVE_TWO_REGS x20, x21, 32 422 SAVE_TWO_REGS xFP, xLR, 48 423 424 mov xFP, sp // Use xFP for frame pointer, as it's callee-saved. 425 .cfi_def_cfa_register xFP 426 427 add x10, x2, #(__SIZEOF_POINTER__ + 0xf) // Reserve space for ArtMethod*, arguments and 428 and x10, x10, # ~0xf // round up for 16-byte stack alignment. 429 sub sp, sp, x10 // Adjust SP for ArtMethod*, args and alignment padding. 430 431 mov xSELF, x3 // Move thread pointer into SELF register. 432 433 // Copy arguments into stack frame. 434 // Use simple copy routine for now. 435 // 4 bytes per slot. 436 // X1 - source address 437 // W2 - args length 438 // X9 - destination address. 439 // W10 - temporary 440 add x9, sp, #8 // Destination address is bottom of stack + null. 441 442 // Copy parameters into the stack. Use numeric label as this is a macro and Clang's assembler 443 // does not have unique-id variables. 444 cbz w2, 2f 4451: 446 sub w2, w2, #4 // Need 65536 bytes of range. 447 ldr w10, [x1, x2] 448 str w10, [x9, x2] 449 cbnz w2, 1b 450 4512: 452 // Store null into ArtMethod* at bottom of frame. 453 str xzr, [sp] 454.endm 455 456.macro INVOKE_STUB_CALL_AND_RETURN 457 458 REFRESH_MARKING_REGISTER 459 REFRESH_SUSPEND_CHECK_REGISTER 460 461 // load method-> METHOD_QUICK_CODE_OFFSET 462 ldr x9, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 463 // Branch to method. 464 blr x9 465 466 // Pop the ArtMethod* (null), arguments and alignment padding from the stack. 467 mov sp, xFP 468 .cfi_def_cfa_register sp 469 470 // Restore saved registers including value address and shorty address. 471 RESTORE_REG x19, 24 472 RESTORE_TWO_REGS x20, x21, 32 473 RESTORE_TWO_REGS xFP, xLR, 48 474 RESTORE_TWO_REGS_DECREASE_FRAME x4, x5, SAVE_SIZE 475 476 // Store result (w0/x0/s0/d0) appropriately, depending on resultType. 477 ldrb w10, [x5] 478 479 // Check the return type and store the correct register into the jvalue in memory. 480 // Use numeric label as this is a macro and Clang's assembler does not have unique-id variables. 481 482 // Don't set anything for a void type. 483 cmp w10, #'V' 484 beq 1f 485 486 // Is it a double? 487 cmp w10, #'D' 488 beq 2f 489 490 // Is it a float? 491 cmp w10, #'F' 492 beq 3f 493 494 // Just store x0. Doesn't matter if it is 64 or 32 bits. 495 str x0, [x4] 496 4971: // Finish up. 498 ret 499 5002: // Store double. 501 str d0, [x4] 502 ret 503 5043: // Store float. 505 str s0, [x4] 506 ret 507 508.endm 509 510 511// Macro for loading an argument into a register. 512// label - the base name of the label of the load routine, 513// reg - the register to load, 514// args - pointer to current argument, incremented by size, 515// size - the size of the register - 4 or 8 bytes, 516// nh4_reg - the register to fill with the address of the next handler for 4-byte values, 517// nh4_l - the base name of the label of the next handler for 4-byte values, 518// nh8_reg - the register to fill with the address of the next handler for 8-byte values, 519// nh8_l - the base name of the label of the next handler for 8-byte values, 520// cont - the base name of the label for continuing the shorty processing loop, 521// suffix - suffix added to all labels to make labels unique for different users. 522.macro INVOKE_STUB_LOAD_REG label, reg, args, size, nh4_reg, nh4_l, nh8_reg, nh8_l, cont, suffix 523\label\suffix: 524 ldr \reg, [\args], #\size 525 adr \nh4_reg, \nh4_l\suffix 526 adr \nh8_reg, \nh8_l\suffix 527 b \cont\suffix 528.endm 529 530// Macro for skipping an argument that does not fit into argument registers. 531// label - the base name of the label of the skip routine, 532// args - pointer to current argument, incremented by size, 533// size - the size of the argument - 4 or 8 bytes, 534// cont - the base name of the label for continuing the shorty processing loop, 535// suffix - suffix added to all labels to make labels unique for different users. 536.macro INVOKE_STUB_SKIP_ARG label, args, size, cont, suffix 537\label\suffix: 538 add \args, \args, #\size 539 b \cont\suffix 540.endm 541 542// Fill registers x/w1 to x/w7 and s/d0 to s/d7 with parameters. 543// Parse the passed shorty to determine which register to load. 544// x5 - shorty, 545// x9 - points to arguments on the stack, 546// suffix - suffix added to all labels to make labels unique for different users. 547.macro INVOKE_STUB_LOAD_ALL_ARGS suffix 548 add x10, x5, #1 // Load shorty address, plus one to skip the return type. 549 550 // Load this (if instance method) and addresses for routines that load WXSD registers. 551 .ifc \suffix, _instance 552 ldr w1, [x9], #4 // Load "this" parameter, and increment arg pointer. 553 adr x11, .Lload_w2\suffix 554 adr x12, .Lload_x2\suffix 555 .else 556 adr x11, .Lload_w1\suffix 557 adr x12, .Lload_x1\suffix 558 .endif 559 adr x13, .Lload_s0\suffix 560 adr x14, .Lload_d0\suffix 561 562 // Loop to fill registers. 563.Lfill_regs\suffix: 564 ldrb w17, [x10], #1 // Load next character in signature, and increment. 565 cbz w17, .Lcall_method\suffix // Exit at end of signature. Shorty 0 terminated. 566 567 cmp w17, #'J' // Is this a long? 568 beq .Lload_long\suffix 569 570 cmp w17, #'F' // Is this a float? 571 beq .Lload_float\suffix 572 573 cmp w17, #'D' // Is this a double? 574 beq .Lload_double\suffix 575 576 // Everything else uses a 4-byte GPR. 577 br x11 578 579.Lload_long\suffix: 580 br x12 581 582.Lload_float\suffix: 583 br x13 584 585.Lload_double\suffix: 586 br x14 587 588// Handlers for loading other args (not float/double/long) into W registers. 589 .ifnc \suffix, _instance 590 INVOKE_STUB_LOAD_REG \ 591 .Lload_w1, w1, x9, 4, x11, .Lload_w2, x12, .Lload_x2, .Lfill_regs, \suffix 592 .endif 593 INVOKE_STUB_LOAD_REG .Lload_w2, w2, x9, 4, x11, .Lload_w3, x12, .Lload_x3, .Lfill_regs, \suffix 594 INVOKE_STUB_LOAD_REG .Lload_w3, w3, x9, 4, x11, .Lload_w4, x12, .Lload_x4, .Lfill_regs, \suffix 595 INVOKE_STUB_LOAD_REG .Lload_w4, w4, x9, 4, x11, .Lload_w5, x12, .Lload_x5, .Lfill_regs, \suffix 596 INVOKE_STUB_LOAD_REG .Lload_w5, w5, x9, 4, x11, .Lload_w6, x12, .Lload_x6, .Lfill_regs, \suffix 597 INVOKE_STUB_LOAD_REG .Lload_w6, w6, x9, 4, x11, .Lload_w7, x12, .Lload_x7, .Lfill_regs, \suffix 598 INVOKE_STUB_LOAD_REG .Lload_w7, w7, x9, 4, x11, .Lskip4, x12, .Lskip8, .Lfill_regs, \suffix 599 600// Handlers for loading longs into X registers. 601 .ifnc \suffix, _instance 602 INVOKE_STUB_LOAD_REG \ 603 .Lload_x1, x1, x9, 8, x11, .Lload_w2, x12, .Lload_x2, .Lfill_regs, \suffix 604 .endif 605 INVOKE_STUB_LOAD_REG .Lload_x2, x2, x9, 8, x11, .Lload_w3, x12, .Lload_x3, .Lfill_regs, \suffix 606 INVOKE_STUB_LOAD_REG .Lload_x3, x3, x9, 8, x11, .Lload_w4, x12, .Lload_x4, .Lfill_regs, \suffix 607 INVOKE_STUB_LOAD_REG .Lload_x4, x4, x9, 8, x11, .Lload_w5, x12, .Lload_x5, .Lfill_regs, \suffix 608 INVOKE_STUB_LOAD_REG .Lload_x5, x5, x9, 8, x11, .Lload_w6, x12, .Lload_x6, .Lfill_regs, \suffix 609 INVOKE_STUB_LOAD_REG .Lload_x6, x6, x9, 8, x11, .Lload_w7, x12, .Lload_x7, .Lfill_regs, \suffix 610 INVOKE_STUB_LOAD_REG .Lload_x7, x7, x9, 8, x11, .Lskip4, x12, .Lskip8, .Lfill_regs, \suffix 611 612// Handlers for loading singles into S registers. 613 INVOKE_STUB_LOAD_REG .Lload_s0, s0, x9, 4, x13, .Lload_s1, x14, .Lload_d1, .Lfill_regs, \suffix 614 INVOKE_STUB_LOAD_REG .Lload_s1, s1, x9, 4, x13, .Lload_s2, x14, .Lload_d2, .Lfill_regs, \suffix 615 INVOKE_STUB_LOAD_REG .Lload_s2, s2, x9, 4, x13, .Lload_s3, x14, .Lload_d3, .Lfill_regs, \suffix 616 INVOKE_STUB_LOAD_REG .Lload_s3, s3, x9, 4, x13, .Lload_s4, x14, .Lload_d4, .Lfill_regs, \suffix 617 INVOKE_STUB_LOAD_REG .Lload_s4, s4, x9, 4, x13, .Lload_s5, x14, .Lload_d5, .Lfill_regs, \suffix 618 INVOKE_STUB_LOAD_REG .Lload_s5, s5, x9, 4, x13, .Lload_s6, x14, .Lload_d6, .Lfill_regs, \suffix 619 INVOKE_STUB_LOAD_REG .Lload_s6, s6, x9, 4, x13, .Lload_s7, x14, .Lload_d7, .Lfill_regs, \suffix 620 INVOKE_STUB_LOAD_REG .Lload_s7, s7, x9, 4, x13, .Lskip4, x14, .Lskip8, .Lfill_regs, \suffix 621 622// Handlers for loading doubles into D registers. 623 INVOKE_STUB_LOAD_REG .Lload_d0, d0, x9, 8, x13, .Lload_s1, x14, .Lload_d1, .Lfill_regs, \suffix 624 INVOKE_STUB_LOAD_REG .Lload_d1, d1, x9, 8, x13, .Lload_s2, x14, .Lload_d2, .Lfill_regs, \suffix 625 INVOKE_STUB_LOAD_REG .Lload_d2, d2, x9, 8, x13, .Lload_s3, x14, .Lload_d3, .Lfill_regs, \suffix 626 INVOKE_STUB_LOAD_REG .Lload_d3, d3, x9, 8, x13, .Lload_s4, x14, .Lload_d4, .Lfill_regs, \suffix 627 INVOKE_STUB_LOAD_REG .Lload_d4, d4, x9, 8, x13, .Lload_s5, x14, .Lload_d5, .Lfill_regs, \suffix 628 INVOKE_STUB_LOAD_REG .Lload_d5, d5, x9, 8, x13, .Lload_s6, x14, .Lload_d6, .Lfill_regs, \suffix 629 INVOKE_STUB_LOAD_REG .Lload_d6, d6, x9, 8, x13, .Lload_s7, x14, .Lload_d7, .Lfill_regs, \suffix 630 INVOKE_STUB_LOAD_REG .Lload_d7, d7, x9, 8, x13, .Lskip4, x14, .Lskip8, .Lfill_regs, \suffix 631 632// Handlers for skipping arguments that do not fit into registers. 633 INVOKE_STUB_SKIP_ARG .Lskip4, x9, 4, .Lfill_regs, \suffix 634 INVOKE_STUB_SKIP_ARG .Lskip8, x9, 8, .Lfill_regs, \suffix 635 636.Lcall_method\suffix: 637.endm 638 639/* 640 * extern"C" void art_quick_invoke_stub(ArtMethod *method, x0 641 * uint32_t *args, x1 642 * uint32_t argsize, w2 643 * Thread *self, x3 644 * JValue *result, x4 645 * char *shorty); x5 646 * +----------------------+ 647 * | | 648 * | C/C++ frame | 649 * | LR'' | 650 * | FP'' | <- SP' 651 * +----------------------+ 652 * +----------------------+ 653 * | x28 | <- TODO: Remove callee-saves. 654 * | : | 655 * | x19 | 656 * | SP' | 657 * | X5 | 658 * | X4 | Saved registers 659 * | LR' | 660 * | FP' | <- FP 661 * +----------------------+ 662 * | uint32_t out[n-1] | 663 * | : : | Outs 664 * | uint32_t out[0] | 665 * | ArtMethod* | <- SP value=null 666 * +----------------------+ 667 * 668 * Outgoing registers: 669 * x0 - Method* 670 * x1-x7 - integer parameters. 671 * d0-d7 - Floating point parameters. 672 * xSELF = self 673 * SP = & of ArtMethod* 674 * x1 = "this" pointer. 675 * 676 */ 677ENTRY art_quick_invoke_stub 678 // Spill registers as per AACPS64 calling convention. 679 INVOKE_STUB_CREATE_FRAME 680 681 // Load args into registers. 682 INVOKE_STUB_LOAD_ALL_ARGS _instance 683 684 // Call the method and return. 685 INVOKE_STUB_CALL_AND_RETURN 686END art_quick_invoke_stub 687 688/* extern"C" 689 * void art_quick_invoke_static_stub(ArtMethod *method, x0 690 * uint32_t *args, x1 691 * uint32_t argsize, w2 692 * Thread *self, x3 693 * JValue *result, x4 694 * char *shorty); x5 695 */ 696ENTRY art_quick_invoke_static_stub 697 // Spill registers as per AACPS64 calling convention. 698 INVOKE_STUB_CREATE_FRAME 699 700 // Load args into registers. 701 INVOKE_STUB_LOAD_ALL_ARGS _static 702 703 // Call the method and return. 704 INVOKE_STUB_CALL_AND_RETURN 705END art_quick_invoke_static_stub 706 707 708 709/* extern"C" void art_quick_osr_stub(void** stack, x0 710 * size_t stack_size_in_bytes, x1 711 * const uint8_t* native_pc, x2 712 * JValue *result, x3 713 * char *shorty, x4 714 * Thread *self) x5 715 */ 716ENTRY art_quick_osr_stub 717 SAVE_SIZE=22*8 718 SAVE_TWO_REGS_INCREASE_FRAME x3, x4, SAVE_SIZE 719 SAVE_TWO_REGS x19, x20, 16 720 SAVE_TWO_REGS x21, x22, 32 721 SAVE_TWO_REGS x23, x24, 48 722 SAVE_TWO_REGS x25, x26, 64 723 SAVE_TWO_REGS x27, x28, 80 724 SAVE_TWO_REGS xFP, xLR, 96 725 stp d8, d9, [sp, #112] 726 stp d10, d11, [sp, #128] 727 stp d12, d13, [sp, #144] 728 stp d14, d15, [sp, #160] 729 730 mov xSELF, x5 // Move thread pointer into SELF register. 731 REFRESH_MARKING_REGISTER 732 REFRESH_SUSPEND_CHECK_REGISTER 733 734 INCREASE_FRAME 16 735 str xzr, [sp] // Store null for ArtMethod* slot 736 // Branch to stub. 737 CFI_REMEMBER_STATE 738 bl .Losr_entry 739 DECREASE_FRAME 16 740 741 // Restore saved registers including value address and shorty address. 742 ldp d8, d9, [sp, #112] 743 ldp d10, d11, [sp, #128] 744 ldp d12, d13, [sp, #144] 745 ldp d14, d15, [sp, #160] 746 RESTORE_TWO_REGS x19, x20, 16 747 RESTORE_TWO_REGS x21, x22, 32 748 RESTORE_TWO_REGS x23, x24, 48 749 RESTORE_TWO_REGS x25, x26, 64 750 RESTORE_TWO_REGS x27, x28, 80 751 RESTORE_TWO_REGS xFP, xLR, 96 752 RESTORE_TWO_REGS_DECREASE_FRAME x3, x4, SAVE_SIZE 753 754 // The compiler put the result in x0. Doesn't matter if it is 64 or 32 bits. 755 str x0, [x3] 756 ret 757 758.Losr_entry: 759 CFI_RESTORE_STATE_AND_DEF_CFA sp, (SAVE_SIZE+16) 760 761 mov x9, sp // Save stack pointer. 762 .cfi_def_cfa_register x9 763 764 // Update stack pointer for the callee 765 sub sp, sp, x1 766 767 // Update link register slot expected by the callee. 768 sub w1, w1, #8 769 str lr, [sp, x1] 770 771 // Copy arguments into stack frame. 772 // Use simple copy routine for now. 773 // 4 bytes per slot. 774 // X0 - source address 775 // W1 - args length 776 // SP - destination address. 777 // W10 - temporary 778.Losr_loop_entry: 779 cbz w1, .Losr_loop_exit 780 sub w1, w1, #4 781 ldr w10, [x0, x1] 782 str w10, [sp, x1] 783 b .Losr_loop_entry 784 785.Losr_loop_exit: 786 // Branch to the OSR entry point. 787 br x2 788 789END art_quick_osr_stub 790 791 /* 792 * On entry x0 is the long jump context. This is expected to be returned from a previous 793 * entrypoint call which threw an exception or deoptimized. 794 * IP0 and IP1 shall be clobbered rather than retrieved from gprs_. 795 */ 796 797ENTRY art_quick_do_long_jump 798 // Reserve space for the gprs + fprs; 799 INCREASE_FRAME ARM64_LONG_JUMP_CONTEXT_SIZE 800 801 mov x1, sp 802 add x2, sp, #ARM64_LONG_JUMP_GPRS_SIZE 803 804 CALL_SYMBOL artContextCopyForLongJump // Context* context, uintptr_t* gprs, uintptr_t* fprs 805 806 add x0, sp, #ARM64_LONG_JUMP_GPRS_SIZE 807 808 // Load FPRs 809 ldp d0, d1, [x0, #0] 810 ldp d2, d3, [x0, #16] 811 ldp d4, d5, [x0, #32] 812 ldp d6, d7, [x0, #48] 813 ldp d8, d9, [x0, #64] 814 ldp d10, d11, [x0, #80] 815 ldp d12, d13, [x0, #96] 816 ldp d14, d15, [x0, #112] 817 ldp d16, d17, [x0, #128] 818 ldp d18, d19, [x0, #144] 819 ldp d20, d21, [x0, #160] 820 ldp d22, d23, [x0, #176] 821 ldp d24, d25, [x0, #192] 822 ldp d26, d27, [x0, #208] 823 ldp d28, d29, [x0, #224] 824 ldp d30, d31, [x0, #240] 825 826 ldp x0, x1, [sp, #0] 827 ldp x2, x3, [sp, #16] 828 ldp x4, x5, [sp, #32] 829 ldp x6, x7, [sp, #48] 830 ldp x8, x9, [sp, #64] 831 ldp x10, x11, [sp, #80] 832 ldp x12, x13, [sp, #96] 833 ldp x14, x15, [sp, #112] 834 // Do not load IP0 (x16) and IP1 (x17), these shall be clobbered below. 835 // Don't load the platform register (x18) either. 836 ldr x19, [sp, #152] // xSELF. 837 ldp x20, x21, [sp, #160] // For Baker RB, wMR (w20) is reloaded below. 838 ldp x22, x23, [sp, #176] 839 ldp x24, x25, [sp, #192] 840 ldp x26, x27, [sp, #208] 841 ldp x28, x29, [sp, #224] 842 ldp x30, xIP0, [sp, #240] // LR and SP, load SP to IP0. 843 844 // Load PC to IP1, it's at the end (after the space for the unused XZR). 845 ldr xIP1, [sp, #33*8] 846 847 // Set SP. Do not access fprs_ and gprs_ from now, they are below SP. 848 mov sp, xIP0 849 .cfi_def_cfa_offset 0 850 851 REFRESH_MARKING_REGISTER 852 REFRESH_SUSPEND_CHECK_REGISTER 853 854 br xIP1 855END art_quick_do_long_jump 856 857 /* 858 * Entry from managed code that tries to lock the object in a fast path and 859 * calls `artLockObjectFromCode()` for the difficult cases, may block for GC. 860 * x0 holds the possibly null object to lock. 861 */ 862ENTRY art_quick_lock_object 863 LOCK_OBJECT_FAST_PATH x0, art_quick_lock_object_no_inline, /*can_be_null*/ 1 864END art_quick_lock_object 865 866 /* 867 * Entry from managed code that calls `artLockObjectFromCode()`, may block for GC. 868 * x0 holds the possibly null object to lock. 869 */ 870 .extern artLockObjectFromCode 871ENTRY art_quick_lock_object_no_inline 872 // This is also the slow path for art_quick_lock_object. 873 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case we block 874 mov x1, xSELF // pass Thread::Current 875 CALL_SYMBOL artLockObjectFromCode // (Object* obj, Thread*) 876 RESTORE_SAVE_REFS_ONLY_FRAME 877 REFRESH_MARKING_REGISTER 878 RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER 879END art_quick_lock_object_no_inline 880 881 /* 882 * Entry from managed code that tries to unlock the object in a fast path and calls 883 * `artUnlockObjectFromCode()` for the difficult cases and delivers exception on failure. 884 * x0 holds the possibly null object to unlock. 885 */ 886ENTRY art_quick_unlock_object 887 UNLOCK_OBJECT_FAST_PATH x0, art_quick_unlock_object_no_inline, /*can_be_null*/ 1 888END art_quick_unlock_object 889 890 /* 891 * Entry from managed code that calls `artUnlockObjectFromCode()` 892 * and delivers exception on failure. 893 * x0 holds the possibly null object to unlock. 894 */ 895 .extern artUnlockObjectFromCode 896ENTRY art_quick_unlock_object_no_inline 897 // This is also the slow path for art_quick_unlock_object. 898 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case exception allocation 899 // triggers GC 900 mov x1, xSELF // pass Thread::Current 901 CALL_SYMBOL artUnlockObjectFromCode // (Object* obj, Thread*) 902 RESTORE_SAVE_REFS_ONLY_FRAME 903 REFRESH_MARKING_REGISTER 904 RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER 905END art_quick_unlock_object_no_inline 906 907 /* 908 * Entry from managed code that calls artInstanceOfFromCode and on failure calls 909 * artThrowClassCastExceptionForObject. 910 */ 911 .extern artInstanceOfFromCode 912 .extern artThrowClassCastExceptionForObject 913ENTRY art_quick_check_instance_of 914 // Type check using the bit string passes null as the target class. In that case just throw. 915 cbz x1, .Lthrow_class_cast_exception_for_bitstring_check 916 917 // Store arguments and link register 918 // Stack needs to be 16B aligned on calls. 919 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 32 920 SAVE_REG xLR, 24 921 922 // Call runtime code 923 CALL_SYMBOL artInstanceOfFromCode 924 925 // Restore LR. 926 RESTORE_REG xLR, 24 927 928 // Check for exception 929 CFI_REMEMBER_STATE 930 cbz x0, .Lthrow_class_cast_exception 931 932 // Restore and return 933 // TODO: We do not need to restore X0 and X1 on success. We also do not need 934 // to record CFI for them as the information is not very useful. 935 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 936 ret 937 938.Lthrow_class_cast_exception: 939 CFI_RESTORE_STATE_AND_DEF_CFA sp, 32 940 // Restore 941 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 942 943.Lthrow_class_cast_exception_for_bitstring_check: 944 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 945 mov x2, xSELF // pass Thread::Current 946 CALL_SYMBOL artThrowClassCastExceptionForObject // (Object*, Class*, Thread*) 947 CALL_SYMBOL art_quick_do_long_jump // (Context*) 948 brk 0 // Unreached 949END art_quick_check_instance_of 950 951// Restore xReg's value from [sp, #offset] if xReg is not the same as xExclude. 952.macro POP_REG_NE xReg, offset, xExclude 953 .ifnc \xReg, \xExclude 954 ldr \xReg, [sp, #\offset] // restore xReg 955 .cfi_restore \xReg 956 .endif 957.endm 958 959// Restore xReg1's value from [sp, #offset] if xReg1 is not the same as xExclude. 960// Restore xReg2's value from [sp, #(offset + 8)] if xReg2 is not the same as xExclude. 961.macro POP_REGS_NE xReg1, xReg2, offset, xExclude 962 .ifc \xReg1, \xExclude 963 ldr \xReg2, [sp, #(\offset + 8)] // restore xReg2 964 .else 965 .ifc \xReg2, \xExclude 966 ldr \xReg1, [sp, #\offset] // restore xReg1 967 .else 968 ldp \xReg1, \xReg2, [sp, #\offset] // restore xReg1 and xReg2 969 .endif 970 .endif 971 .cfi_restore \xReg1 972 .cfi_restore \xReg2 973.endm 974 975 // Helper macros for `art_quick_aput_obj`. 976#ifdef USE_READ_BARRIER 977#ifdef USE_BAKER_READ_BARRIER 978.macro BAKER_RB_CHECK_GRAY_BIT_AND_LOAD wDest, xObj, offset, gray_slow_path_label 979 ldr wIP0, [\xObj, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 980 tbnz wIP0, #LOCK_WORD_READ_BARRIER_STATE_SHIFT, \gray_slow_path_label 981 // False dependency to avoid needing load/load fence. 982 add \xObj, \xObj, xIP0, lsr #32 983 ldr \wDest, [\xObj, #\offset] // Heap reference = 32b; zero-extends to xN. 984 UNPOISON_HEAP_REF \wDest 985.endm 986 987.macro BAKER_RB_LOAD_AND_MARK wDest, xObj, offset, mark_function 988 ldr \wDest, [\xObj, #\offset] // Heap reference = 32b; zero-extends to xN. 989 UNPOISON_HEAP_REF \wDest 990 // Save LR in a register preserved by `art_quick_read_barrier_mark_regNN` 991 // and unused by the `art_quick_aput_obj`. 992 mov x5, lr 993 CALL_SYMBOL \mark_function 994 mov lr, x5 // Restore LR. 995.endm 996#else // USE_BAKER_READ_BARRIER 997 .extern artReadBarrierSlow 998.macro READ_BARRIER_SLOW xDest, wDest, xObj, offset 999 // Store registers used in art_quick_aput_obj (x0-x4, LR), stack is 16B aligned. 1000 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 48 1001 SAVE_TWO_REGS x2, x3, 16 1002 SAVE_TWO_REGS x4, xLR, 32 1003 1004 // mov x0, \xRef // pass ref in x0 (no-op for now since parameter ref is unused) 1005 .ifnc \xObj, x1 1006 mov x1, \xObj // pass xObj 1007 .endif 1008 mov w2, #\offset // pass offset 1009 CALL_SYMBOL artReadBarrierSlow // artReadBarrierSlow(ref, xObj, offset) 1010 // No need to unpoison return value in w0, artReadBarrierSlow() would do the unpoisoning. 1011 .ifnc \wDest, w0 1012 mov \wDest, w0 // save return value in wDest 1013 .endif 1014 1015 // Conditionally restore saved registers 1016 POP_REG_NE x0, 0, \xDest 1017 POP_REG_NE x1, 8, \xDest 1018 POP_REG_NE x2, 16, \xDest 1019 POP_REG_NE x3, 24, \xDest 1020 POP_REG_NE x4, 32, \xDest 1021 RESTORE_REG xLR, 40 1022 DECREASE_FRAME 48 1023.endm 1024#endif // USE_BAKER_READ_BARRIER 1025#endif // USE_READ_BARRIER 1026 1027ENTRY art_quick_aput_obj 1028 cbz x2, .Laput_obj_null 1029#if defined(USE_READ_BARRIER) && !defined(USE_BAKER_READ_BARRIER) 1030 READ_BARRIER_SLOW x3, w3, x0, MIRROR_OBJECT_CLASS_OFFSET 1031 READ_BARRIER_SLOW x3, w3, x3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET 1032 READ_BARRIER_SLOW x4, w4, x2, MIRROR_OBJECT_CLASS_OFFSET 1033#else // !defined(USE_READ_BARRIER) || defined(USE_BAKER_READ_BARRIER) 1034#ifdef USE_READ_BARRIER 1035 cbnz wMR, .Laput_obj_gc_marking 1036#endif // USE_READ_BARRIER 1037 ldr w3, [x0, #MIRROR_OBJECT_CLASS_OFFSET] // Heap reference = 32b; zero-extends to x3. 1038 UNPOISON_HEAP_REF w3 1039 ldr w3, [x3, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] // Heap reference = 32b; zero-extends to x3. 1040 UNPOISON_HEAP_REF w3 1041 ldr w4, [x2, #MIRROR_OBJECT_CLASS_OFFSET] // Heap reference = 32b; zero-extends to x4. 1042 UNPOISON_HEAP_REF w4 1043#endif // !defined(USE_READ_BARRIER) || defined(USE_BAKER_READ_BARRIER) 1044 cmp w3, w4 // value's type == array's component type - trivial assignability 1045 bne .Laput_obj_check_assignability 1046.Laput_obj_store: 1047 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1048 POISON_HEAP_REF w2 1049 str w2, [x3, x1, lsl #2] // Heap reference = 32b. 1050 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1051 lsr x0, x0, #CARD_TABLE_CARD_SHIFT 1052 strb w3, [x3, x0] 1053 ret 1054 1055.Laput_obj_null: 1056 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1057 str w2, [x3, x1, lsl #2] // Heap reference = 32b. 1058 ret 1059 1060.Laput_obj_check_assignability: 1061 // Store arguments and link register 1062 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 32 1063 SAVE_TWO_REGS x2, xLR, 16 1064 1065 // Call runtime code 1066 mov x0, x3 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1067 mov x1, x4 // Heap reference, 32b, "uncompress" = do nothing, already zero-extended 1068 CALL_SYMBOL artIsAssignableFromCode 1069 1070 // Check for exception 1071 CFI_REMEMBER_STATE 1072 cbz x0, .Laput_obj_throw_array_store_exception 1073 1074 // Restore 1075 RESTORE_TWO_REGS x2, xLR, 16 1076 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1077 1078 add x3, x0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 1079 POISON_HEAP_REF w2 1080 str w2, [x3, x1, lsl #2] // Heap reference = 32b. 1081 ldr x3, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1082 lsr x0, x0, #CARD_TABLE_CARD_SHIFT 1083 strb w3, [x3, x0] 1084 ret 1085 1086.Laput_obj_throw_array_store_exception: 1087 CFI_RESTORE_STATE_AND_DEF_CFA sp, 32 1088 RESTORE_TWO_REGS x2, xLR, 16 1089 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 32 1090 1091#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 1092 CFI_REMEMBER_STATE 1093#endif // defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 1094 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 1095 mov x1, x2 // Pass value. 1096 mov x2, xSELF // Pass Thread::Current. 1097 CALL_SYMBOL artThrowArrayStoreException // (Object*, Object*, Thread*). 1098 CALL_SYMBOL art_quick_do_long_jump // (Context*) 1099 brk 0 // Unreached 1100 1101#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 1102 CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 1103.Laput_obj_gc_marking: 1104 BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ 1105 w3, x0, MIRROR_OBJECT_CLASS_OFFSET, .Laput_obj_mark_array_class 1106.Laput_obj_mark_array_class_continue: 1107 BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ 1108 w3, x3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, .Laput_obj_mark_array_element 1109.Laput_obj_mark_array_element_continue: 1110 BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ 1111 w4, x2, MIRROR_OBJECT_CLASS_OFFSET, .Laput_obj_mark_object_class 1112.Laput_obj_mark_object_class_continue: 1113 cmp w3, w4 // value's type == array's component type - trivial assignability 1114 bne .Laput_obj_check_assignability 1115 b .Laput_obj_store 1116 1117.Laput_obj_mark_array_class: 1118 BAKER_RB_LOAD_AND_MARK w3, x0, MIRROR_OBJECT_CLASS_OFFSET, art_quick_read_barrier_mark_reg03 1119 b .Laput_obj_mark_array_class_continue 1120 1121.Laput_obj_mark_array_element: 1122 BAKER_RB_LOAD_AND_MARK \ 1123 w3, x3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, art_quick_read_barrier_mark_reg03 1124 b .Laput_obj_mark_array_element_continue 1125 1126.Laput_obj_mark_object_class: 1127 BAKER_RB_LOAD_AND_MARK w4, x2, MIRROR_OBJECT_CLASS_OFFSET, art_quick_read_barrier_mark_reg04 1128 b .Laput_obj_mark_object_class_continue 1129#endif // defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 1130END art_quick_aput_obj 1131 1132// Macro to facilitate adding new allocation entrypoints. 1133.macro ONE_ARG_DOWNCALL name, entrypoint, return 1134 .extern \entrypoint 1135ENTRY \name 1136 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1137 mov x1, xSELF // pass Thread::Current 1138 CALL_SYMBOL \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1139 RESTORE_SAVE_REFS_ONLY_FRAME 1140 REFRESH_MARKING_REGISTER 1141 \return 1142END \name 1143.endm 1144 1145// Macro to facilitate adding new allocation entrypoints. 1146.macro TWO_ARG_DOWNCALL name, entrypoint, return 1147 .extern \entrypoint 1148ENTRY \name 1149 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1150 mov x2, xSELF // pass Thread::Current 1151 CALL_SYMBOL \entrypoint // (uint32_t type_idx, Method* method, Thread*) 1152 RESTORE_SAVE_REFS_ONLY_FRAME 1153 REFRESH_MARKING_REGISTER 1154 \return 1155END \name 1156.endm 1157 1158// Macro to facilitate adding new allocation entrypoints. 1159.macro THREE_ARG_DOWNCALL name, entrypoint, return 1160 .extern \entrypoint 1161ENTRY \name 1162 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1163 mov x3, xSELF // pass Thread::Current 1164 CALL_SYMBOL \entrypoint 1165 RESTORE_SAVE_REFS_ONLY_FRAME 1166 REFRESH_MARKING_REGISTER 1167 \return 1168END \name 1169.endm 1170 1171// Macro to facilitate adding new allocation entrypoints. 1172.macro FOUR_ARG_DOWNCALL name, entrypoint, return 1173 .extern \entrypoint 1174ENTRY \name 1175 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1176 mov x4, xSELF // pass Thread::Current 1177 CALL_SYMBOL \entrypoint // 1178 RESTORE_SAVE_REFS_ONLY_FRAME 1179 REFRESH_MARKING_REGISTER 1180 \return 1181END \name 1182.endm 1183 1184 /* 1185 * Macro for resolution and initialization of indexed DEX file 1186 * constants such as classes and strings. 1187 */ 1188.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL \ 1189 name, entrypoint, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 1190 .extern \entrypoint 1191ENTRY \name 1192 SETUP_SAVE_EVERYTHING_FRAME \runtime_method_offset // save everything for stack crawl 1193 mov x1, xSELF // pass Thread::Current 1194 CALL_SYMBOL \entrypoint // (int32_t index, Thread* self) 1195 cbz w0, 1f // If result is null, deliver the OOME. 1196 DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_X0 x1, /* is_ref= */ 1 11971: 1198 DELIVER_PENDING_EXCEPTION_FRAME_READY 1199END \name 1200.endm 1201 1202.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT name, entrypoint 1203 ONE_ARG_SAVE_EVERYTHING_DOWNCALL \ 1204 \name, \entrypoint, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET 1205.endm 1206 1207.macro RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER 1208 CFI_REMEMBER_STATE 1209 cbz w0, 1f // result zero branch over 1210 DEOPT_OR_RETURN x1, /*is_ref=*/1 // check for deopt or return 12111: 1212 CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 1213 DELIVER_PENDING_EXCEPTION 1214.endm 1215 1216 1217 /* 1218 * Entry from managed code that calls artHandleFillArrayDataFromCode and delivers exception on 1219 * failure. 1220 */ 1221TWO_ARG_DOWNCALL art_quick_handle_fill_data, \ 1222 artHandleFillArrayDataFromCode, \ 1223 RETURN_OR_DEOPT_IF_INT_RESULT_IS_ZERO_OR_DELIVER 1224 1225 /* 1226 * Entry from managed code when uninitialized static storage, this stub will run the class 1227 * initializer and deliver the exception on error. On success the static storage base is 1228 * returned. 1229 */ 1230ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT \ 1231 art_quick_initialize_static_storage, artInitializeStaticStorageFromCode 1232ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_resolve_type, artResolveTypeFromCode 1233ONE_ARG_SAVE_EVERYTHING_DOWNCALL \ 1234 art_quick_resolve_type_and_verify_access, artResolveTypeAndVerifyAccessFromCode 1235ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_handle, artResolveMethodHandleFromCode 1236ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_type, artResolveMethodTypeFromCode 1237ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode 1238 1239GENERATE_FIELD_ENTRYPOINTS 1240 1241// Generate the allocation entrypoints for each allocator. 1242GENERATE_ALLOC_ENTRYPOINTS_FOR_NON_TLAB_ALLOCATORS 1243// Comment out allocators that have arm64 specific asm. 1244// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) 1245// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB) 1246GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) 1247GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_region_tlab, RegionTLAB) 1248// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB) 1249// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_region_tlab, RegionTLAB) 1250// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_region_tlab, RegionTLAB) 1251// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_region_tlab, RegionTLAB) 1252// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_region_tlab, RegionTLAB) 1253GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_region_tlab, RegionTLAB) 1254GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_region_tlab, RegionTLAB) 1255GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_region_tlab, RegionTLAB) 1256 1257// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB) 1258// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB) 1259GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_tlab, TLAB) 1260GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_tlab, TLAB) 1261// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_tlab, TLAB) 1262// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_tlab, TLAB) 1263// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_tlab, TLAB) 1264// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_tlab, TLAB) 1265// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_tlab, TLAB) 1266GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_tlab, TLAB) 1267GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_tlab, TLAB) 1268GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_tlab, TLAB) 1269 1270// If isInitialized=1 then the compiler assumes the object's class has already been initialized. 1271// If isInitialized=0 the compiler can only assume it's been at least resolved. 1272.macro ART_QUICK_ALLOC_OBJECT_ROSALLOC c_name, cxx_name, isInitialized 1273ENTRY \c_name 1274 // Fast path rosalloc allocation. 1275 // x0: type, xSELF(x19): Thread::Current 1276 // x1-x7: free. 1277 ldr x3, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] // Check if the thread local 1278 // allocation stack has room. 1279 // ldp won't work due to large offset. 1280 ldr x4, [xSELF, #THREAD_LOCAL_ALLOC_STACK_END_OFFSET] 1281 cmp x3, x4 1282 bhs .Lslow_path\c_name 1283 ldr w3, [x0, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (x3) 1284 cmp x3, #ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE // Check if the size is for a thread 1285 // local allocation. 1286 // If the class is not yet visibly initialized, or it is finalizable, 1287 // the object size will be very large to force the branch below to be taken. 1288 // 1289 // See Class::SetStatus() in class.cc for more details. 1290 bhs .Lslow_path\c_name 1291 // Compute the rosalloc bracket index 1292 // from the size. Since the size is 1293 // already aligned we can combine the 1294 // two shifts together. 1295 add x4, xSELF, x3, lsr #(ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT - POINTER_SIZE_SHIFT) 1296 // Subtract pointer size since there 1297 // are no runs for 0 byte allocations 1298 // and the size is already aligned. 1299 ldr x4, [x4, #(THREAD_ROSALLOC_RUNS_OFFSET - __SIZEOF_POINTER__)] 1300 // Load the free list head (x3). This 1301 // will be the return val. 1302 ldr x3, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1303 cbz x3, .Lslow_path\c_name 1304 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1305 ldr x1, [x3, #ROSALLOC_SLOT_NEXT_OFFSET] // Load the next pointer of the head 1306 // and update the list head with the 1307 // next pointer. 1308 str x1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1309 // Store the class pointer in the 1310 // header. This also overwrites the 1311 // next pointer. The offsets are 1312 // asserted to match. 1313 1314#if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET 1315#error "Class pointer needs to overwrite next pointer." 1316#endif 1317 POISON_HEAP_REF w0 1318 str w0, [x3, #MIRROR_OBJECT_CLASS_OFFSET] 1319 // Push the new object onto the thread 1320 // local allocation stack and 1321 // increment the thread local 1322 // allocation stack top. 1323 ldr x1, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1324 str w3, [x1], #COMPRESSED_REFERENCE_SIZE // (Increment x1 as a side effect.) 1325 str x1, [xSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1326 // Decrement the size of the free list 1327 1328 // After this "STR" the object is published to the thread local allocation stack, 1329 // and it will be observable from a runtime internal (eg. Heap::VisitObjects) point of view. 1330 // It is not yet visible to the running (user) compiled code until after the return. 1331 // 1332 // To avoid the memory barrier prior to the "STR", a trick is employed, by differentiating 1333 // the state of the allocation stack slot. It can be a pointer to one of: 1334 // 0) Null entry, because the stack was bumped but the new pointer wasn't written yet. 1335 // (The stack initial state is "null" pointers). 1336 // 1) A partially valid object, with an invalid class pointer to the next free rosalloc slot. 1337 // 2) A fully valid object, with a valid class pointer pointing to a real class. 1338 // Other states are not allowed. 1339 // 1340 // An object that is invalid only temporarily, and will eventually become valid. 1341 // The internal runtime code simply checks if the object is not null or is partial and then 1342 // ignores it. 1343 // 1344 // (Note: The actual check is done by seeing if a non-null object has a class pointer pointing 1345 // to ClassClass, and that the ClassClass's class pointer is self-cyclic. A rosalloc free slot 1346 // "next" pointer is not-cyclic.) 1347 // 1348 // See also b/28790624 for a listing of CLs dealing with this race. 1349 ldr w1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1350 sub x1, x1, #1 1351 // TODO: consider combining this store 1352 // and the list head store above using 1353 // strd. 1354 str w1, [x4, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1355 1356 mov x0, x3 // Set the return value and return. 1357 // No barrier. The class is already observably initialized (otherwise the fast 1358 // path size check above would fail) and new-instance allocations are protected 1359 // from publishing by the compiler which inserts its own StoreStore barrier. 1360 ret 1361.Lslow_path\c_name: 1362 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1363 mov x1, xSELF // pass Thread::Current 1364 CALL_SYMBOL \cxx_name 1365 RESTORE_SAVE_REFS_ONLY_FRAME 1366 REFRESH_MARKING_REGISTER 1367 RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER 1368END \c_name 1369.endm 1370 1371ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_resolved_rosalloc, \ 1372 artAllocObjectFromCodeResolvedRosAlloc, /* isInitialized */ 0 1373ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_initialized_rosalloc, \ 1374 artAllocObjectFromCodeInitializedRosAlloc, /* isInitialized */ 1 1375 1376// If isInitialized=1 then the compiler assumes the object's class has already been initialized. 1377// If isInitialized=0 the compiler can only assume it's been at least resolved. 1378.macro ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED slowPathLabel isInitialized 1379 ldr x4, [xSELF, #THREAD_LOCAL_POS_OFFSET] 1380 ldr x5, [xSELF, #THREAD_LOCAL_END_OFFSET] 1381 ldr w7, [x0, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (x7). 1382 add x6, x4, x7 // Add object size to tlab pos. 1383 cmp x6, x5 // Check if it fits, overflow works 1384 // since the tlab pos and end are 32 1385 // bit values. 1386 1387 // If the class is not yet visibly initialized, or it is finalizable, 1388 // the object size will be very large to force the branch below to be taken. 1389 // 1390 // See Class::SetStatus() in class.cc for more details. 1391 bhi \slowPathLabel 1392 str x6, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1393 POISON_HEAP_REF w0 1394 str w0, [x4, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1395 mov x0, x4 1396 // No barrier. The class is already observably initialized (otherwise the fast 1397 // path size check above would fail) and new-instance allocations are protected 1398 // from publishing by the compiler which inserts its own StoreStore barrier. 1399 ret 1400.endm 1401 1402// The common code for art_quick_alloc_object_*region_tlab 1403// Currently the implementation ignores isInitialized. TODO(b/172087402): clean this up. 1404// Caller must execute a constructor fence after this. 1405.macro GENERATE_ALLOC_OBJECT_RESOLVED_TLAB name, entrypoint, isInitialized 1406ENTRY \name 1407 // Fast path region tlab allocation. 1408 // x0: type, xSELF(x19): Thread::Current 1409 // x1-x7: free. 1410 ALLOC_OBJECT_TLAB_FAST_PATH_RESOLVED .Lslow_path\name, \isInitialized 1411.Lslow_path\name: 1412 SETUP_SAVE_REFS_ONLY_FRAME // Save callee saves in case of GC. 1413 mov x1, xSELF // Pass Thread::Current. 1414 CALL_SYMBOL \entrypoint // (mirror::Class*, Thread*) 1415 RESTORE_SAVE_REFS_ONLY_FRAME 1416 REFRESH_MARKING_REGISTER 1417 RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER 1418END \name 1419.endm 1420 1421GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ 1422 art_quick_alloc_object_resolved_region_tlab, \ 1423 artAllocObjectFromCodeResolvedRegionTLAB, /* isInitialized */ 0 1424GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ 1425 art_quick_alloc_object_initialized_region_tlab, \ 1426 artAllocObjectFromCodeInitializedRegionTLAB, /* isInitialized */ 1 1427GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ 1428 art_quick_alloc_object_resolved_tlab, \ 1429 artAllocObjectFromCodeResolvedTLAB, /* isInitialized */ 0 1430GENERATE_ALLOC_OBJECT_RESOLVED_TLAB \ 1431 art_quick_alloc_object_initialized_tlab, \ 1432 artAllocObjectFromCodeInitializedTLAB, /* isInitialized */ 1 1433 1434.macro ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE \ 1435 slowPathLabel, xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1436 and \xTemp1, \xTemp1, #OBJECT_ALIGNMENT_MASK_TOGGLED64 // Apply alignment mask 1437 // (addr + 7) & ~7. The mask must 1438 // be 64 bits to keep high bits in 1439 // case of overflow. 1440 // Negative sized arrays are handled here since xCount holds a zero extended 32 bit value. 1441 // Negative ints become large 64 bit unsigned ints which will always be larger than max signed 1442 // 32 bit int. Since the max shift for arrays is 3, it can not become a negative 64 bit int. 1443 cmp \xTemp1, #MIN_LARGE_OBJECT_THRESHOLD // Possibly a large object, go slow 1444 bhs \slowPathLabel // path. 1445 1446 ldr \xTemp0, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Check tlab for space, note that 1447 // we use (end - begin) to handle 1448 // negative size arrays. It is 1449 // assumed that a negative size will 1450 // always be greater unsigned than 1451 // region size. 1452 ldr \xTemp2, [xSELF, #THREAD_LOCAL_END_OFFSET] 1453 sub \xTemp2, \xTemp2, \xTemp0 1454 cmp \xTemp1, \xTemp2 1455 1456 // The array class is always initialized here. Unlike new-instance, 1457 // this does not act as a double test. 1458 bhi \slowPathLabel 1459 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber x0 and x1. 1460 // Move old thread_local_pos to x0 1461 // for the return value. 1462 mov x0, \xTemp0 1463 add \xTemp0, \xTemp0, \xTemp1 1464 str \xTemp0, [xSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1465 POISON_HEAP_REF \wClass 1466 str \wClass, [x0, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1467 str \wCount, [x0, #MIRROR_ARRAY_LENGTH_OFFSET] // Store the array length. 1468// new-array is special. The class is loaded and immediately goes to the Initialized state 1469// before it is published. Therefore the only fence needed is for the publication of the object. 1470// See ClassLinker::CreateArrayClass() for more details. 1471 1472// For publication of the new array, we don't need a 'dmb ishst' here. 1473// The compiler generates 'dmb ishst' for all new-array insts. 1474 ret 1475.endm 1476 1477// Caller must execute a constructor fence after this. 1478.macro GENERATE_ALLOC_ARRAY_TLAB name, entrypoint, size_setup 1479ENTRY \name 1480 // Fast path array allocation for region tlab allocation. 1481 // x0: mirror::Class* type 1482 // x1: int32_t component_count 1483 // x2-x7: free. 1484 mov x3, x0 1485 \size_setup x3, w3, x1, w1, x4, w4, x5, w5, x6, w6 1486 ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE \ 1487 .Lslow_path\name, x3, w3, x1, w1, x4, w4, x5, w5, x6, w6 1488.Lslow_path\name: 1489 // x0: mirror::Class* klass 1490 // x1: int32_t component_count 1491 // x2: Thread* self 1492 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 1493 mov x2, xSELF // pass Thread::Current 1494 CALL_SYMBOL \entrypoint 1495 RESTORE_SAVE_REFS_ONLY_FRAME 1496 REFRESH_MARKING_REGISTER 1497 RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER 1498END \name 1499.endm 1500 1501.macro COMPUTE_ARRAY_SIZE_UNKNOWN \ 1502 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1503 // Array classes are never finalizable or uninitialized, no need to check. 1504 ldr \wTemp0, [\xClass, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] // Load component type 1505 UNPOISON_HEAP_REF \wTemp0 1506 ldr \wTemp0, [\xTemp0, #MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET] 1507 lsr \xTemp0, \xTemp0, #PRIMITIVE_TYPE_SIZE_SHIFT_SHIFT // Component size shift is in high 16 1508 // bits. 1509 // xCount is holding a 32 bit value, 1510 // it can not overflow. 1511 lsl \xTemp1, \xCount, \xTemp0 // Calculate data size 1512 // Add array data offset and alignment. 1513 add \xTemp1, \xTemp1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1514#if MIRROR_LONG_ARRAY_DATA_OFFSET != MIRROR_INT_ARRAY_DATA_OFFSET + 4 1515#error Long array data offset must be 4 greater than int array data offset. 1516#endif 1517 1518 add \xTemp0, \xTemp0, #1 // Add 4 to the length only if the 1519 // component size shift is 3 1520 // (for 64 bit alignment). 1521 and \xTemp0, \xTemp0, #4 1522 add \xTemp1, \xTemp1, \xTemp0 1523.endm 1524 1525.macro COMPUTE_ARRAY_SIZE_8 \ 1526 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1527 // Add array data offset and alignment. 1528 add \xTemp1, \xCount, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1529.endm 1530 1531.macro COMPUTE_ARRAY_SIZE_16 \ 1532 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1533 lsl \xTemp1, \xCount, #1 1534 // Add array data offset and alignment. 1535 add \xTemp1, \xTemp1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1536.endm 1537 1538.macro COMPUTE_ARRAY_SIZE_32 \ 1539 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1540 lsl \xTemp1, \xCount, #2 1541 // Add array data offset and alignment. 1542 add \xTemp1, \xTemp1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1543.endm 1544 1545.macro COMPUTE_ARRAY_SIZE_64 \ 1546 xClass, wClass, xCount, wCount, xTemp0, wTemp0, xTemp1, wTemp1, xTemp2, wTemp2 1547 lsl \xTemp1, \xCount, #3 1548 // Add array data offset and alignment. 1549 add \xTemp1, \xTemp1, #(MIRROR_WIDE_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1550.endm 1551 1552// TODO(ngeoffray): art_quick_alloc_array_resolved_region_tlab is not used for arm64, remove 1553// the entrypoint once all backends have been updated to use the size variants. 1554GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_region_tlab, \ 1555 artAllocArrayFromCodeResolvedRegionTLAB, \ 1556 COMPUTE_ARRAY_SIZE_UNKNOWN 1557GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_region_tlab, \ 1558 artAllocArrayFromCodeResolvedRegionTLAB, \ 1559 COMPUTE_ARRAY_SIZE_8 1560GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_region_tlab, \ 1561 artAllocArrayFromCodeResolvedRegionTLAB, \ 1562 COMPUTE_ARRAY_SIZE_16 1563GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_region_tlab, \ 1564 artAllocArrayFromCodeResolvedRegionTLAB, \ 1565 COMPUTE_ARRAY_SIZE_32 1566GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_region_tlab, \ 1567 artAllocArrayFromCodeResolvedRegionTLAB, \ 1568 COMPUTE_ARRAY_SIZE_64 1569GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_tlab, \ 1570 artAllocArrayFromCodeResolvedTLAB, \ 1571 COMPUTE_ARRAY_SIZE_UNKNOWN 1572GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_tlab, \ 1573 artAllocArrayFromCodeResolvedTLAB, \ 1574 COMPUTE_ARRAY_SIZE_8 1575GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_tlab, \ 1576 artAllocArrayFromCodeResolvedTLAB, \ 1577 COMPUTE_ARRAY_SIZE_16 1578GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_tlab, \ 1579 artAllocArrayFromCodeResolvedTLAB, \ 1580 COMPUTE_ARRAY_SIZE_32 1581GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, \ 1582 artAllocArrayFromCodeResolvedTLAB, \ 1583 COMPUTE_ARRAY_SIZE_64 1584 1585 /* 1586 * Called by managed code when the thread has been asked to suspend. 1587 */ 1588 .extern artTestSuspendFromCode 1589ENTRY art_quick_test_suspend 1590 // Save callee saves for stack crawl. 1591 SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET 1592 mov x0, xSELF 1593 CALL_SYMBOL artTestSuspendFromCode // (Thread*) 1594 1595 CFI_REMEMBER_STATE 1596 cbnz x0, .Ltest_suspend_deoptimize 1597 1598 // Normal return. 1599 RESTORE_SAVE_EVERYTHING_FRAME 1600 REFRESH_MARKING_REGISTER 1601 REFRESH_SUSPEND_CHECK_REGISTER 1602 ret 1603 1604.Ltest_suspend_deoptimize: 1605 // Deoptimize. 1606 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 1607 CALL_SYMBOL art_quick_do_long_jump // (Context*) 1608 brk 0 // Unreached 1609END art_quick_test_suspend 1610 1611 /* 1612 * Redirection point from implicit suspend check fault handler. 1613 */ 1614 .extern artImplicitSuspendFromCode 1615ENTRY art_quick_implicit_suspend 1616 // Save callee saves for stack crawl. 1617 SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET 1618 mov x0, xSELF 1619 CALL_SYMBOL artImplicitSuspendFromCode // (Thread*) 1620 1621 CFI_REMEMBER_STATE 1622 cbnz x0, .Limplicit_suspend_deopt 1623 1624 RESTORE_SAVE_EVERYTHING_FRAME 1625 REFRESH_MARKING_REGISTER 1626 REFRESH_SUSPEND_CHECK_REGISTER 1627 br lr // Do not use RET as we do not enter the entrypoint with "BL". 1628 1629.Limplicit_suspend_deopt: 1630 // Deoptimize. 1631 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 1632 CALL_SYMBOL art_quick_do_long_jump // (Context*) 1633 brk 0 // Unreached 1634END art_quick_implicit_suspend 1635 1636 /* 1637 * Called by managed code that is attempting to call a method on a proxy class. On entry 1638 * x0 holds the proxy method and x1 holds the receiver; The frame size of the invoked proxy 1639 * method agrees with a ref and args callee save frame. 1640 */ 1641 .extern artQuickProxyInvokeHandler 1642ENTRY art_quick_proxy_invoke_handler 1643 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 1644 mov x2, xSELF // pass Thread::Current 1645 mov x3, sp // pass SP 1646 CALL_SYMBOL artQuickProxyInvokeHandler // (Method* proxy method, receiver, Thread*, SP) 1647 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 1648 CFI_REMEMBER_STATE 1649 cbnz x2, .Lexception_in_proxy // success if no exception is pending 1650 RESTORE_SAVE_REFS_AND_ARGS_FRAME // Restore frame 1651 REFRESH_MARKING_REGISTER 1652 fmov d0, x0 // Store result in d0 in case it was float or double 1653 ret // return on success 1654.Lexception_in_proxy: 1655 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_REFS_AND_ARGS 1656 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1657 DELIVER_PENDING_EXCEPTION 1658END art_quick_proxy_invoke_handler 1659 1660 /* 1661 * Called to resolve an imt conflict. 1662 * x0 is the conflict ArtMethod. 1663 * xIP1 is a hidden argument that holds the target interface method. 1664 * 1665 * Note that this stub writes to xIP0, xIP1, and x0. 1666 */ 1667ENTRY art_quick_imt_conflict_trampoline 1668 ldr xIP0, [x0, #ART_METHOD_JNI_OFFSET_64] // Load ImtConflictTable 1669 ldr x0, [xIP0] // Load first entry in ImtConflictTable. 1670.Limt_table_iterate: 1671 cmp x0, xIP1 1672 // Branch if found. Benchmarks have shown doing a branch here is better. 1673 beq .Limt_table_found 1674 // If the entry is null, the interface method is not in the ImtConflictTable. 1675 cbz x0, .Lconflict_trampoline 1676 // Iterate over the entries of the ImtConflictTable. 1677 ldr x0, [xIP0, #(2 * __SIZEOF_POINTER__)]! 1678 b .Limt_table_iterate 1679.Limt_table_found: 1680 // We successfully hit an entry in the table. Load the target method 1681 // and jump to it. 1682 ldr x0, [xIP0, #__SIZEOF_POINTER__] 1683 ldr xIP0, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1684 br xIP0 1685.Lconflict_trampoline: 1686 // Call the runtime stub to populate the ImtConflictTable and jump to the 1687 // resolved method. 1688 mov x0, xIP1 // Load interface method 1689 INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline 1690END art_quick_imt_conflict_trampoline 1691 1692ENTRY art_quick_resolution_trampoline 1693 SETUP_SAVE_REFS_AND_ARGS_FRAME 1694 mov x2, xSELF 1695 mov x3, sp 1696 CALL_SYMBOL artQuickResolutionTrampoline // (called, receiver, Thread*, SP) 1697 CFI_REMEMBER_STATE 1698 cbz x0, 1f 1699 mov xIP0, x0 // Remember returned code pointer in xIP0. 1700 ldr x0, [sp, #0] // artQuickResolutionTrampoline puts called method in *SP. 1701 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1702 REFRESH_MARKING_REGISTER 1703 br xIP0 17041: 1705 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_REFS_AND_ARGS 1706 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1707 DELIVER_PENDING_EXCEPTION 1708END art_quick_resolution_trampoline 1709 1710/* 1711 * Generic JNI frame layout: 1712 * 1713 * #-------------------# 1714 * | | 1715 * | caller method... | 1716 * #-------------------# <--- SP on entry 1717 * | Return X30/LR | 1718 * | X29/FP | callee save 1719 * | X28 | callee save 1720 * | X27 | callee save 1721 * | X26 | callee save 1722 * | X25 | callee save 1723 * | X24 | callee save 1724 * | X23 | callee save 1725 * | X22 | callee save 1726 * | X21 | callee save 1727 * | X20 | callee save 1728 * | X7 | arg7 1729 * | X6 | arg6 1730 * | X5 | arg5 1731 * | X4 | arg4 1732 * | X3 | arg3 1733 * | X2 | arg2 1734 * | X1 | arg1 1735 * | D7 | float arg 8 1736 * | D6 | float arg 7 1737 * | D5 | float arg 6 1738 * | D4 | float arg 5 1739 * | D3 | float arg 4 1740 * | D2 | float arg 3 1741 * | D1 | float arg 2 1742 * | D0 | float arg 1 1743 * | padding | // 8B 1744 * | Method* | <- X0 (Managed frame similar to SaveRefsAndArgs.) 1745 * #-------------------# 1746 * | local ref cookie | // 4B 1747 * | padding | // 0B or 4B to align stack args on 8B address 1748 * #-------------------# 1749 * | JNI Stack Args | // Empty if all args fit into registers x0-x7, d0-d7. 1750 * #-------------------# <--- SP on native call (1) 1751 * | Free scratch | 1752 * #-------------------# 1753 * | SP for JNI call | // Pointer to (1). 1754 * #-------------------# 1755 * | Hidden arg | // For @CriticalNative 1756 * #-------------------# 1757 * | | 1758 * | Stack for Regs | The trampoline assembly will pop these values 1759 * | | into registers for native call 1760 * #-------------------# 1761 */ 1762 /* 1763 * Called to do a generic JNI down-call 1764 */ 1765 .extern artQuickGenericJniTrampoline 1766ENTRY art_quick_generic_jni_trampoline 1767 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_X0 1768 1769 // Save SP, so we can have static CFI info. 1770 mov x28, sp 1771 .cfi_def_cfa_register x28 1772 1773 mov xIP0, #GENERIC_JNI_TRAMPOLINE_RESERVED_AREA 1774 sub sp, sp, xIP0 1775 1776 // prepare for artQuickGenericJniTrampoline call 1777 // (Thread*, managed_sp, reserved_area) 1778 // x0 x1 x2 <= C calling convention 1779 // xSELF x28 sp <= where they are 1780 1781 mov x0, xSELF // Thread* 1782 mov x1, x28 // SP for the managed frame. 1783 mov x2, sp // reserved area for arguments and other saved data (up to managed frame) 1784 CALL_SYMBOL artQuickGenericJniTrampoline // (Thread*, sp) 1785 1786 // The C call will have registered the complete save-frame on success. 1787 // The result of the call is: 1788 // x0: pointer to native code, 0 on error. 1789 // The bottom of the reserved area contains values for arg registers, 1790 // hidden arg register and SP for out args for the call. 1791 1792 // Check for error (class init check or locking for synchronized native method can throw). 1793 cbz x0, .Lexception_in_native 1794 1795 // Save the code pointer 1796 mov xIP0, x0 1797 1798 // Load parameters from frame into registers. 1799 ldp x0, x1, [sp] 1800 ldp x2, x3, [sp, #16] 1801 ldp x4, x5, [sp, #32] 1802 ldp x6, x7, [sp, #48] 1803 1804 ldp d0, d1, [sp, #64] 1805 ldp d2, d3, [sp, #80] 1806 ldp d4, d5, [sp, #96] 1807 ldp d6, d7, [sp, #112] 1808 1809 // Load hidden arg (x15) for @CriticalNative and SP for out args. 1810 ldp x15, xIP1, [sp, #128] 1811 1812 // Apply the new SP for out args, releasing unneeded reserved area. 1813 mov sp, xIP1 1814 1815 blr xIP0 // native call. 1816 1817 // result sign extension is handled in C code 1818 // prepare for artQuickGenericJniEndTrampoline call 1819 // (Thread*, result, result_f) 1820 // x0 x1 x2 <= C calling convention 1821 mov x1, x0 // Result (from saved). 1822 mov x0, xSELF // Thread register. 1823 fmov x2, d0 // d0 will contain floating point result, but needs to go into x2 1824 1825 CALL_SYMBOL artQuickGenericJniEndTrampoline 1826 1827 // Pending exceptions possible. 1828 ldr x2, [xSELF, THREAD_EXCEPTION_OFFSET] 1829 cbnz x2, .Lexception_in_native 1830 1831 // Tear down the alloca. 1832 mov sp, x28 1833 1834 LOAD_RUNTIME_INSTANCE x1 1835 ldr x1, [x1, #RUNTIME_INSTRUMENTATION_OFFSET] 1836 ldrb w1, [x1, #INSTRUMENTATION_RUN_EXIT_HOOKS_OFFSET] 1837 CFI_REMEMBER_STATE 1838 cbnz w1, .Lcall_method_exit_hook 1839.Lcall_method_exit_hook_done: 1840 1841 // Tear down the callee-save frame. 1842 .cfi_def_cfa_register sp 1843 // Restore callee-saves and LR as in `RESTORE_SAVE_REFS_AND_ARGS_FRAME` 1844 // but do not restore argument registers. 1845 // Note: Likewise, we could avoid restoring X20 in the case of Baker 1846 // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER 1847 // later; but it's not worth handling this special case. 1848#if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 224) 1849#error "FRAME_SIZE_SAVE_REFS_AND_ARGS(ARM64) size not as expected." 1850#endif 1851 RESTORE_REG x20, 136 1852 RESTORE_TWO_REGS x21, x22, 144 1853 RESTORE_TWO_REGS x23, x24, 160 1854 RESTORE_TWO_REGS x25, x26, 176 1855 RESTORE_TWO_REGS x27, x28, 192 1856 RESTORE_TWO_REGS x29, xLR, 208 1857 // Remove the frame. 1858 DECREASE_FRAME FRAME_SIZE_SAVE_REFS_AND_ARGS 1859 1860 REFRESH_MARKING_REGISTER 1861 1862 // store into fpr, for when it's a fpr return... 1863 fmov d0, x0 1864 ret 1865 1866.Lcall_method_exit_hook: 1867 CFI_RESTORE_STATE_AND_DEF_CFA x28, FRAME_SIZE_SAVE_REFS_AND_ARGS 1868 fmov d0, x0 1869 mov x4, FRAME_SIZE_SAVE_REFS_AND_ARGS 1870 CALL_SYMBOL art_quick_method_exit_hook 1871 b .Lcall_method_exit_hook_done 1872 1873.Lexception_in_native: 1874 // Move to x1 then sp to please assembler. 1875 ldr x1, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 1876 add sp, x1, #-1 // Remove the GenericJNI tag. 1877 CALL_SYMBOL art_deliver_pending_exception 1878END art_quick_generic_jni_trampoline 1879 1880ENTRY art_deliver_pending_exception 1881 # This will create a new save-all frame, required by the runtime. 1882 DELIVER_PENDING_EXCEPTION 1883END art_deliver_pending_exception 1884 1885/* 1886 * Called to bridge from the quick to interpreter ABI. On entry the arguments match those 1887 * of a quick call: 1888 * x0 = method being called/to bridge to. 1889 * x1..x7, d0..d7 = arguments to that method. 1890 */ 1891ENTRY art_quick_to_interpreter_bridge 1892 SETUP_SAVE_REFS_AND_ARGS_FRAME // Set up frame and save arguments. 1893 1894 // x0 will contain mirror::ArtMethod* method. 1895 mov x1, xSELF // How to get Thread::Current() ??? 1896 mov x2, sp 1897 1898 // uint64_t artQuickToInterpreterBridge(mirror::ArtMethod* method, Thread* self, 1899 // mirror::ArtMethod** sp) 1900 CALL_SYMBOL artQuickToInterpreterBridge 1901 1902 RESTORE_SAVE_REFS_AND_ARGS_FRAME // TODO: no need to restore arguments in this case. 1903 REFRESH_MARKING_REGISTER 1904 1905 fmov d0, x0 1906 1907 RETURN_OR_DELIVER_PENDING_EXCEPTION 1908END art_quick_to_interpreter_bridge 1909 1910/* 1911 * Called to attempt to execute an obsolete method. 1912 */ 1913ONE_ARG_RUNTIME_EXCEPTION art_invoke_obsolete_method_stub, artInvokeObsoleteMethod 1914 1915 /* 1916 * Compiled code has requested that we deoptimize into the interpreter. The deoptimization 1917 * will long jump to the upcall with a special exception of -1. 1918 */ 1919 .extern artDeoptimizeFromCompiledCode 1920ENTRY art_quick_deoptimize_from_compiled_code 1921 SETUP_SAVE_EVERYTHING_FRAME 1922 mov x1, xSELF // Pass thread. 1923 CALL_SYMBOL artDeoptimizeFromCompiledCode // (DeoptimizationKind, Thread*) 1924 CALL_SYMBOL art_quick_do_long_jump // (Context*) 1925 brk 0 // Unreached 1926END art_quick_deoptimize_from_compiled_code 1927 1928 1929 /* 1930 * String's indexOf. 1931 * 1932 * TODO: Not very optimized. 1933 * On entry: 1934 * x0: string object (known non-null) 1935 * w1: char to match (known <= 0xFFFF) 1936 * w2: Starting offset in string data 1937 */ 1938ENTRY art_quick_indexof 1939#if (STRING_COMPRESSION_FEATURE) 1940 ldr w4, [x0, #MIRROR_STRING_COUNT_OFFSET] 1941#else 1942 ldr w3, [x0, #MIRROR_STRING_COUNT_OFFSET] 1943#endif 1944 add x0, x0, #MIRROR_STRING_VALUE_OFFSET 1945#if (STRING_COMPRESSION_FEATURE) 1946 /* w4 holds count (with flag) and w3 holds actual length */ 1947 lsr w3, w4, #1 1948#endif 1949 /* Clamp start to [0..count] */ 1950 cmp w2, #0 1951 csel w2, wzr, w2, lt 1952 cmp w2, w3 1953 csel w2, w3, w2, gt 1954 1955 /* Save a copy to compute result */ 1956 mov x5, x0 1957 1958#if (STRING_COMPRESSION_FEATURE) 1959 tbz w4, #0, .Lstring_indexof_compressed 1960#endif 1961 /* Build pointer to start of data to compare and pre-bias */ 1962 add x0, x0, x2, lsl #1 1963 sub x0, x0, #2 1964 /* Compute iteration count */ 1965 sub w2, w3, w2 1966 1967 /* 1968 * At this point we have: 1969 * x0: start of the data to test 1970 * w1: char to compare 1971 * w2: iteration count 1972 * x5: original start of string data 1973 */ 1974 1975 subs w2, w2, #4 1976 b.lt .Lindexof_remainder 1977 1978.Lindexof_loop4: 1979 ldrh w6, [x0, #2]! 1980 ldrh w7, [x0, #2]! 1981 ldrh wIP0, [x0, #2]! 1982 ldrh wIP1, [x0, #2]! 1983 cmp w6, w1 1984 b.eq .Lmatch_0 1985 cmp w7, w1 1986 b.eq .Lmatch_1 1987 cmp wIP0, w1 1988 b.eq .Lmatch_2 1989 cmp wIP1, w1 1990 b.eq .Lmatch_3 1991 subs w2, w2, #4 1992 b.ge .Lindexof_loop4 1993 1994.Lindexof_remainder: 1995 adds w2, w2, #4 1996 b.eq .Lindexof_nomatch 1997 1998.Lindexof_loop1: 1999 ldrh w6, [x0, #2]! 2000 cmp w6, w1 2001 b.eq .Lmatch_3 2002 subs w2, w2, #1 2003 b.ne .Lindexof_loop1 2004 2005.Lindexof_nomatch: 2006 mov x0, #-1 2007 ret 2008 2009.Lmatch_0: 2010 sub x0, x0, #6 2011 sub x0, x0, x5 2012 asr x0, x0, #1 2013 ret 2014.Lmatch_1: 2015 sub x0, x0, #4 2016 sub x0, x0, x5 2017 asr x0, x0, #1 2018 ret 2019.Lmatch_2: 2020 sub x0, x0, #2 2021 sub x0, x0, x5 2022 asr x0, x0, #1 2023 ret 2024.Lmatch_3: 2025 sub x0, x0, x5 2026 asr x0, x0, #1 2027 ret 2028#if (STRING_COMPRESSION_FEATURE) 2029 /* 2030 * Comparing compressed string character-per-character with 2031 * input character 2032 */ 2033.Lstring_indexof_compressed: 2034 add x0, x0, x2 2035 sub x0, x0, #1 2036 sub w2, w3, w2 2037.Lstring_indexof_compressed_loop: 2038 subs w2, w2, #1 2039 b.lt .Lindexof_nomatch 2040 ldrb w6, [x0, #1]! 2041 cmp w6, w1 2042 b.eq .Lstring_indexof_compressed_matched 2043 b .Lstring_indexof_compressed_loop 2044.Lstring_indexof_compressed_matched: 2045 sub x0, x0, x5 2046 ret 2047#endif 2048END art_quick_indexof 2049 2050 .extern artStringBuilderAppend 2051ENTRY art_quick_string_builder_append 2052 SETUP_SAVE_REFS_ONLY_FRAME // save callee saves in case of GC 2053 add x1, sp, #(FRAME_SIZE_SAVE_REFS_ONLY + __SIZEOF_POINTER__) // pass args 2054 mov x2, xSELF // pass Thread::Current 2055 CALL_SYMBOL artStringBuilderAppend // (uint32_t, const unit32_t*, Thread*) 2056 RESTORE_SAVE_REFS_ONLY_FRAME 2057 REFRESH_MARKING_REGISTER 2058 RETURN_OR_DEOPT_IF_RESULT_IS_NON_NULL_OR_DELIVER 2059END art_quick_string_builder_append 2060 2061 /* 2062 * Create a function `name` calling the ReadBarrier::Mark routine, 2063 * getting its argument and returning its result through W register 2064 * `wreg` (corresponding to X register `xreg`), saving and restoring 2065 * all caller-save registers. 2066 * 2067 * The generated function follows a non-standard runtime calling convention: 2068 * - register `reg` (which may be different from W0) is used to pass the (sole) argument, 2069 * - register `reg` (which may be different from W0) is used to return the result, 2070 * - all other registers are callee-save (the values they hold are preserved). 2071 */ 2072.macro READ_BARRIER_MARK_REG name, wreg, xreg 2073ENTRY \name 2074 // Reference is null, no work to do at all. 2075 cbz \wreg, .Lret_rb_\name 2076 // Use wIP0 as temp and check the mark bit of the reference. wIP0 is not used by the compiler. 2077 ldr wIP0, [\xreg, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 2078 tbz wIP0, #LOCK_WORD_MARK_BIT_SHIFT, .Lnot_marked_rb_\name 2079.Lret_rb_\name: 2080 ret 2081.Lnot_marked_rb_\name: 2082 // Check if the top two bits are one, if this is the case it is a forwarding address. 2083 tst wIP0, wIP0, lsl #1 2084 bmi .Lret_forwarding_address\name 2085.Lslow_rb_\name: 2086 /* 2087 * Allocate 44 stack slots * 8 = 352 bytes: 2088 * - 19 slots for core registers X0-15, X17, X19, LR 2089 * - 1 slot padding 2090 * - 24 slots for floating-point registers D0-D7 and D16-D31 2091 */ 2092 // We must not clobber IP1 since code emitted for HLoadClass and HLoadString 2093 // relies on IP1 being preserved. 2094 // Save all potentially live caller-save core registers. 2095 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 352 2096 SAVE_TWO_REGS x2, x3, 16 2097 SAVE_TWO_REGS x4, x5, 32 2098 SAVE_TWO_REGS x6, x7, 48 2099 SAVE_TWO_REGS x8, x9, 64 2100 SAVE_TWO_REGS x10, x11, 80 2101 SAVE_TWO_REGS x12, x13, 96 2102 SAVE_TWO_REGS x14, x15, 112 2103 SAVE_TWO_REGS x17, x19, 128 // Skip x16, i.e. IP0, and x18, the platform register. 2104 SAVE_REG xLR, 144 // Save also return address. 2105 // Save all potentially live caller-save floating-point registers. 2106 stp d0, d1, [sp, #160] 2107 stp d2, d3, [sp, #176] 2108 stp d4, d5, [sp, #192] 2109 stp d6, d7, [sp, #208] 2110 stp d16, d17, [sp, #224] 2111 stp d18, d19, [sp, #240] 2112 stp d20, d21, [sp, #256] 2113 stp d22, d23, [sp, #272] 2114 stp d24, d25, [sp, #288] 2115 stp d26, d27, [sp, #304] 2116 stp d28, d29, [sp, #320] 2117 stp d30, d31, [sp, #336] 2118 2119 .ifnc \wreg, w0 2120 mov w0, \wreg // Pass arg1 - obj from `wreg` 2121 .endif 2122 CALL_SYMBOL artReadBarrierMark // artReadBarrierMark(obj) 2123 .ifnc \wreg, w0 2124 mov \wreg, w0 // Return result into `wreg` 2125 .endif 2126 2127 // Restore core regs, except `xreg`, as `wreg` is used to return the 2128 // result of this function (simply remove it from the stack instead). 2129 POP_REGS_NE x0, x1, 0, \xreg 2130 POP_REGS_NE x2, x3, 16, \xreg 2131 POP_REGS_NE x4, x5, 32, \xreg 2132 POP_REGS_NE x6, x7, 48, \xreg 2133 POP_REGS_NE x8, x9, 64, \xreg 2134 POP_REGS_NE x10, x11, 80, \xreg 2135 POP_REGS_NE x12, x13, 96, \xreg 2136 POP_REGS_NE x14, x15, 112, \xreg 2137 POP_REGS_NE x17, x19, 128, \xreg 2138 POP_REG_NE xLR, 144, \xreg // Restore also return address. 2139 // Restore floating-point registers. 2140 ldp d0, d1, [sp, #160] 2141 ldp d2, d3, [sp, #176] 2142 ldp d4, d5, [sp, #192] 2143 ldp d6, d7, [sp, #208] 2144 ldp d16, d17, [sp, #224] 2145 ldp d18, d19, [sp, #240] 2146 ldp d20, d21, [sp, #256] 2147 ldp d22, d23, [sp, #272] 2148 ldp d24, d25, [sp, #288] 2149 ldp d26, d27, [sp, #304] 2150 ldp d28, d29, [sp, #320] 2151 ldp d30, d31, [sp, #336] 2152 // Remove frame and return. 2153 DECREASE_FRAME 352 2154 ret 2155.Lret_forwarding_address\name: 2156 // Shift left by the forwarding address shift. This clears out the state bits since they are 2157 // in the top 2 bits of the lock word. 2158 lsl \wreg, wIP0, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 2159 ret 2160END \name 2161.endm 2162 2163READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg00, w0, x0 2164READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, w1, x1 2165READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, w2, x2 2166READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, w3, x3 2167READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg04, w4, x4 2168READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, w5, x5 2169READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, w6, x6 2170READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, w7, x7 2171READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, w8, x8 2172READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, w9, x9 2173READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, w10, x10 2174READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, w11, x11 2175READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, w12, x12 2176READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, w13, x13 2177READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, w14, x14 2178READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg15, w15, x15 2179// READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg16, w16, x16 ip0 is blocked 2180READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg17, w17, x17 2181// READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg18, w18, x18 x18 is blocked 2182READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg19, w19, x19 2183READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg20, w20, x20 2184READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg21, w21, x21 2185READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg22, w22, x22 2186READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg23, w23, x23 2187READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg24, w24, x24 2188READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg25, w25, x25 2189READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg26, w26, x26 2190READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg27, w27, x27 2191READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg28, w28, x28 2192READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg29, w29, x29 2193 2194 2195.macro SELECT_X_OR_W_FOR_MACRO macro_to_use, x, w, xreg 2196 .if \xreg 2197 \macro_to_use \x 2198 .else 2199 \macro_to_use \w 2200 .endif 2201.endm 2202 2203.macro FOR_REGISTERS macro_for_register, macro_for_reserved_register, xreg 2204 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x0, w0, \xreg 2205 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x1, w1, \xreg 2206 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x2, w2, \xreg 2207 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x3, w3, \xreg 2208 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x4, w4, \xreg 2209 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x5, w5, \xreg 2210 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x6, w6, \xreg 2211 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x7, w7, \xreg 2212 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x8, w8, \xreg 2213 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x9, w9, \xreg 2214 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x10, w10, \xreg 2215 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x11, w11, \xreg 2216 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x12, w12, \xreg 2217 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x13, w13, \xreg 2218 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x14, w14, \xreg 2219 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x15, w15, \xreg 2220 \macro_for_reserved_register // IP0 is reserved 2221 \macro_for_reserved_register // IP1 is reserved 2222 \macro_for_reserved_register // x18 is reserved 2223 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x19, w19, \xreg 2224 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x20, w20, \xreg 2225 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x21, w21, \xreg 2226 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x22, w22, \xreg 2227 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x23, w23, \xreg 2228 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x24, w24, \xreg 2229 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x25, w25, \xreg 2230 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x26, w26, \xreg 2231 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x27, w27, \xreg 2232 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x28, w28, \xreg 2233 SELECT_X_OR_W_FOR_MACRO \macro_for_register, x29, w29, \xreg 2234 \macro_for_reserved_register // lr is reserved 2235 \macro_for_reserved_register // sp is reserved 2236.endm 2237 2238.macro FOR_XREGISTERS macro_for_register, macro_for_reserved_register 2239 FOR_REGISTERS \macro_for_register, \macro_for_reserved_register, /* xreg */ 1 2240.endm 2241 2242.macro FOR_WREGISTERS macro_for_register, macro_for_reserved_register 2243 FOR_REGISTERS \macro_for_register, \macro_for_reserved_register, /* xreg */ 0 2244.endm 2245 2246.macro BRK0_BRK0 2247 brk 0 2248 brk 0 2249.endm 2250 2251#if BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET != BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET 2252#error "Array and field introspection code sharing requires same LDR offset." 2253#endif 2254.macro INTROSPECTION_ARRAY_LOAD index_reg 2255 ldr wIP0, [xIP0, \index_reg, lsl #2] 2256 BRANCH_SYMBOL art_quick_read_barrier_mark_introspection 2257.endm 2258 2259.macro MOV_WIP0_TO_WREG_AND_BL_LR reg 2260 mov \reg, wIP0 2261 br lr // Do not use RET as we do not enter the entrypoint with "BL". 2262.endm 2263 2264.macro READ_BARRIER_MARK_INTROSPECTION_SLOW_PATH ldr_offset 2265 /* 2266 * Allocate 42 stack slots * 8 = 336 bytes: 2267 * - 18 slots for core registers X0-15, X19, LR 2268 * - 24 slots for floating-point registers D0-D7 and D16-D31 2269 */ 2270 // Save all potentially live caller-save core registers. 2271 SAVE_TWO_REGS_INCREASE_FRAME x0, x1, 336 2272 SAVE_TWO_REGS x2, x3, 16 2273 SAVE_TWO_REGS x4, x5, 32 2274 SAVE_TWO_REGS x6, x7, 48 2275 SAVE_TWO_REGS x8, x9, 64 2276 SAVE_TWO_REGS x10, x11, 80 2277 SAVE_TWO_REGS x12, x13, 96 2278 SAVE_TWO_REGS x14, x15, 112 2279 // Skip x16, x17, i.e. IP0, IP1, and x18, the platform register. 2280 SAVE_TWO_REGS x19, xLR, 128 // Save return address. 2281 // Save all potentially live caller-save floating-point registers. 2282 stp d0, d1, [sp, #144] 2283 stp d2, d3, [sp, #160] 2284 stp d4, d5, [sp, #176] 2285 stp d6, d7, [sp, #192] 2286 stp d16, d17, [sp, #208] 2287 stp d18, d19, [sp, #224] 2288 stp d20, d21, [sp, #240] 2289 stp d22, d23, [sp, #256] 2290 stp d24, d25, [sp, #272] 2291 stp d26, d27, [sp, #288] 2292 stp d28, d29, [sp, #304] 2293 stp d30, d31, [sp, #320] 2294 2295 mov x0, xIP0 2296 CALL_SYMBOL artReadBarrierMark // artReadBarrierMark(obj) 2297 mov xIP0, x0 2298 2299 // Restore core regs, except x0 and x1 as the return register switch case 2300 // address calculation is smoother with an extra register. 2301 RESTORE_TWO_REGS x2, x3, 16 2302 RESTORE_TWO_REGS x4, x5, 32 2303 RESTORE_TWO_REGS x6, x7, 48 2304 RESTORE_TWO_REGS x8, x9, 64 2305 RESTORE_TWO_REGS x10, x11, 80 2306 RESTORE_TWO_REGS x12, x13, 96 2307 RESTORE_TWO_REGS x14, x15, 112 2308 // Skip x16, x17, i.e. IP0, IP1, and x18, the platform register. 2309 RESTORE_TWO_REGS x19, xLR, 128 // Restore return address. 2310 // Restore caller-save floating-point registers. 2311 ldp d0, d1, [sp, #144] 2312 ldp d2, d3, [sp, #160] 2313 ldp d4, d5, [sp, #176] 2314 ldp d6, d7, [sp, #192] 2315 ldp d16, d17, [sp, #208] 2316 ldp d18, d19, [sp, #224] 2317 ldp d20, d21, [sp, #240] 2318 ldp d22, d23, [sp, #256] 2319 ldp d24, d25, [sp, #272] 2320 ldp d26, d27, [sp, #288] 2321 ldp d28, d29, [sp, #304] 2322 ldp d30, d31, [sp, #320] 2323 2324 ldr x0, [lr, #\ldr_offset] // Load the instruction. 2325 adr xIP1, .Lmark_introspection_return_switch 2326 bfi xIP1, x0, #3, #5 // Calculate switch case address. 2327 RESTORE_TWO_REGS_DECREASE_FRAME x0, x1, 336 2328 br xIP1 2329.endm 2330 2331 /* 2332 * Use introspection to load a reference from the same address as the LDR 2333 * instruction in generated code would load (unless loaded by the thunk, 2334 * see below), call ReadBarrier::Mark() with that reference if needed 2335 * and return it in the same register as the LDR instruction would load. 2336 * 2337 * The entrypoint is called through a thunk that differs across load kinds. 2338 * For field and array loads the LDR instruction in generated code follows 2339 * the branch to the thunk, i.e. the LDR is at [LR, #-4], and the thunk 2340 * knows the holder and performs the gray bit check, returning to the LDR 2341 * instruction if the object is not gray, so this entrypoint no longer 2342 * needs to know anything about the holder. For GC root loads, the LDR 2343 * instruction in generated code precedes the branch to the thunk (i.e. 2344 * the LDR is at [LR, #-8]) and the thunk does not do the gray bit check. 2345 * 2346 * For field accesses and array loads with a constant index the thunk loads 2347 * the reference into IP0 using introspection and calls the main entrypoint, 2348 * art_quick_read_barrier_mark_introspection. With heap poisoning enabled, 2349 * the passed reference is poisoned. 2350 * 2351 * For array accesses with non-constant index, the thunk inserts the bits 2352 * 16-21 of the LDR instruction to the entrypoint address, effectively 2353 * calculating a switch case label based on the index register (bits 16-20) 2354 * and adding an extra offset (bit 21 is set) to differentiate from the 2355 * main entrypoint, then moves the base register to IP0 and jumps to the 2356 * switch case. Therefore we need to align the main entrypoint to 512 bytes, 2357 * accounting for a 256-byte offset followed by 32 array entrypoints 2358 * starting at art_quick_read_barrier_mark_introspection_arrays, each 2359 * containing an LDR (register) and a branch to the main entrypoint. 2360 * 2361 * For GC root accesses we cannot use the main entrypoint because of the 2362 * different offset where the LDR instruction in generated code is located. 2363 * (And even with heap poisoning enabled, GC roots are not poisoned.) 2364 * To re-use the same entrypoint pointer in generated code, we make sure 2365 * that the gc root entrypoint (a copy of the entrypoint with a different 2366 * offset for introspection loads) is located at a known offset (768 bytes, 2367 * or BAKER_MARK_INTROSPECTION_GC_ROOT_ENTRYPOINT_OFFSET) from the main 2368 * entrypoint and the GC root thunk adjusts the entrypoint pointer, moves 2369 * the root register to IP0 and jumps to the customized entrypoint, 2370 * art_quick_read_barrier_mark_introspection_gc_roots. The thunk also 2371 * performs all the fast-path checks, so we need just the slow path. 2372 * The UnsafeCASObject intrinsic is also using the GC root entrypoint with 2373 * MOV instead of LDR, the destination register is in the same bits. 2374 * 2375 * The code structure is 2376 * art_quick_read_barrier_mark_introspection: 2377 * Up to 256 bytes for the main entrypoint code. 2378 * Padding to 256 bytes if needed. 2379 * art_quick_read_barrier_mark_introspection_arrays: 2380 * Exactly 256 bytes for array load switch cases (32x2 instructions). 2381 * .Lmark_introspection_return_switch: 2382 * Exactly 256 bytes for return switch cases (32x2 instructions). 2383 * art_quick_read_barrier_mark_introspection_gc_roots: 2384 * GC root entrypoint code. 2385 */ 2386ENTRY_ALIGNED art_quick_read_barrier_mark_introspection, 512 2387 // At this point, IP0 contains the reference, IP1 can be freely used. 2388 // For heap poisoning, the reference is poisoned, so unpoison it first. 2389 UNPOISON_HEAP_REF wIP0 2390 // If reference is null, just return it in the right register. 2391 cbz wIP0, .Lmark_introspection_return 2392 // Use wIP1 as temp and check the mark bit of the reference. 2393 ldr wIP1, [xIP0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 2394 tbz wIP1, #LOCK_WORD_MARK_BIT_SHIFT, .Lmark_introspection_unmarked 2395.Lmark_introspection_return: 2396 // Without an extra register for the return switch case address calculation, 2397 // we exploit the high word of the xIP0 to temporarily store the ref_reg*8, 2398 // so the return switch below must move wIP0 instead of xIP0 to the register. 2399 ldr wIP1, [lr, #BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET] // Load the instruction. 2400 bfi xIP0, xIP1, #(32 + 3), #5 // Extract ref_reg*8 to high word in xIP0. 2401 adr xIP1, .Lmark_introspection_return_switch 2402 bfxil xIP1, xIP0, #32, #8 // Calculate return switch case address. 2403 br xIP1 2404.Lmark_introspection_unmarked: 2405 // Check if the top two bits are one, if this is the case it is a forwarding address. 2406 tst wIP1, wIP1, lsl #1 2407 bmi .Lmark_introspection_forwarding_address 2408 READ_BARRIER_MARK_INTROSPECTION_SLOW_PATH BAKER_MARK_INTROSPECTION_FIELD_LDR_OFFSET 2409 2410.Lmark_introspection_forwarding_address: 2411 // Shift left by the forwarding address shift. This clears out the state bits since they are 2412 // in the top 2 bits of the lock word. 2413 lsl wIP0, wIP1, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 2414 b .Lmark_introspection_return 2415 2416 // We're very close to the alloted 256B for the entrypoint code before the 2417 // array switch cases. Should we go a little bit over the limit, we can 2418 // move some code after the array switch cases and return switch cases. 2419 .balign 256 2420 .hidden art_quick_read_barrier_mark_introspection_arrays 2421 .global art_quick_read_barrier_mark_introspection_arrays 2422art_quick_read_barrier_mark_introspection_arrays: 2423 FOR_XREGISTERS INTROSPECTION_ARRAY_LOAD, BRK0_BRK0 2424.Lmark_introspection_return_switch: 2425 FOR_WREGISTERS MOV_WIP0_TO_WREG_AND_BL_LR, BRK0_BRK0 2426 .hidden art_quick_read_barrier_mark_introspection_gc_roots 2427 .global art_quick_read_barrier_mark_introspection_gc_roots 2428art_quick_read_barrier_mark_introspection_gc_roots: 2429 READ_BARRIER_MARK_INTROSPECTION_SLOW_PATH BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_OFFSET 2430END art_quick_read_barrier_mark_introspection 2431 2432.extern artInvokePolymorphic 2433ENTRY art_quick_invoke_polymorphic 2434 SETUP_SAVE_REFS_AND_ARGS_FRAME // Save callee saves in case allocation triggers GC. 2435 mov x0, x1 // x0 := receiver 2436 mov x1, xSELF // x1 := Thread::Current() 2437 mov x2, sp // x2 := SP 2438 CALL_SYMBOL artInvokePolymorphic // artInvokePolymorphic(receiver, thread, save_area) 2439 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2440 REFRESH_MARKING_REGISTER 2441 fmov d0, x0 // Result is in x0. Copy to floating return register. 2442 RETURN_OR_DELIVER_PENDING_EXCEPTION 2443END art_quick_invoke_polymorphic 2444 2445 /* 2446 * Slow path for MethodHandle.invokeExact intrinsic. 2447 * That intrinsic has a custom calling convention: the argument allocation doesn't start from 2448 * the receiver (MethodHandle) object, but from the argument following it. That's done to match 2449 * expectation of the underlying method when MethodHandle targets a method. That also affects 2450 * the way arguments are spilled onto the stack. 2451 */ 2452.extern artInvokePolymorphicWithHiddenReceiver 2453ENTRY art_quick_invoke_polymorphic_with_hidden_receiver 2454 // On entry: w0 := receiver 2455 SETUP_SAVE_REFS_AND_ARGS_FRAME // Save callee saves in case allocation triggers GC. 2456 mov x1, xSELF // x1 := Thread::Current() 2457 mov x2, sp // x2 := SP 2458 CALL_SYMBOL artInvokePolymorphicWithHiddenReceiver // invoke with (receiver, thread, save_area) 2459 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2460 REFRESH_MARKING_REGISTER 2461 fmov d0, x0 // Result is in x0. Copy to floating return register. 2462 RETURN_OR_DELIVER_PENDING_EXCEPTION 2463END art_quick_invoke_polymorphic_with_hidden_receiver 2464 2465.extern artInvokeCustom 2466ENTRY art_quick_invoke_custom 2467 SETUP_SAVE_REFS_AND_ARGS_FRAME // Save callee saves in case allocation triggers GC. 2468 // x0 := call_site_idx 2469 mov x1, xSELF // x1 := Thread::Current() 2470 mov x2, sp // x2 := SP 2471 CALL_SYMBOL artInvokeCustom // artInvokeCustom(call_site_idx, thread, save_area) 2472 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2473 REFRESH_MARKING_REGISTER 2474 fmov d0, x0 // Copy result to double result register. 2475 RETURN_OR_DELIVER_PENDING_EXCEPTION 2476END art_quick_invoke_custom 2477 2478// x0 contains the class, x8 contains the inline cache. x9-x15 can be used. 2479ENTRY art_quick_update_inline_cache 2480#if (INLINE_CACHE_SIZE != 5) 2481#error "INLINE_CACHE_SIZE not as expected." 2482#endif 2483#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 2484 // Don't update the cache if we are marking. 2485 cbnz wMR, .Ldone 2486#endif 2487.Lentry1: 2488 ldr w9, [x8, #INLINE_CACHE_CLASSES_OFFSET] 2489 cmp w9, w0 2490 beq .Ldone 2491 cbnz w9, .Lentry2 2492 add x10, x8, #INLINE_CACHE_CLASSES_OFFSET 2493 ldxr w9, [x10] 2494 cbnz w9, .Lentry1 2495 stxr w9, w0, [x10] 2496 cbz w9, .Ldone 2497 b .Lentry1 2498.Lentry2: 2499 ldr w9, [x8, #INLINE_CACHE_CLASSES_OFFSET+4] 2500 cmp w9, w0 2501 beq .Ldone 2502 cbnz w9, .Lentry3 2503 add x10, x8, #INLINE_CACHE_CLASSES_OFFSET+4 2504 ldxr w9, [x10] 2505 cbnz w9, .Lentry2 2506 stxr w9, w0, [x10] 2507 cbz w9, .Ldone 2508 b .Lentry2 2509.Lentry3: 2510 ldr w9, [x8, #INLINE_CACHE_CLASSES_OFFSET+8] 2511 cmp w9, w0 2512 beq .Ldone 2513 cbnz w9, .Lentry4 2514 add x10, x8, #INLINE_CACHE_CLASSES_OFFSET+8 2515 ldxr w9, [x10] 2516 cbnz w9, .Lentry3 2517 stxr w9, w0, [x10] 2518 cbz w9, .Ldone 2519 b .Lentry3 2520.Lentry4: 2521 ldr w9, [x8, #INLINE_CACHE_CLASSES_OFFSET+12] 2522 cmp w9, w0 2523 beq .Ldone 2524 cbnz w9, .Lentry5 2525 add x10, x8, #INLINE_CACHE_CLASSES_OFFSET+12 2526 ldxr w9, [x10] 2527 cbnz w9, .Lentry4 2528 stxr w9, w0, [x10] 2529 cbz w9, .Ldone 2530 b .Lentry4 2531.Lentry5: 2532 // Unconditionally store, the inline cache is megamorphic. 2533 str w0, [x8, #INLINE_CACHE_CLASSES_OFFSET+16] 2534.Ldone: 2535 ret 2536END art_quick_update_inline_cache 2537 2538// On entry, method is at the bottom of the stack. 2539ENTRY art_quick_compile_optimized 2540 SETUP_SAVE_EVERYTHING_FRAME 2541 ldr x0, [sp, #FRAME_SIZE_SAVE_EVERYTHING] // pass ArtMethod 2542 mov x1, xSELF // pass Thread::Current 2543 CALL_SYMBOL artCompileOptimized // (ArtMethod*, Thread*) 2544 RESTORE_SAVE_EVERYTHING_FRAME 2545 // We don't need to restore the marking register here, as 2546 // artCompileOptimized doesn't allow thread suspension. 2547 ret 2548END art_quick_compile_optimized 2549 2550ENTRY art_quick_record_entry_trace_event 2551 ldr xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2552 // xIP0 has the trace buffer pointer. This is loaded on the fast path before 2553 // checking if we need to call this method. This will be still valid here. 2554 cmp xIP1, xIP0 2555 bhs .Lupdate_entry 2556 mov xIP1, #TRACE_BUFFER_SIZE 2557 add xIP1, xIP0, xIP1 2558.Lupdate_entry: 2559 sub xIP1, xIP1, 8 2560 str x0, [xIP1] 2561 str xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2562 ret 2563END art_quick_record_entry_trace_event 2564 2565ENTRY art_quick_record_exit_trace_event 2566 ldr xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2567 // xIP0 has the trace buffer pointer. This is loaded on the fast path before 2568 // checking if we need to call this method. This will be still valid here. 2569 cmp xIP1, xIP0 2570 bhs .Lupdate_entry_exit 2571 mov xIP1, #TRACE_BUFFER_SIZE 2572 add xIP1, xIP0, xIP1 2573.Lupdate_entry_exit: 2574 sub xIP1, xIP1, 8 2575 mov xIP0, #1 2576 str xIP0, [xIP1] 2577 str xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2578 ret 2579END art_quick_record_exit_trace_event 2580 2581ENTRY art_quick_record_long_running_entry_trace_event 2582 ldr xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2583 sub xIP1, xIP1, 16 2584 // xIP0 has the trace buffer pointer. This is loaded on the fast path before 2585 // checking if we need to call this method. This will be still valid here. 2586 cmp xIP0, xIP1 2587 bhi .Lhandle_overflow 2588 // Store the method pointer 2589 str x0, [xIP1, 8] 2590 // Store the timestamp with the last bit set to 0 to indicate method entry event 2591 mrs xIP0, cntvct_el0 2592 bfc xIP0, 0, 1 2593 str xIP0, [xIP1] 2594 str xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2595 ret 2596.Lhandle_overflow: 2597 // Call runtime to flush buffer. We expect the frequency of this case to be low. For ANR it 2598 // might be interesting to record the past data instead of overwriting with new entries. 2599 SETUP_SAVE_EVERYTHING_FRAME 2600 ldr x0, [sp, #FRAME_SIZE_SAVE_EVERYTHING] // pass ArtMethod* 2601 mov x1, xSELF // pass Thread::Current 2602 mov x2, 1 // set to true for entry events 2603 bl artRecordLongRunningMethodTraceEvent // (ArtMethod*, Thread*, is_entry) 2604 RESTORE_SAVE_EVERYTHING_FRAME // Note: will restore xSELF 2605 REFRESH_MARKING_REGISTER 2606 ret 2607END art_quick_record_long_running_entry_trace_event 2608 2609ENTRY art_quick_record_long_running_exit_trace_event 2610 ldr xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2611 // xIP0 has the trace buffer pointer. This is loaded on the fast path before checking if we 2612 // need to call this method. This will be still valid here. 2613 // We just need one entry for the method exit. We only record the timestamp. Method will be 2614 // available from the corresponding method entry event. 2615 cmp xIP0, xIP1 2616 bhs .Lhandle_overflow_exit 2617 // Get the timestamp of the method exit event. 2618 mrs xIP0, cntvct_el0 2619 // Load the previous event. 2620 ldr xIP1, [xIP1] 2621 // If lsb is set for the previous event, then the previous event is an exit event. This means 2622 // there was a long running method earlier on the call stack. Record the exit event for this 2623 // method to construct the call chain. 2624 tst xIP1, 1 2625 bne .Lrecord_exit_event 2626 // The previous event was an entry event. This exit corresponds to the previous method entry. 2627 // Check if the method is long running by getting the current timestamp and comparing with the 2628 // previous event's timestamp. 2629 mrs xIP0, cntvct_el0 2630 sub xIP1, xIP0, xIP1 2631 cmp xIP1, #LONG_RUNNING_METHOD_THRESHOLD 2632 bhs .Lrecord_exit_event 2633 // This wasn't a long running method. Erase the previously recorded method entry event. We 2634 // don't need to record entry / exit for this method. 2635 ldr xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2636 add xIP1, xIP1, 16 2637 str xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2638 ret 2639.Lrecord_exit_event: 2640 // For method exits we only record the current timestamp. We can infer the method from the 2641 // corresponding method entry event. 2642 ldr xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2643 sub xIP1, xIP1, 8 2644 // Set the lsb of the timestamp to 1 to indicate a method exit event. 2645 orr xIP0, xIP0, #1 2646 str xIP0, [xIP1] 2647 str xIP1, [xSELF, #TRACE_BUFFER_CURRENT_OFFSET] 2648 ret 2649.Lhandle_overflow_exit: 2650 // Call runtime to flush buffer. We expect the frequency of this case to be low. For ANR it 2651 // might be interesting to record the past data instead of overwriting with new entries. 2652 SETUP_SAVE_EVERYTHING_FRAME 2653 ldr xzr, [sp, #FRAME_SIZE_SAVE_EVERYTHING] // pass nullptr. ArtMethod* isn't required for exits 2654 mov x1, xSELF // pass Thread::Current 2655 mov x2, 0 // set to false for exit events 2656 bl artRecordLongRunningMethodTraceEvent // (ArtMethod*, Thread*, is_entry) 2657 RESTORE_SAVE_EVERYTHING_FRAME // Note: will restore xSELF 2658 REFRESH_MARKING_REGISTER 2659 ret 2660END art_quick_record_long_running_exit_trace_event 2661 2662 .extern artMethodEntryHook 2663ENTRY art_quick_method_entry_hook 2664 SETUP_SAVE_EVERYTHING_FRAME 2665 2666 ldr x0, [sp, #FRAME_SIZE_SAVE_EVERYTHING] // pass ArtMethod* 2667 mov x1, xSELF // pass Thread::Current 2668 mov x2, sp // pass SP 2669 CALL_SYMBOL artMethodEntryHook // (ArtMethod*, Thread*, SP) 2670 2671 CFI_REMEMBER_STATE 2672 cbnz x0, .Lentryhook_deopt 2673 2674 // Normal return. 2675 RESTORE_SAVE_EVERYTHING_FRAME // Note: will restore xSELF 2676 REFRESH_MARKING_REGISTER 2677 ret 2678 2679.Lentryhook_deopt: 2680 // Deoptimize. 2681 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 2682 CALL_SYMBOL art_quick_do_long_jump // (Context*) 2683 brk 0 // Unreached 2684END art_quick_method_entry_hook 2685 2686 .extern artMethodExitHook 2687ENTRY art_quick_method_exit_hook 2688 SETUP_SAVE_EVERYTHING_FRAME 2689 2690 // frame_size is passed from JITed code in x4 2691 add x3, sp, #16 // floating-point result ptr in kSaveEverything frame 2692 add x2, sp, #272 // integer result ptr in kSaveEverything frame 2693 add x1, sp, #FRAME_SIZE_SAVE_EVERYTHING // ArtMethod** 2694 mov x0, xSELF // Thread::Current 2695 CALL_SYMBOL artMethodExitHook // (Thread*, ArtMethod**, gpr_res*, fpr_res*, 2696 // frame_size) 2697 2698 CFI_REMEMBER_STATE 2699 cbnz x0, .Lexithook_deopt_or_exception 2700 2701 // Normal return. 2702 RESTORE_SAVE_EVERYTHING_FRAME 2703 REFRESH_MARKING_REGISTER 2704 ret 2705 2706.Lexithook_deopt_or_exception: 2707 // Deoptimize or exception thrown. 2708 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 2709 CALL_SYMBOL art_quick_do_long_jump // (Context*) 2710 brk 0 // Unreached 2711END art_quick_method_exit_hook 2712