1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "asm_support_arm.S" 18#include "interpreter/cfi_asm_support.h" 19 20#include "arch/quick_alloc_entrypoints.S" 21 22 /* Deliver the given exception */ 23 .extern artDeliverExceptionFromCode 24 /* Deliver an exception pending on a thread */ 25 .extern artDeliverPendingException 26 27.macro SETUP_SAVE_REFS_AND_ARGS_FRAME rTemp 28 SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY 29 LOAD_RUNTIME_INSTANCE \rTemp @ Load Runtime::Current into rTemp. 30 @ Load kSaveRefsAndArgs Method* into rTemp. 31 ldr \rTemp, [\rTemp, #RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET] 32 str \rTemp, [sp, #0] @ Place Method* at bottom of stack. 33 str sp, [rSELF, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. 34.endm 35 36.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_R0 37 SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY 38 str r0, [sp, #0] @ Store ArtMethod* to bottom of stack. 39 str sp, [rSELF, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. 40.endm 41 42 /* 43 * Macro that sets up the callee save frame to conform with 44 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 45 * when core registers are already saved. 46 */ 47.macro SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED rTemp, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 48 @ 14 words of callee saves and args already saved. 49 vpush {d0-d15} @ 32 words, 2 for each of the 16 saved doubles. 50 .cfi_adjust_cfa_offset 128 51 sub sp, #8 @ 2 words of space, alignment padding and Method* 52 .cfi_adjust_cfa_offset 8 53 LOAD_RUNTIME_INSTANCE \rTemp @ Load Runtime::Current into rTemp. 54 @ Load kSaveEverything Method* into rTemp. 55 ldr \rTemp, [\rTemp, #\runtime_method_offset] 56 str \rTemp, [sp, #0] @ Place Method* at bottom of stack. 57 str sp, [rSELF, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. 58 59 // Ugly compile-time check, but we only have the preprocessor. 60#if (FRAME_SIZE_SAVE_EVERYTHING != 56 + 128 + 8) 61#error "FRAME_SIZE_SAVE_EVERYTHING(ARM) size not as expected." 62#endif 63.endm 64 65 /* 66 * Macro that sets up the callee save frame to conform with 67 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 68 */ 69.macro SETUP_SAVE_EVERYTHING_FRAME rTemp, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 70 push {r0-r12, lr} @ 14 words of callee saves and args. 71 .cfi_adjust_cfa_offset 56 72 .cfi_rel_offset r0, 0 73 .cfi_rel_offset r1, 4 74 .cfi_rel_offset r2, 8 75 .cfi_rel_offset r3, 12 76 .cfi_rel_offset r4, 16 77 .cfi_rel_offset r5, 20 78 .cfi_rel_offset r6, 24 79 .cfi_rel_offset r7, 28 80 .cfi_rel_offset r8, 32 81 .cfi_rel_offset r9, 36 82 .cfi_rel_offset r10, 40 83 .cfi_rel_offset r11, 44 84 .cfi_rel_offset ip, 48 85 .cfi_rel_offset lr, 52 86 SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED \rTemp, \runtime_method_offset 87.endm 88 89.macro RESTORE_SAVE_EVERYTHING_FRAME 90 add sp, #8 @ rewind sp 91 .cfi_adjust_cfa_offset -8 92 vpop {d0-d15} 93 .cfi_adjust_cfa_offset -128 94 pop {r0-r12, lr} @ 14 words of callee saves 95 .cfi_restore r0 96 .cfi_restore r1 97 .cfi_restore r2 98 .cfi_restore r3 99 .cfi_restore r4 100 .cfi_restore r5 101 .cfi_restore r6 102 .cfi_restore r7 103 .cfi_restore r8 104 .cfi_restore r9 105 .cfi_restore r10 106 .cfi_restore r11 107 .cfi_restore r12 108 .cfi_restore lr 109 .cfi_adjust_cfa_offset -56 110.endm 111 112.macro RESTORE_SAVE_EVERYTHING_FRAME_KEEP_R0 113 add sp, #8 @ rewind sp 114 .cfi_adjust_cfa_offset -8 115 vpop {d0-d15} 116 .cfi_adjust_cfa_offset -128 117 add sp, #4 @ skip r0 118 .cfi_adjust_cfa_offset -4 119 .cfi_restore r0 @ debugger can no longer restore caller's r0 120 pop {r1-r12, lr} @ 13 words of callee saves 121 .cfi_restore r1 122 .cfi_restore r2 123 .cfi_restore r3 124 .cfi_restore r4 125 .cfi_restore r5 126 .cfi_restore r6 127 .cfi_restore r7 128 .cfi_restore r8 129 .cfi_restore r9 130 .cfi_restore r10 131 .cfi_restore r11 132 .cfi_restore r12 133 .cfi_restore lr 134 .cfi_adjust_cfa_offset -52 135.endm 136 137.macro RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 138 ldr r1, [rSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 139 cbnz r1, 1f 140 DEOPT_OR_RETURN r1 // Check if deopt is required 1411: 142 DELIVER_PENDING_EXCEPTION 143.endm 144 145.macro DEOPT_OR_RETURN temp, is_ref = 0 146 ldr \temp, [rSELF, #THREAD_DEOPT_CHECK_REQUIRED_OFFSET] 147 cbnz \temp, 2f 148 bx lr 1492: 150 SETUP_SAVE_EVERYTHING_FRAME \temp 151 mov r2, \is_ref // pass if result is a reference 152 mov r1, r0 // pass the result 153 mov r0, rSELF // Thread::Current 154 bl artDeoptimizeIfNeeded 155 CFI_REMEMBER_STATE 156 RESTORE_SAVE_EVERYTHING_FRAME 157 REFRESH_MARKING_REGISTER 158 bx lr 159 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 160.endm 161 162.macro DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_R0 temp, is_ref 163 ldr \temp, [rSELF, #THREAD_DEOPT_CHECK_REQUIRED_OFFSET] 164 cbnz \temp, 2f 165 CFI_REMEMBER_STATE 166 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_R0 167 REFRESH_MARKING_REGISTER 168 bx lr 169 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 1702: 171 str r0, [sp, SAVE_EVERYTHING_FRAME_R0_OFFSET] // update result in the frame 172 mov r2, \is_ref // pass if result is a reference 173 mov r1, r0 // pass the result 174 mov r0, rSELF // Thread::Current 175 bl artDeoptimizeIfNeeded 176 CFI_REMEMBER_STATE 177 RESTORE_SAVE_EVERYTHING_FRAME 178 REFRESH_MARKING_REGISTER 179 bx lr 180 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_EVERYTHING 181.endm 182 183.macro NO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 184 .extern \cxx_name 185ENTRY \c_name 186 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r0 @ save all registers as basis for long jump context 187 mov r0, rSELF @ pass Thread::Current 188 bl \cxx_name @ \cxx_name(Thread*) 189END \c_name 190.endm 191 192.macro NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name 193 .extern \cxx_name 194ENTRY \c_name 195 SETUP_SAVE_EVERYTHING_FRAME r0 @ save all registers as basis for long jump context 196 mov r0, rSELF @ pass Thread::Current 197 bl \cxx_name @ \cxx_name(Thread*) 198END \c_name 199.endm 200 201.macro ONE_ARG_RUNTIME_EXCEPTION c_name, cxx_name 202 .extern \cxx_name 203ENTRY \c_name 204 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r1 @ save all registers as basis for long jump context 205 mov r1, rSELF @ pass Thread::Current 206 bl \cxx_name @ \cxx_name(Thread*) 207END \c_name 208.endm 209 210.macro TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name 211 .extern \cxx_name 212ENTRY \c_name 213 SETUP_SAVE_EVERYTHING_FRAME r2 @ save all registers as basis for long jump context 214 mov r2, rSELF @ pass Thread::Current 215 bl \cxx_name @ \cxx_name(Thread*) 216END \c_name 217.endm 218 219.macro RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 220 cbnz r0, 1f @ result non-zero branch over 221 DEOPT_OR_RETURN r1 2221: 223 DELIVER_PENDING_EXCEPTION 224.endm 225 226.macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER 227 cbz r0, 1f @ result zero branch over 228 DEOPT_OR_RETURN r1, /*is_ref=*/1 2291: 230 DELIVER_PENDING_EXCEPTION 231.endm 232 233// Macros taking opportunity of code similarities for downcalls. 234.macro ONE_ARG_REF_DOWNCALL name, entrypoint, return 235 .extern \entrypoint 236ENTRY \name 237 SETUP_SAVE_REFS_ONLY_FRAME r1 @ save callee saves in case of GC 238 mov r1, rSELF @ pass Thread::Current 239 bl \entrypoint @ (uint32_t field_idx, Thread*) 240 RESTORE_SAVE_REFS_ONLY_FRAME 241 REFRESH_MARKING_REGISTER 242 \return 243END \name 244.endm 245 246.macro TWO_ARG_REF_DOWNCALL name, entrypoint, return 247 .extern \entrypoint 248ENTRY \name 249 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 250 mov r2, rSELF @ pass Thread::Current 251 bl \entrypoint @ (field_idx, Object*, Thread*) 252 RESTORE_SAVE_REFS_ONLY_FRAME 253 REFRESH_MARKING_REGISTER 254 \return 255END \name 256.endm 257 258.macro THREE_ARG_REF_DOWNCALL name, entrypoint, return 259 .extern \entrypoint 260ENTRY \name 261 SETUP_SAVE_REFS_ONLY_FRAME r3 @ save callee saves in case of GC 262 mov r3, rSELF @ pass Thread::Current 263 bl \entrypoint @ (field_idx, Object*, new_val, Thread*) 264 RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here 265 REFRESH_MARKING_REGISTER 266 \return 267END \name 268.endm 269 270 /* 271 * Called by managed code, saves callee saves and then calls artThrowException 272 * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception. 273 */ 274ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode 275 276 /* 277 * Called by managed code to create and deliver a NullPointerException. 278 */ 279NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode 280 281 /* 282 * Call installed by a signal handler to create and deliver a NullPointerException. 283 */ 284 .extern art_quick_throw_null_pointer_exception_from_signal 285ENTRY art_quick_throw_null_pointer_exception_from_signal 286 // The fault handler pushes the gc map address, i.e. "return address", to stack 287 // and passes the fault address in LR. So we need to set up the CFI info accordingly. 288 .cfi_def_cfa_offset __SIZEOF_POINTER__ 289 .cfi_rel_offset lr, 0 290 push {r0-r12} @ 13 words of callee saves and args; LR already saved. 291 .cfi_adjust_cfa_offset 52 292 .cfi_rel_offset r0, 0 293 .cfi_rel_offset r1, 4 294 .cfi_rel_offset r2, 8 295 .cfi_rel_offset r3, 12 296 .cfi_rel_offset r4, 16 297 .cfi_rel_offset r5, 20 298 .cfi_rel_offset r6, 24 299 .cfi_rel_offset r7, 28 300 .cfi_rel_offset r8, 32 301 .cfi_rel_offset r9, 36 302 .cfi_rel_offset r10, 40 303 .cfi_rel_offset r11, 44 304 .cfi_rel_offset ip, 48 305 306 @ save all registers as basis for long jump context 307 SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED r1 308 mov r0, lr @ pass the fault address stored in LR by the fault handler. 309 mov r1, rSELF @ pass Thread::Current 310 bl artThrowNullPointerExceptionFromSignal @ (Thread*) 311END art_quick_throw_null_pointer_exception_from_signal 312 313 /* 314 * Called by managed code to create and deliver an ArithmeticException. 315 */ 316NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_div_zero, artThrowDivZeroFromCode 317 318 /* 319 * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds 320 * index, arg2 holds limit. 321 */ 322TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_array_bounds, artThrowArrayBoundsFromCode 323 324 /* 325 * Called by managed code to create and deliver a StringIndexOutOfBoundsException 326 * as if thrown from a call to String.charAt(). Arg1 holds index, arg2 holds limit. 327 */ 328TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_string_bounds, artThrowStringBoundsFromCode 329 330 /* 331 * Called by managed code to create and deliver a StackOverflowError. 332 */ 333NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode 334 335 /* 336 * All generated callsites for interface invokes and invocation slow paths will load arguments 337 * as usual - except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 338 * the method_idx. This wrapper will save arg1-arg3, and call the appropriate C helper. 339 * NOTE: "this" is first visible argument of the target, and so can be found in arg1/r1. 340 * 341 * The helper will attempt to locate the target and return a 64-bit result in r0/r1 consisting 342 * of the target Method* in r0 and method->code_ in r1. 343 * 344 * If unsuccessful, the helper will return null/null. There will bea pending exception in the 345 * thread and we branch to another stub to deliver it. 346 * 347 * On success this wrapper will restore arguments and *jump* to the target, leaving the lr 348 * pointing back to the original caller. 349 * 350 * Clobbers IP (R12). 351 */ 352.macro INVOKE_TRAMPOLINE_BODY cxx_name 353 .extern \cxx_name 354 SETUP_SAVE_REFS_AND_ARGS_FRAME r2 @ save callee saves in case allocation triggers GC 355 mov r2, rSELF @ pass Thread::Current 356 mov r3, sp 357 bl \cxx_name @ (method_idx, this, Thread*, SP) 358 mov r12, r1 @ save Method*->code_ 359 RESTORE_SAVE_REFS_AND_ARGS_FRAME 360 REFRESH_MARKING_REGISTER 361 cbz r0, 1f @ did we find the target? if not go to exception delivery 362 bx r12 @ tail call to target 3631: 364 DELIVER_PENDING_EXCEPTION 365.endm 366.macro INVOKE_TRAMPOLINE c_name, cxx_name 367ENTRY \c_name 368 INVOKE_TRAMPOLINE_BODY \cxx_name 369END \c_name 370.endm 371 372INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, artInvokeInterfaceTrampolineWithAccessCheck 373 374INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, artInvokeStaticTrampolineWithAccessCheck 375INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, artInvokeDirectTrampolineWithAccessCheck 376INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, artInvokeSuperTrampolineWithAccessCheck 377INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, artInvokeVirtualTrampolineWithAccessCheck 378 379 /* 380 * Quick invocation stub internal. 381 * On entry: 382 * r0 = method pointer 383 * r1 = argument array or null for no argument methods 384 * r2 = size of argument array in bytes 385 * r3 = (managed) thread pointer 386 * [sp] = JValue* result 387 * [sp + 4] = result_in_float 388 * [sp + 8] = core register argument array 389 * [sp + 12] = fp register argument array 390 * +-------------------------+ 391 * | uint32_t* fp_reg_args | 392 * | uint32_t* core_reg_args | 393 * | result_in_float | <- Caller frame 394 * | Jvalue* result | 395 * +-------------------------+ 396 * | lr | 397 * | r11 | 398 * | r9 | 399 * | r4 | <- r11 400 * +-------------------------+ 401 * | uint32_t out[n-1] | 402 * | : : | Outs 403 * | uint32_t out[0] | 404 * | StackRef<ArtMethod> | <- SP value=null 405 * +-------------------------+ 406 */ 407ENTRY art_quick_invoke_stub_internal 408 SPILL_ALL_CALLEE_SAVE_GPRS @ spill regs (9) 409 mov r11, sp @ save the stack pointer 410 .cfi_def_cfa_register r11 411 412 mov r9, r3 @ move managed thread pointer into r9 413 414 add r4, r2, #4 @ create space for method pointer in frame 415 sub r4, sp, r4 @ reserve & align *stack* to 16 bytes: native calling 416 and r4, #0xFFFFFFF0 @ convention only aligns to 8B, so we have to ensure ART 417 mov sp, r4 @ 16B alignment ourselves. 418 419 mov r4, r0 @ save method* 420 add r0, sp, #4 @ pass stack pointer + method ptr as dest for memcpy 421 bl memcpy @ memcpy (dest, src, bytes) 422 mov ip, #0 @ set ip to 0 423 str ip, [sp] @ store null for method* at bottom of frame 424 425 ldr ip, [r11, #48] @ load fp register argument array pointer 426 vldm ip, {s0-s15} @ copy s0 - s15 427 428 ldr ip, [r11, #44] @ load core register argument array pointer 429 mov r0, r4 @ restore method* 430 add ip, ip, #4 @ skip r0 431 ldm ip, {r1-r3} @ copy r1 - r3 432 433 REFRESH_MARKING_REGISTER 434 435 ldr ip, [r0, #ART_METHOD_QUICK_CODE_OFFSET_32] @ get pointer to the code 436 blx ip @ call the method 437 438 mov sp, r11 @ restore the stack pointer 439 .cfi_def_cfa_register sp 440 441 ldr r4, [sp, #40] @ load result_is_float 442 ldr r9, [sp, #36] @ load the result pointer 443 cmp r4, #0 444 ite eq 445 strdeq r0, [r9] @ store r0/r1 into result pointer 446 vstrne d0, [r9] @ store s0-s1/d0 into result pointer 447 448 pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} @ restore spill regs 449END art_quick_invoke_stub_internal 450 451 /* 452 * On stack replacement stub. 453 * On entry: 454 * r0 = stack to copy 455 * r1 = size of stack 456 * r2 = pc to call 457 * r3 = JValue* result 458 * [sp] = shorty 459 * [sp + 4] = thread 460 */ 461ENTRY art_quick_osr_stub 462 SPILL_ALL_CALLEE_SAVE_GPRS @ Spill regs (9) 463 vpush {s16-s31} @ Spill fp-regs (16) 464 .cfi_adjust_cfa_offset 64 465 SAVE_SIZE=(9*4+16*4) 466 mov r11, sp @ Save the stack pointer 467 .cfi_def_cfa r11, SAVE_SIZE @ CFA = r11 + SAVE_SIZE 468 CFI_REMEMBER_STATE 469 mov r10, r1 @ Save size of stack 470 ldr r9, [r11, #(SAVE_SIZE+4)] @ Move managed thread pointer into r9 471 REFRESH_MARKING_REGISTER 472 mov r6, r2 @ Save the pc to call 473 sub r7, sp, #12 @ Reserve space for stack pointer, 474 @ JValue* result, and ArtMethod* slot. 475 and r7, #0xFFFFFFF0 @ Align stack pointer 476 mov sp, r7 @ Update stack pointer 477 str r11, [sp, #4] @ Save old stack pointer 478 str r3, [sp, #8] @ Save JValue* result 479 mov ip, #0 480 str ip, [sp] @ Store null for ArtMethod* at bottom of frame 481 // r11 isn't properly spilled in the osr method, so we need use DWARF expression. 482 // NB: the CFI must be before the call since this is the address gdb will lookup. 483 // NB: gdb expects that cfa_expression returns the CFA value (not address to it). 484 .cfi_escape /* CFA = [sp + 4] + SAVE_SIZE */ \ 485 0x0f, 6, /* DW_CFA_def_cfa_expression(len) */ \ 486 0x92, 13, 4, /* DW_OP_bregx(reg,offset) */ \ 487 0x06, /* DW_OP_deref */ \ 488 0x23, SAVE_SIZE /* DW_OP_plus_uconst(val) */ 489 bl .Losr_entry @ Call the method 490 ldr r10, [sp, #8] @ Restore JValue* result 491 ldr sp, [sp, #4] @ Restore saved stack pointer 492 .cfi_def_cfa sp, SAVE_SIZE @ CFA = sp + SAVE_SIZE 493 strd r0, [r10] @ Store r0/r1 into result pointer 494 vpop {s16-s31} 495 .cfi_adjust_cfa_offset -64 496 pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} 497.Losr_entry: 498 CFI_RESTORE_STATE_AND_DEF_CFA r11, SAVE_SIZE @ CFA = r11 + SAVE_SIZE 499 sub sp, sp, r10 @ Reserve space for callee stack 500 sub r10, r10, #4 501 str lr, [sp, r10] @ Store link register per the compiler ABI 502 mov r2, r10 503 mov r1, r0 504 mov r0, sp 505 bl memcpy @ memcpy (dest r0, src r1, bytes r2) 506 bx r6 507END art_quick_osr_stub 508 509 /* 510 * On entry r0 is uint32_t* gprs_ and r1 is uint32_t* fprs_. 511 * Both must reside on the stack, between current SP and target SP. 512 * The r12 (IP) shall be clobbered rather than retrieved from gprs_. 513 */ 514ARM_ENTRY art_quick_do_long_jump 515 vldm r1, {s0-s31} @ Load all fprs from argument fprs_. 516 mov sp, r0 @ Make SP point to gprs_. 517 @ Do not access fprs_ from now, they may be below SP. 518 ldm sp, {r0-r11} @ load r0-r11 from gprs_. 519 ldr r12, [sp, #60] @ Load the value of PC (r15) from gprs_ (60 = 4 * 15) into IP (r12). 520 ldr lr, [sp, #56] @ Load LR from gprs_, 56 = 4 * 14. 521 ldr sp, [sp, #52] @ Load SP from gprs_ 52 = 4 * 13. 522 @ Do not access gprs_ from now, they are below SP. 523 REFRESH_MARKING_REGISTER 524 bx r12 @ Do long jump. 525END art_quick_do_long_jump 526 527 /* 528 * Entry from managed code that calls artHandleFillArrayDataFromCode and delivers exception on 529 * failure. 530 */ 531TWO_ARG_REF_DOWNCALL art_quick_handle_fill_data, artHandleFillArrayDataFromCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 532 533 /* 534 * Entry from managed code that tries to lock the object in a fast path and 535 * calls `artLockObjectFromCode()` for the difficult cases, may block for GC. 536 * r0 holds the possibly null object to lock. 537 */ 538ENTRY art_quick_lock_object 539 // Note: the slow path is actually the art_quick_lock_object_no_inline (tail call). 540 LOCK_OBJECT_FAST_PATH r0, r1, r2, r3, .Llock_object_slow, /*can_be_null*/ 1 541END art_quick_lock_object 542 543 /* 544 * Entry from managed code that calls `artLockObjectFromCode()`, may block for GC. 545 * r0 holds the possibly null object to lock. 546 */ 547 .extern artLockObjectFromCode 548ENTRY art_quick_lock_object_no_inline 549 // This is also the slow path for art_quick_lock_object. 550 // Note that we need a local label as the assembler emits bad instructions 551 // for CBZ/CBNZ if we try to jump to `art_quick_lock_object_no_inline`. 552.Llock_object_slow: 553 SETUP_SAVE_REFS_ONLY_FRAME r1 @ save callee saves in case we block 554 mov r1, rSELF @ pass Thread::Current 555 bl artLockObjectFromCode @ (Object* obj, Thread*) 556 RESTORE_SAVE_REFS_ONLY_FRAME 557 REFRESH_MARKING_REGISTER 558 RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 559END art_quick_lock_object_no_inline 560 561 /* 562 * Entry from managed code that tries to unlock the object in a fast path and calls 563 * `artUnlockObjectFromCode()` for the difficult cases and delivers exception on failure. 564 * r0 holds the possibly null object to unlock. 565 */ 566ENTRY art_quick_unlock_object 567 // Note: the slow path is actually the art_quick_unlock_object_no_inline (tail call). 568 UNLOCK_OBJECT_FAST_PATH r0, r1, r2, r3, .Lunlock_object_slow, /*can_be_null*/ 1 569END art_quick_unlock_object 570 571 /* 572 * Entry from managed code that calls `artUnlockObjectFromCode()` 573 * and delivers exception on failure. 574 * r0 holds the possibly null object to unlock. 575 */ 576 .extern artUnlockObjectFromCode 577ENTRY art_quick_unlock_object_no_inline 578 // This is also the slow path for art_quick_unlock_object. 579 // Note that we need a local label as the assembler emits bad instructions 580 // for CBZ/CBNZ if we try to jump to `art_quick_unlock_object_no_inline`. 581.Lunlock_object_slow: 582 @ save callee saves in case exception allocation triggers GC 583 SETUP_SAVE_REFS_ONLY_FRAME r1 584 mov r1, rSELF @ pass Thread::Current 585 bl artUnlockObjectFromCode @ (Object* obj, Thread*) 586 RESTORE_SAVE_REFS_ONLY_FRAME 587 REFRESH_MARKING_REGISTER 588 RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 589END art_quick_unlock_object_no_inline 590 591 /* 592 * Entry from managed code that calls artInstanceOfFromCode and on failure calls 593 * artThrowClassCastExceptionForObject. 594 */ 595 .extern artInstanceOfFromCode 596 .extern artThrowClassCastExceptionForObject 597ENTRY art_quick_check_instance_of 598 // Type check using the bit string passes null as the target class. In that case just throw. 599 cbz r1, .Lthrow_class_cast_exception_for_bitstring_check 600 601 push {r0-r2, lr} @ save arguments, padding (r2) and link register 602 .cfi_adjust_cfa_offset 16 603 .cfi_rel_offset r0, 0 604 .cfi_rel_offset r1, 4 605 .cfi_rel_offset r2, 8 606 .cfi_rel_offset lr, 12 607 bl artInstanceOfFromCode 608 cbz r0, .Lthrow_class_cast_exception 609 pop {r0-r2, pc} 610 611.Lthrow_class_cast_exception: 612 pop {r0-r2, lr} 613 .cfi_adjust_cfa_offset -16 614 .cfi_restore r0 615 .cfi_restore r1 616 .cfi_restore r2 617 .cfi_restore lr 618 619.Lthrow_class_cast_exception_for_bitstring_check: 620 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r2 @ save all registers as basis for long jump context 621 mov r2, rSELF @ pass Thread::Current 622 bl artThrowClassCastExceptionForObject @ (Object*, Class*, Thread*) 623 bkpt 624END art_quick_check_instance_of 625 626// Restore rReg's value from [sp, #offset] if rReg is not the same as rExclude. 627.macro POP_REG_NE rReg, offset, rExclude 628 .ifnc \rReg, \rExclude 629 ldr \rReg, [sp, #\offset] @ restore rReg 630 .cfi_restore \rReg 631 .endif 632.endm 633 634// Save rReg's value to [sp, #offset]. 635.macro PUSH_REG rReg, offset 636 str \rReg, [sp, #\offset] @ save rReg 637 .cfi_rel_offset \rReg, \offset 638.endm 639 640 // Helper macros for `art_quick_aput_obj`. 641#ifdef USE_READ_BARRIER 642#ifdef USE_BAKER_READ_BARRIER 643.macro BAKER_RB_CHECK_GRAY_BIT_AND_LOAD rDest, rObj, offset, gray_slow_path_label 644 ldr ip, [\rObj, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 645 tst ip, #LOCK_WORD_READ_BARRIER_STATE_MASK_SHIFTED 646 bne \gray_slow_path_label 647 // False dependency to avoid needing load/load fence. 648 add \rObj, \rObj, ip, lsr #32 649 ldr \rDest, [\rObj, #\offset] 650 UNPOISON_HEAP_REF \rDest 651.endm 652 653.macro BAKER_RB_LOAD_AND_MARK rDest, rObj, offset, mark_function 654 ldr \rDest, [\rObj, #\offset] 655 UNPOISON_HEAP_REF \rDest 656 str lr, [sp, #-8]! @ Save LR with correct stack alignment. 657 .cfi_rel_offset lr, 0 658 .cfi_adjust_cfa_offset 8 659 bl \mark_function 660 ldr lr, [sp], #8 @ Restore LR. 661 .cfi_restore lr 662 .cfi_adjust_cfa_offset -8 663.endm 664#else // USE_BAKER_READ_BARRIER 665 .extern artReadBarrierSlow 666.macro READ_BARRIER_SLOW rDest, rObj, offset 667 push {r0-r3, ip, lr} @ 6 words for saved registers (used in art_quick_aput_obj) 668 .cfi_adjust_cfa_offset 24 669 .cfi_rel_offset r0, 0 670 .cfi_rel_offset r1, 4 671 .cfi_rel_offset r2, 8 672 .cfi_rel_offset r3, 12 673 .cfi_rel_offset ip, 16 674 .cfi_rel_offset lr, 20 675 sub sp, #8 @ push padding 676 .cfi_adjust_cfa_offset 8 677 @ mov r0, \rRef @ pass ref in r0 (no-op for now since parameter ref is unused) 678 .ifnc \rObj, r1 679 mov r1, \rObj @ pass rObj 680 .endif 681 mov r2, #\offset @ pass offset 682 bl artReadBarrierSlow @ artReadBarrierSlow(ref, rObj, offset) 683 @ No need to unpoison return value in r0, artReadBarrierSlow() would do the unpoisoning. 684 .ifnc \rDest, r0 685 mov \rDest, r0 @ save return value in rDest 686 .endif 687 add sp, #8 @ pop padding 688 .cfi_adjust_cfa_offset -8 689 POP_REG_NE r0, 0, \rDest @ conditionally restore saved registers 690 POP_REG_NE r1, 4, \rDest 691 POP_REG_NE r2, 8, \rDest 692 POP_REG_NE r3, 12, \rDest 693 POP_REG_NE ip, 16, \rDest 694 add sp, #20 695 .cfi_adjust_cfa_offset -20 696 pop {lr} @ restore lr 697 .cfi_adjust_cfa_offset -4 698 .cfi_restore lr 699.endm 700#endif // USE_BAKER_READ_BARRIER 701#endif // USE_READ_BARRIER 702 703 .hidden art_quick_aput_obj 704ENTRY art_quick_aput_obj 705#if defined(USE_READ_BARRIER) && !defined(USE_BAKER_READ_BARRIER) 706 @ The offset to .Ldo_aput_null is too large to use cbz due to expansion from `READ_BARRIER_SLOW`. 707 tst r2, r2 708 beq .Laput_obj_null 709 READ_BARRIER_SLOW r3, r0, MIRROR_OBJECT_CLASS_OFFSET 710 READ_BARRIER_SLOW r3, r3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET 711 READ_BARRIER_SLOW r4, r2, MIRROR_OBJECT_CLASS_OFFSET 712#else // !defined(USE_READ_BARRIER) || defined(USE_BAKER_READ_BARRIER) 713 cbz r2, .Laput_obj_null 714#ifdef USE_READ_BARRIER 715 cmp rMR, #0 716 bne .Laput_obj_gc_marking 717#endif // USE_READ_BARRIER 718 ldr r3, [r0, #MIRROR_OBJECT_CLASS_OFFSET] 719 UNPOISON_HEAP_REF r3 720 // R4 is a scratch register in managed ARM ABI. 721 ldr r4, [r2, #MIRROR_OBJECT_CLASS_OFFSET] 722 UNPOISON_HEAP_REF r4 723 ldr r3, [r3, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] 724 UNPOISON_HEAP_REF r3 725#endif // !defined(USE_READ_BARRIER) || defined(USE_BAKER_READ_BARRIER) 726 cmp r3, r4 @ value's type == array's component type - trivial assignability 727 bne .Laput_obj_check_assignability 728.Laput_obj_store: 729 add r3, r0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 730 POISON_HEAP_REF r2 731 str r2, [r3, r1, lsl #2] 732 ldr r3, [rSELF, #THREAD_CARD_TABLE_OFFSET] 733 lsr r0, r0, #CARD_TABLE_CARD_SHIFT 734 strb r3, [r3, r0] 735 blx lr 736 737.Laput_obj_null: 738 add r3, r0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 739 str r2, [r3, r1, lsl #2] 740 blx lr 741 742.Laput_obj_check_assignability: 743 push {r0-r2, lr} @ save arguments 744 .cfi_adjust_cfa_offset 16 745 .cfi_rel_offset lr, 12 746 mov r1, r4 747 mov r0, r3 748 bl artIsAssignableFromCode 749 cbz r0, .Lthrow_array_store_exception 750 CFI_REMEMBER_STATE 751 pop {r0-r2, lr} 752 .cfi_restore lr 753 .cfi_adjust_cfa_offset -16 754 add r3, r0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 755 POISON_HEAP_REF r2 756 str r2, [r3, r1, lsl #2] 757 ldr r3, [rSELF, #THREAD_CARD_TABLE_OFFSET] 758 lsr r0, r0, #CARD_TABLE_CARD_SHIFT 759 strb r3, [r3, r0] 760 blx lr 761 762.Lthrow_array_store_exception: 763 CFI_RESTORE_STATE_AND_DEF_CFA sp, 16 764 pop {r0-r2, lr} 765 .cfi_restore lr 766 .cfi_adjust_cfa_offset -16 767#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 768 CFI_REMEMBER_STATE 769#endif // defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 770 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r3 771 mov r1, r2 772 mov r2, rSELF @ Pass Thread::Current. 773 bl artThrowArrayStoreException @ (Class*, Class*, Thread*) 774 bkpt @ Unreachable. 775 776#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 777 CFI_RESTORE_STATE_AND_DEF_CFA sp, 0 778.Laput_obj_gc_marking: 779 BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ 780 r3, r0, MIRROR_OBJECT_CLASS_OFFSET, .Laput_obj_mark_array_class 781.Laput_obj_mark_array_class_continue: 782 BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ 783 r3, r3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, .Laput_obj_mark_array_element 784.Laput_obj_mark_array_element_continue: 785 BAKER_RB_CHECK_GRAY_BIT_AND_LOAD \ 786 r4, r2, MIRROR_OBJECT_CLASS_OFFSET, .Laput_obj_mark_object_class 787.Laput_obj_mark_object_class_continue: 788 789 cmp r3, r4 @ value's type == array's component type - trivial assignability 790 // All registers are set up for correctly `.Laput_obj_check_assignability`. 791 bne .Laput_obj_check_assignability 792 b .Laput_obj_store 793 794.Laput_obj_mark_array_class: 795 BAKER_RB_LOAD_AND_MARK r3, r0, MIRROR_OBJECT_CLASS_OFFSET, art_quick_read_barrier_mark_reg03 796 b .Laput_obj_mark_array_class_continue 797 798.Laput_obj_mark_array_element: 799 BAKER_RB_LOAD_AND_MARK \ 800 r3, r3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, art_quick_read_barrier_mark_reg03 801 b .Laput_obj_mark_array_element_continue 802 803.Laput_obj_mark_object_class: 804 BAKER_RB_LOAD_AND_MARK r4, r2, MIRROR_OBJECT_CLASS_OFFSET, art_quick_read_barrier_mark_reg04 805 b .Laput_obj_mark_object_class_continue 806#endif // defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 807END art_quick_aput_obj 808 809// Macro to facilitate adding new allocation entrypoints. 810.macro ONE_ARG_DOWNCALL name, entrypoint, return 811 .extern \entrypoint 812ENTRY \name 813 SETUP_SAVE_REFS_ONLY_FRAME r1 @ save callee saves in case of GC 814 mov r1, rSELF @ pass Thread::Current 815 bl \entrypoint @ (uint32_t type_idx, Method* method, Thread*) 816 RESTORE_SAVE_REFS_ONLY_FRAME 817 REFRESH_MARKING_REGISTER 818 \return 819END \name 820.endm 821 822// Macro to facilitate adding new allocation entrypoints. 823.macro TWO_ARG_DOWNCALL name, entrypoint, return 824 .extern \entrypoint 825ENTRY \name 826 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 827 mov r2, rSELF @ pass Thread::Current 828 bl \entrypoint @ (uint32_t type_idx, Method* method, Thread*) 829 RESTORE_SAVE_REFS_ONLY_FRAME 830 REFRESH_MARKING_REGISTER 831 \return 832END \name 833.endm 834 835// Macro to facilitate adding new array allocation entrypoints. 836.macro THREE_ARG_DOWNCALL name, entrypoint, return 837 .extern \entrypoint 838ENTRY \name 839 SETUP_SAVE_REFS_ONLY_FRAME r3 @ save callee saves in case of GC 840 mov r3, rSELF @ pass Thread::Current 841 @ (uint32_t type_idx, Method* method, int32_t component_count, Thread*) 842 bl \entrypoint 843 RESTORE_SAVE_REFS_ONLY_FRAME 844 REFRESH_MARKING_REGISTER 845 \return 846END \name 847.endm 848 849// Macro to facilitate adding new allocation entrypoints. 850.macro FOUR_ARG_DOWNCALL name, entrypoint, return 851 .extern \entrypoint 852ENTRY \name 853 SETUP_SAVE_REFS_ONLY_FRAME r12 @ save callee saves in case of GC 854 str rSELF, [sp, #-16]! @ expand the frame and pass Thread::Current 855 .cfi_adjust_cfa_offset 16 856 bl \entrypoint 857 add sp, #16 @ strip the extra frame 858 .cfi_adjust_cfa_offset -16 859 RESTORE_SAVE_REFS_ONLY_FRAME 860 REFRESH_MARKING_REGISTER 861 \return 862END \name 863.endm 864 865 /* 866 * Macro for resolution and initialization of indexed DEX file 867 * constants such as classes and strings. 868 */ 869.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET 870 .extern \entrypoint 871ENTRY \name 872 SETUP_SAVE_EVERYTHING_FRAME r1, \runtime_method_offset @ save everything in case of GC 873 mov r1, rSELF @ pass Thread::Current 874 bl \entrypoint @ (uint32_t index, Thread*) 875 cbz r0, 1f @ If result is null, deliver the OOME. 876 str r0, [sp, #136] @ store result in the frame 877 DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_R0 r1, /* is_ref= */ 1 8781: 879 DELIVER_PENDING_EXCEPTION_FRAME_READY 880END \name 881.endm 882 883.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT name, entrypoint 884 ONE_ARG_SAVE_EVERYTHING_DOWNCALL \name, \entrypoint, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET 885.endm 886 887ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode 888ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_resolve_type, artResolveTypeFromCode 889ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_type_and_verify_access, artResolveTypeAndVerifyAccessFromCode 890ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_handle, artResolveMethodHandleFromCode 891ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_type, artResolveMethodTypeFromCode 892ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode 893 894// Note: Functions `art{Get,Set}<Kind>{Static,Instance}FromCompiledCode` are 895// defined with a macro in runtime/entrypoints/quick/quick_field_entrypoints.cc. 896 897 /* 898 * Called by managed code to resolve a static field and load a non-wide value. 899 */ 900ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 901ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 902ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 903ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 904ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 905ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 906 /* 907 * Called by managed code to resolve a static field and load a 64-bit primitive value. 908 */ 909 .extern artGet64StaticFromCompiledCode 910ENTRY art_quick_get64_static 911 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 912 mov r1, rSELF @ pass Thread::Current 913 bl artGet64StaticFromCompiledCode @ (uint32_t field_idx, Thread*) 914 ldr r2, [rSELF, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ 915 RESTORE_SAVE_REFS_ONLY_FRAME 916 REFRESH_MARKING_REGISTER 917 cbnz r2, 1f @ success if no exception pending 918 DEOPT_OR_RETURN r2 @ check if deopt is required or return 9191: 920 DELIVER_PENDING_EXCEPTION 921END art_quick_get64_static 922 923 /* 924 * Called by managed code to resolve an instance field and load a non-wide value. 925 */ 926TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 927TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 928TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 929TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 930TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 931TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 932 /* 933 * Called by managed code to resolve an instance field and load a 64-bit primitive value. 934 */ 935 .extern artGet64InstanceFromCompiledCode 936ENTRY art_quick_get64_instance 937 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 938 mov r2, rSELF @ pass Thread::Current 939 bl artGet64InstanceFromCompiledCode @ (field_idx, Object*, Thread*) 940 ldr r2, [rSELF, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ 941 RESTORE_SAVE_REFS_ONLY_FRAME 942 REFRESH_MARKING_REGISTER 943 cbnz r2, 1f @ success if no exception pending 944 DEOPT_OR_RETURN r2 @ check if deopt is required or return 9451: 946 DELIVER_PENDING_EXCEPTION 947END art_quick_get64_instance 948 949 /* 950 * Called by managed code to resolve a static field and store a value. 951 */ 952TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 953TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 954TWO_ARG_REF_DOWNCALL art_quick_set32_static, artSet32StaticFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 955TWO_ARG_REF_DOWNCALL art_quick_set_obj_static, artSetObjStaticFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 956 957 /* 958 * Called by managed code to resolve an instance field and store a non-wide value. 959 */ 960THREE_ARG_REF_DOWNCALL art_quick_set8_instance, artSet8InstanceFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 961THREE_ARG_REF_DOWNCALL art_quick_set16_instance, artSet16InstanceFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 962THREE_ARG_REF_DOWNCALL art_quick_set32_instance, artSet32InstanceFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 963THREE_ARG_REF_DOWNCALL art_quick_set_obj_instance, artSetObjInstanceFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 964 965 /* 966 * Called by managed code to resolve an instance field and store a wide value. 967 */ 968 .extern artSet64InstanceFromCompiledCode 969ENTRY art_quick_set64_instance 970 SETUP_SAVE_REFS_ONLY_FRAME r12 @ save callee saves in case of GC 971 @ r2:r3 contain the wide argument 972 str rSELF, [sp, #-16]! @ expand the frame and pass Thread::Current 973 .cfi_adjust_cfa_offset 16 974 bl artSet64InstanceFromCompiledCode @ (field_idx, Object*, new_val, Thread*) 975 add sp, #16 @ release out args 976 .cfi_adjust_cfa_offset -16 977 RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here 978 REFRESH_MARKING_REGISTER 979 RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 980END art_quick_set64_instance 981 982 .extern artSet64StaticFromCompiledCode 983ENTRY art_quick_set64_static 984 SETUP_SAVE_REFS_ONLY_FRAME r12 @ save callee saves in case of GC 985 @ r2:r3 contain the wide argument 986 str rSELF, [sp, #-16]! @ expand the frame and pass Thread::Current 987 .cfi_adjust_cfa_offset 16 988 bl artSet64StaticFromCompiledCode @ (field_idx, new_val, Thread*) 989 add sp, #16 @ release out args 990 .cfi_adjust_cfa_offset -16 991 RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here 992 REFRESH_MARKING_REGISTER 993 RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 994END art_quick_set64_static 995 996// Generate the allocation entrypoints for each allocator. 997GENERATE_ALLOC_ENTRYPOINTS_FOR_NON_TLAB_ALLOCATORS 998// Comment out allocators that have arm specific asm. 999// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) 1000// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB) 1001GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) 1002GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_region_tlab, RegionTLAB) 1003// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB) 1004// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_region_tlab, RegionTLAB) 1005// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_region_tlab, RegionTLAB) 1006// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_region_tlab, RegionTLAB) 1007// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_region_tlab, RegionTLAB) 1008GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_region_tlab, RegionTLAB) 1009GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_region_tlab, RegionTLAB) 1010GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_region_tlab, RegionTLAB) 1011 1012// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB) 1013// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB) 1014GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_tlab, TLAB) 1015GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_tlab, TLAB) 1016// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_tlab, TLAB) 1017// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_tlab, TLAB) 1018// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_tlab, TLAB) 1019// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_tlab, TLAB) 1020// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_tlab, TLAB) 1021GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_tlab, TLAB) 1022GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_tlab, TLAB) 1023GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_tlab, TLAB) 1024 1025// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_RESOLVED_OBJECT(_rosalloc, RosAlloc). 1026// 1027// If isInitialized=1 then the compiler assumes the object's class has already been initialized. 1028// If isInitialized=0 the compiler can only assume it's been at least resolved. 1029.macro ART_QUICK_ALLOC_OBJECT_ROSALLOC c_name, cxx_name, isInitialized 1030ENTRY \c_name 1031 // Fast path rosalloc allocation. 1032 // r0: type/return value, rSELF (r9): Thread::Current 1033 // r1, r2, r3, r12: free. 1034 ldr r3, [rSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] // Check if the thread local 1035 // allocation stack has room. 1036 // TODO: consider using ldrd. 1037 ldr r12, [rSELF, #THREAD_LOCAL_ALLOC_STACK_END_OFFSET] 1038 cmp r3, r12 1039 bhs .Lslow_path\c_name 1040 1041 ldr r3, [r0, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (r3) 1042 cmp r3, #ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE // Check if the size is for a thread 1043 // local allocation. 1044 // If the class is not yet visibly initialized, or it is finalizable, 1045 // the object size will be very large to force the branch below to be taken. 1046 // 1047 // See Class::SetStatus() in class.cc for more details. 1048 bhs .Lslow_path\c_name 1049 // Compute the rosalloc bracket index 1050 // from the size. Since the size is 1051 // already aligned we can combine the 1052 // two shifts together. 1053 add r12, rSELF, r3, lsr #(ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT - POINTER_SIZE_SHIFT) 1054 // Subtract pointer size since ther 1055 // are no runs for 0 byte allocations 1056 // and the size is already aligned. 1057 // Load the rosalloc run (r12) 1058 ldr r12, [r12, #(THREAD_ROSALLOC_RUNS_OFFSET - __SIZEOF_POINTER__)] 1059 // Load the free list head (r3). This 1060 // will be the return val. 1061 ldr r3, [r12, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1062 cbz r3, .Lslow_path\c_name 1063 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber r0 and r1. 1064 ldr r1, [r3, #ROSALLOC_SLOT_NEXT_OFFSET] // Load the next pointer of the head 1065 // and update the list head with the 1066 // next pointer. 1067 str r1, [r12, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1068 // Store the class pointer in the 1069 // header. This also overwrites the 1070 // next pointer. The offsets are 1071 // asserted to match. 1072#if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET 1073#error "Class pointer needs to overwrite next pointer." 1074#endif 1075 POISON_HEAP_REF r0 1076 str r0, [r3, #MIRROR_OBJECT_CLASS_OFFSET] 1077 // Push the new object onto the thread 1078 // local allocation stack and 1079 // increment the thread local 1080 // allocation stack top. 1081 ldr r1, [rSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1082 str r3, [r1], #COMPRESSED_REFERENCE_SIZE // (Increment r1 as a side effect.) 1083 str r1, [rSELF, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1084 // Decrement the size of the free list 1085 1086 // After this "STR" the object is published to the thread local allocation stack, 1087 // and it will be observable from a runtime internal (eg. Heap::VisitObjects) point of view. 1088 // It is not yet visible to the running (user) compiled code until after the return. 1089 // 1090 // To avoid the memory barrier prior to the "STR", a trick is employed, by differentiating 1091 // the state of the allocation stack slot. It can be a pointer to one of: 1092 // 0) Null entry, because the stack was bumped but the new pointer wasn't written yet. 1093 // (The stack initial state is "null" pointers). 1094 // 1) A partially valid object, with an invalid class pointer to the next free rosalloc slot. 1095 // 2) A fully valid object, with a valid class pointer pointing to a real class. 1096 // Other states are not allowed. 1097 // 1098 // An object that is invalid only temporarily, and will eventually become valid. 1099 // The internal runtime code simply checks if the object is not null or is partial and then 1100 // ignores it. 1101 // 1102 // (Note: The actual check is done by seeing if a non-null object has a class pointer pointing 1103 // to ClassClass, and that the ClassClass's class pointer is self-cyclic. A rosalloc free slot 1104 // "next" pointer is not-cyclic.) 1105 // 1106 // See also b/28790624 for a listing of CLs dealing with this race. 1107 ldr r1, [r12, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1108 sub r1, #1 1109 // TODO: consider combining this store 1110 // and the list head store above using 1111 // strd. 1112 str r1, [r12, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1113 1114 mov r0, r3 // Set the return value and return. 1115 // No barrier. The class is already observably initialized (otherwise the fast 1116 // path size check above would fail) and new-instance allocations are protected 1117 // from publishing by the compiler which inserts its own StoreStore barrier. 1118 bx lr 1119 1120.Lslow_path\c_name: 1121 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 1122 mov r1, rSELF @ pass Thread::Current 1123 bl \cxx_name @ (mirror::Class* cls, Thread*) 1124 RESTORE_SAVE_REFS_ONLY_FRAME 1125 REFRESH_MARKING_REGISTER 1126 RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER 1127END \c_name 1128.endm 1129 1130ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_resolved_rosalloc, artAllocObjectFromCodeResolvedRosAlloc, /* isInitialized */ 0 1131ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_initialized_rosalloc, artAllocObjectFromCodeInitializedRosAlloc, /* isInitialized */ 1 1132 1133// The common fast path code for art_quick_alloc_object_resolved/initialized_tlab 1134// and art_quick_alloc_object_resolved/initialized_region_tlab. 1135// 1136// r0: type, rSELF (r9): Thread::Current, r1, r2, r3, r12: free. 1137// Need to preserve r0 to the slow path. 1138// 1139// If isInitialized=1 then the compiler assumes the object's class has already been initialized. 1140// If isInitialized=0 the compiler can only assume it's been at least resolved. 1141.macro ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH slowPathLabel isInitialized 1142 // Load thread_local_pos (r12) and 1143 // thread_local_end (r3) with ldrd. 1144 // Check constraints for ldrd. 1145#if !((THREAD_LOCAL_POS_OFFSET + 4 == THREAD_LOCAL_END_OFFSET) && (THREAD_LOCAL_POS_OFFSET % 8 == 0)) 1146#error "Thread::thread_local_pos/end must be consecutive and are 8 byte aligned for performance" 1147#endif 1148 ldrd r12, r3, [rSELF, #THREAD_LOCAL_POS_OFFSET] 1149 sub r12, r3, r12 // Compute the remaining buf size. 1150 ldr r3, [r0, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (r3). 1151 cmp r3, r12 // Check if it fits. 1152 // If the class is not yet visibly initialized, or it is finalizable, 1153 // the object size will be very large to force the branch below to be taken. 1154 // 1155 // See Class::SetStatus() in class.cc for more details. 1156 bhi \slowPathLabel 1157 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber r0 and r1. 1158 // Reload old thread_local_pos (r0) 1159 // for the return value. 1160 ldr r2, [rSELF, #THREAD_LOCAL_POS_OFFSET] 1161 add r1, r2, r3 1162 str r1, [rSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1163 // After this "STR" the object is published to the thread local allocation stack, 1164 // and it will be observable from a runtime internal (eg. Heap::VisitObjects) point of view. 1165 // It is not yet visible to the running (user) compiled code until after the return. 1166 // 1167 // To avoid the memory barrier prior to the "STR", a trick is employed, by differentiating 1168 // the state of the object. It can be either: 1169 // 1) A partially valid object, with a null class pointer 1170 // (because the initial state of TLAB buffers is all 0s/nulls). 1171 // 2) A fully valid object, with a valid class pointer pointing to a real class. 1172 // Other states are not allowed. 1173 // 1174 // An object that is invalid only temporarily, and will eventually become valid. 1175 // The internal runtime code simply checks if the object is not null or is partial and then 1176 // ignores it. 1177 // 1178 // (Note: The actual check is done by checking that the object's class pointer is non-null. 1179 // Also, unlike rosalloc, the object can never be observed as null). 1180 ldr r1, [rSELF, #THREAD_LOCAL_OBJECTS_OFFSET] // Increment thread_local_objects. 1181 add r1, r1, #1 1182 str r1, [rSELF, #THREAD_LOCAL_OBJECTS_OFFSET] 1183 POISON_HEAP_REF r0 1184 str r0, [r2, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1185 mov r0, r2 1186 // No barrier. The class is already observably initialized (otherwise the fast 1187 // path size check above would fail) and new-instance allocations are protected 1188 // from publishing by the compiler which inserts its own StoreStore barrier. 1189 bx lr 1190.endm 1191 1192// The common code for art_quick_alloc_object_*region_tlab 1193// Currently the implementation ignores isInitialized. TODO(b/172087402): clean this up. 1194// Caller must execute a constructor fence after this. 1195.macro GENERATE_ALLOC_OBJECT_RESOLVED_TLAB name, entrypoint, isInitialized 1196ENTRY \name 1197 // Fast path tlab allocation. 1198 // r0: type, rSELF (r9): Thread::Current 1199 // r1, r2, r3, r12: free. 1200 ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH .Lslow_path\name, \isInitialized 1201.Lslow_path\name: 1202 SETUP_SAVE_REFS_ONLY_FRAME r2 // Save callee saves in case of GC. 1203 mov r1, rSELF // Pass Thread::Current. 1204 bl \entrypoint // (mirror::Class* klass, Thread*) 1205 RESTORE_SAVE_REFS_ONLY_FRAME 1206 REFRESH_MARKING_REGISTER 1207 RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER 1208END \name 1209.endm 1210 1211GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_resolved_region_tlab, artAllocObjectFromCodeResolvedRegionTLAB, /* isInitialized */ 0 1212GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_initialized_region_tlab, artAllocObjectFromCodeInitializedRegionTLAB, /* isInitialized */ 1 1213GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_resolved_tlab, artAllocObjectFromCodeResolvedTLAB, /* isInitialized */ 0 1214GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_initialized_tlab, artAllocObjectFromCodeInitializedTLAB, /* isInitialized */ 1 1215 1216 1217// The common fast path code for art_quick_alloc_array_resolved/initialized_tlab 1218// and art_quick_alloc_array_resolved/initialized_region_tlab. 1219// 1220// r0: type, r1: component_count, r2: total_size, rSELF (r9): Thread::Current, r3, r12: free. 1221// Need to preserve r0 and r1 to the slow path. 1222.macro ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE slowPathLabel 1223 and r2, r2, #OBJECT_ALIGNMENT_MASK_TOGGLED // Apply alignment mask 1224 // (addr + 7) & ~7. 1225 1226 // Load thread_local_pos (r3) and 1227 // thread_local_end (r12) with ldrd. 1228 // Check constraints for ldrd. 1229#if !((THREAD_LOCAL_POS_OFFSET + 4 == THREAD_LOCAL_END_OFFSET) && (THREAD_LOCAL_POS_OFFSET % 8 == 0)) 1230#error "Thread::thread_local_pos/end must be consecutive and are 8 byte aligned for performance" 1231#endif 1232 ldrd r3, r12, [rSELF, #THREAD_LOCAL_POS_OFFSET] 1233 sub r12, r12, r3 // Compute the remaining buf size. 1234 cmp r2, r12 // Check if the total_size fits. 1235 // The array class is always initialized here. Unlike new-instance, 1236 // this does not act as a double test. 1237 bhi \slowPathLabel 1238 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber r0 and r1. 1239 add r2, r2, r3 1240 str r2, [rSELF, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1241 ldr r2, [rSELF, #THREAD_LOCAL_OBJECTS_OFFSET] // Increment thread_local_objects. 1242 add r2, r2, #1 1243 str r2, [rSELF, #THREAD_LOCAL_OBJECTS_OFFSET] 1244 POISON_HEAP_REF r0 1245 str r0, [r3, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1246 str r1, [r3, #MIRROR_ARRAY_LENGTH_OFFSET] // Store the array length. 1247 mov r0, r3 1248// new-array is special. The class is loaded and immediately goes to the Initialized state 1249// before it is published. Therefore the only fence needed is for the publication of the object. 1250// See ClassLinker::CreateArrayClass() for more details. 1251 1252// For publication of the new array, we don't need a 'dmb ishst' here. 1253// The compiler generates 'dmb ishst' for all new-array insts. 1254 bx lr 1255.endm 1256 1257// Caller must execute a constructor fence after this. 1258.macro GENERATE_ALLOC_ARRAY_TLAB name, entrypoint, size_setup 1259ENTRY \name 1260 // Fast path array allocation for region tlab allocation. 1261 // r0: mirror::Class* type 1262 // r1: int32_t component_count 1263 // rSELF (r9): thread 1264 // r2, r3, r12: free. 1265 \size_setup .Lslow_path\name 1266 ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE .Lslow_path\name 1267.Lslow_path\name: 1268 // r0: mirror::Class* klass 1269 // r1: int32_t component_count 1270 // r2: Thread* self 1271 SETUP_SAVE_REFS_ONLY_FRAME r2 // save callee saves in case of GC 1272 mov r2, rSELF // pass Thread::Current 1273 bl \entrypoint 1274 RESTORE_SAVE_REFS_ONLY_FRAME 1275 REFRESH_MARKING_REGISTER 1276 RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER 1277END \name 1278.endm 1279 1280.macro COMPUTE_ARRAY_SIZE_UNKNOWN slow_path 1281 movw r2, #((MIN_LARGE_OBJECT_THRESHOLD - MIRROR_WIDE_ARRAY_DATA_OFFSET) / 8) 1282 cmp r1, r2 1283 bhi \slow_path 1284 // Array classes are never finalizable 1285 // or uninitialized, no need to check. 1286 ldr r3, [r0, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] // Load component type 1287 UNPOISON_HEAP_REF r3 1288 ldr r3, [r3, #MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET] 1289 lsr r3, r3, #PRIMITIVE_TYPE_SIZE_SHIFT_SHIFT // Component size shift is in high 16 1290 // bits. 1291 lsl r2, r1, r3 // Calculate data size 1292 // Add array data offset and alignment. 1293 add r2, r2, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1294#if MIRROR_WIDE_ARRAY_DATA_OFFSET != MIRROR_INT_ARRAY_DATA_OFFSET + 4 1295#error Long array data offset must be 4 greater than int array data offset. 1296#endif 1297 1298 add r3, r3, #1 // Add 4 to the length only if the 1299 // component size shift is 3 1300 // (for 64 bit alignment). 1301 and r3, r3, #4 1302 add r2, r2, r3 1303.endm 1304 1305.macro COMPUTE_ARRAY_SIZE_8 slow_path 1306 // Possibly a large object, go slow. 1307 // Also does negative array size check. 1308 movw r2, #(MIN_LARGE_OBJECT_THRESHOLD - MIRROR_INT_ARRAY_DATA_OFFSET) 1309 cmp r1, r2 1310 bhi \slow_path 1311 // Add array data offset and alignment. 1312 add r2, r1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1313.endm 1314 1315.macro COMPUTE_ARRAY_SIZE_16 slow_path 1316 // Possibly a large object, go slow. 1317 // Also does negative array size check. 1318 movw r2, #((MIN_LARGE_OBJECT_THRESHOLD - MIRROR_INT_ARRAY_DATA_OFFSET) / 2) 1319 cmp r1, r2 1320 bhi \slow_path 1321 lsl r2, r1, #1 1322 // Add array data offset and alignment. 1323 add r2, r2, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1324.endm 1325 1326.macro COMPUTE_ARRAY_SIZE_32 slow_path 1327 // Possibly a large object, go slow. 1328 // Also does negative array size check. 1329 movw r2, #((MIN_LARGE_OBJECT_THRESHOLD - MIRROR_INT_ARRAY_DATA_OFFSET) / 4) 1330 cmp r1, r2 1331 bhi \slow_path 1332 lsl r2, r1, #2 1333 // Add array data offset and alignment. 1334 add r2, r2, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1335.endm 1336 1337.macro COMPUTE_ARRAY_SIZE_64 slow_path 1338 // Possibly a large object, go slow. 1339 // Also does negative array size check. 1340 movw r2, #((MIN_LARGE_OBJECT_THRESHOLD - MIRROR_LONG_ARRAY_DATA_OFFSET) / 8) 1341 cmp r1, r2 1342 bhi \slow_path 1343 lsl r2, r1, #3 1344 // Add array data offset and alignment. 1345 add r2, r2, #(MIRROR_WIDE_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1346.endm 1347 1348GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN 1349GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_8 1350GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_16 1351GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_32 1352GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_64 1353GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN 1354GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_8 1355GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_16 1356GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_32 1357GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_64 1358 1359 /* 1360 * Called by managed code when the value in rSUSPEND has been decremented to 0. 1361 */ 1362 .extern artTestSuspendFromCode 1363ENTRY art_quick_test_suspend 1364 SETUP_SAVE_EVERYTHING_FRAME r0, RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET @ save everything for GC stack crawl 1365 mov r0, rSELF 1366 bl artTestSuspendFromCode @ (Thread*) 1367 RESTORE_SAVE_EVERYTHING_FRAME 1368 REFRESH_MARKING_REGISTER 1369 bx lr 1370END art_quick_test_suspend 1371 1372 .extern artImplicitSuspendFromCode 1373ENTRY art_quick_implicit_suspend 1374 mov r0, rSELF 1375 SETUP_SAVE_REFS_ONLY_FRAME r1 @ save callee saves for stack crawl 1376 bl artImplicitSuspendFromCode @ (Thread*) 1377 RESTORE_SAVE_REFS_ONLY_FRAME 1378 REFRESH_MARKING_REGISTER 1379 bx lr 1380END art_quick_implicit_suspend 1381 1382 /* 1383 * Called by managed code that is attempting to call a method on a proxy class. On entry 1384 * r0 holds the proxy method and r1 holds the receiver; r2 and r3 may contain arguments. The 1385 * frame size of the invoked proxy method agrees with a ref and args callee save frame. 1386 */ 1387 .extern artQuickProxyInvokeHandler 1388ENTRY art_quick_proxy_invoke_handler 1389 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_R0 1390 mov r2, rSELF @ pass Thread::Current 1391 mov r3, sp @ pass SP 1392 blx artQuickProxyInvokeHandler @ (Method* proxy method, receiver, Thread*, SP) 1393 ldr r2, [rSELF, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ 1394 // Tear down the callee-save frame. Skip arg registers. 1395 add sp, #(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) 1396 .cfi_adjust_cfa_offset -(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) 1397 RESTORE_SAVE_REFS_ONLY_FRAME 1398 REFRESH_MARKING_REGISTER 1399 cbnz r2, 1f @ success if no exception is pending 1400 vmov d0, r0, r1 @ store into fpr, for when it's a fpr return... 1401 bx lr @ return on success 14021: 1403 DELIVER_PENDING_EXCEPTION 1404END art_quick_proxy_invoke_handler 1405 1406 /* 1407 * Called to resolve an imt conflict. 1408 * r0 is the conflict ArtMethod. 1409 * r12 is a hidden argument that holds the target interface method. 1410 * 1411 * Note that this stub writes to r0, r4, and r12. 1412 */ 1413ENTRY art_quick_imt_conflict_trampoline 1414 ldr r0, [r0, #ART_METHOD_JNI_OFFSET_32] // Load ImtConflictTable 1415 ldr r4, [r0] // Load first entry in ImtConflictTable. 1416.Limt_table_iterate: 1417 cmp r4, r12 1418 // Branch if found. Benchmarks have shown doing a branch here is better. 1419 beq .Limt_table_found 1420 // If the entry is null, the interface method is not in the ImtConflictTable. 1421 cbz r4, .Lconflict_trampoline 1422 // Iterate over the entries of the ImtConflictTable. 1423 ldr r4, [r0, #(2 * __SIZEOF_POINTER__)]! 1424 b .Limt_table_iterate 1425.Limt_table_found: 1426 // We successfully hit an entry in the table. Load the target method 1427 // and jump to it. 1428 ldr r0, [r0, #__SIZEOF_POINTER__] 1429 ldr pc, [r0, #ART_METHOD_QUICK_CODE_OFFSET_32] 1430.Lconflict_trampoline: 1431 // Pass interface method to the trampoline. 1432 mov r0, r12 1433 INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline 1434END art_quick_imt_conflict_trampoline 1435 1436 .extern artQuickResolutionTrampoline 1437ENTRY art_quick_resolution_trampoline 1438 SETUP_SAVE_REFS_AND_ARGS_FRAME r2 1439 mov r2, rSELF @ pass Thread::Current 1440 mov r3, sp @ pass SP 1441 blx artQuickResolutionTrampoline @ (Method* called, receiver, Thread*, SP) 1442 cbz r0, 1f @ is code pointer null? goto exception 1443 CFI_REMEMBER_STATE 1444 mov r12, r0 1445 ldr r0, [sp, #0] @ load resolved method in r0 1446 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1447 REFRESH_MARKING_REGISTER 1448 bx r12 @ tail-call into actual code 14491: 1450 CFI_RESTORE_STATE_AND_DEF_CFA sp, FRAME_SIZE_SAVE_REFS_AND_ARGS 1451 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1452 DELIVER_PENDING_EXCEPTION 1453END art_quick_resolution_trampoline 1454 1455 /* 1456 * Called to do a generic JNI down-call 1457 */ 1458ENTRY art_quick_generic_jni_trampoline 1459 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_R0 1460 1461 // Save rSELF 1462 mov r11, rSELF 1463 // Save SP , so we can have static CFI info. r10 is saved in ref_and_args. 1464 mov r10, sp 1465 .cfi_def_cfa_register r10 1466 1467 sub sp, sp, #GENERIC_JNI_TRAMPOLINE_RESERVED_AREA 1468 1469 // prepare for artQuickGenericJniTrampoline call 1470 // (Thread*, managed_sp, reserved_area) 1471 // r0 r1 r2 <= C calling convention 1472 // rSELF r10 sp <= where they are 1473 1474 mov r0, rSELF // Thread* 1475 mov r1, r10 // SP for the managed frame. 1476 mov r2, sp // reserved area for arguments and other saved data (up to managed frame) 1477 blx artQuickGenericJniTrampoline // (Thread*, managed_sp, reserved_area) 1478 1479 // The C call will have registered the complete save-frame on success. 1480 // The result of the call is: 1481 // r0: pointer to native code, 0 on error. 1482 // The bottom of the reserved area contains values for arg registers, 1483 // hidden arg register and SP for out args for the call. 1484 1485 // Check for error (class init check or locking for synchronized native method can throw). 1486 cbz r0, .Lexception_in_native 1487 1488 // Save the code pointer 1489 mov lr, r0 1490 1491 // Load parameters from frame into registers r0-r3 (soft-float), 1492 // hidden arg (r4) for @CriticalNative and SP for out args. 1493 pop {r0-r3, r4, ip} 1494 1495 // Apply the new SP for out args, releasing unneeded reserved area. 1496 mov sp, ip 1497 1498 // Softfloat. 1499 // TODO: Change to hardfloat when supported. 1500 1501 blx lr // native call. 1502 1503 // result sign extension is handled in C code 1504 // prepare for artQuickGenericJniEndTrampoline call 1505 // (Thread*, result, result_f) 1506 // r0 r2,r3 stack <= C calling convention 1507 // r11 r0,r1 r0,r1 <= where they are 1508 sub sp, sp, #8 // Stack alignment. 1509 1510 push {r0-r1} 1511 mov r3, r1 1512 mov r2, r0 1513 mov r0, r11 1514 1515 blx artQuickGenericJniEndTrampoline 1516 1517 // Restore self pointer. 1518 mov rSELF, r11 1519 1520 // Pending exceptions possible. 1521 ldr r2, [rSELF, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ 1522 cbnz r2, .Lexception_in_native 1523 1524 // Tear down the alloca. 1525 mov sp, r10 1526 1527 // store into fpr, for when it's a fpr return... 1528 vmov d0, r0, r1 1529 1530 LOAD_RUNTIME_INSTANCE r2 1531 ldr r2, [r2, #RUN_EXIT_HOOKS_OFFSET_FROM_RUNTIME_INSTANCE] 1532 cbnz r2, .Lcall_method_exit_hook 1533.Lcall_method_exit_hook_done: 1534 1535 // Tear down the callee-save frame. Skip arg registers. 1536 CFI_REMEMBER_STATE 1537 .cfi_def_cfa_register sp 1538 add sp, #(FRAME_SIZE_SAVE_REFS_AND_ARGS - 7 * 4) 1539 .cfi_adjust_cfa_offset -(FRAME_SIZE_SAVE_REFS_AND_ARGS - 7 * 4) 1540 pop {r5-r8, r10-r11, lr} @ This must match the non-args registers restored by 1541 .cfi_restore r5 @ `RESTORE_SAVE_REFS_AND_ARGS_FRAME`. 1542 .cfi_restore r6 1543 .cfi_restore r7 1544 .cfi_restore r8 1545 .cfi_restore r10 1546 .cfi_restore r11 1547 .cfi_restore lr 1548 .cfi_adjust_cfa_offset -(7 * 4) 1549 REFRESH_MARKING_REGISTER 1550 bx lr // ret 1551 1552 // Undo the unwinding information from above since it doesn't apply below. 1553 CFI_RESTORE_STATE_AND_DEF_CFA r10, FRAME_SIZE_SAVE_REFS_AND_ARGS 1554 1555.Lcall_method_exit_hook: 1556 mov r2, #FRAME_SIZE_SAVE_REFS_AND_ARGS 1557 bl art_quick_method_exit_hook 1558 b .Lcall_method_exit_hook_done 1559 1560.Lexception_in_native: 1561 ldr ip, [rSELF, #THREAD_TOP_QUICK_FRAME_OFFSET] 1562 add ip, ip, #-1 // Remove the GenericJNI tag. ADD/SUB writing directly to SP is UNPREDICTABLE. 1563 mov sp, ip 1564 bl art_deliver_pending_exception 1565END art_quick_generic_jni_trampoline 1566 1567ENTRY art_deliver_pending_exception 1568 # This will create a new save-all frame, required by the runtime. 1569 DELIVER_PENDING_EXCEPTION 1570END art_deliver_pending_exception 1571 1572 .extern artQuickToInterpreterBridge 1573ENTRY art_quick_to_interpreter_bridge 1574 SETUP_SAVE_REFS_AND_ARGS_FRAME r1 1575 mov r1, rSELF @ pass Thread::Current 1576 mov r2, sp @ pass SP 1577 blx artQuickToInterpreterBridge @ (Method* method, Thread*, SP) 1578 ldr r2, [rSELF, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ 1579 // Tear down the callee-save frame. Skip arg registers. 1580 add sp, #(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) 1581 .cfi_adjust_cfa_offset -(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) 1582 RESTORE_SAVE_REFS_ONLY_FRAME 1583 REFRESH_MARKING_REGISTER 1584 cbnz r2, 1f @ success if no exception is pending 1585 vmov d0, r0, r1 @ store into fpr, for when it's a fpr return... 1586 bx lr @ return on success 15871: 1588 DELIVER_PENDING_EXCEPTION 1589END art_quick_to_interpreter_bridge 1590 1591/* 1592 * Called to attempt to execute an obsolete method. 1593 */ 1594ONE_ARG_RUNTIME_EXCEPTION art_invoke_obsolete_method_stub, artInvokeObsoleteMethod 1595 1596 /* 1597 * Compiled code has requested that we deoptimize into the interpreter. The deoptimization 1598 * will long jump to the interpreter bridge. 1599 */ 1600 .extern artDeoptimizeFromCompiledCode 1601ENTRY art_quick_deoptimize_from_compiled_code 1602 SETUP_SAVE_EVERYTHING_FRAME r1 1603 mov r1, rSELF @ pass Thread::Current 1604 blx artDeoptimizeFromCompiledCode @ (DeoptimizationKind, Thread*) 1605END art_quick_deoptimize_from_compiled_code 1606 1607 /* 1608 * Signed 64-bit integer multiply. 1609 * 1610 * Consider WXxYZ (r1r0 x r3r2) with a long multiply: 1611 * WX 1612 * x YZ 1613 * -------- 1614 * ZW ZX 1615 * YW YX 1616 * 1617 * The low word of the result holds ZX, the high word holds 1618 * (ZW+YX) + (the high overflow from ZX). YW doesn't matter because 1619 * it doesn't fit in the low 64 bits. 1620 * 1621 * Unlike most ARM math operations, multiply instructions have 1622 * restrictions on using the same register more than once (Rd and Rm 1623 * cannot be the same). 1624 */ 1625 /* mul-long vAA, vBB, vCC */ 1626ENTRY art_quick_mul_long 1627 push {r9-r10} 1628 .cfi_adjust_cfa_offset 8 1629 .cfi_rel_offset r9, 0 1630 .cfi_rel_offset r10, 4 1631 mul ip, r2, r1 @ ip<- ZxW 1632 umull r9, r10, r2, r0 @ r9/r10 <- ZxX 1633 mla r2, r0, r3, ip @ r2<- YxX + (ZxW) 1634 add r10, r2, r10 @ r10<- r10 + low(ZxW + (YxX)) 1635 mov r0,r9 1636 mov r1,r10 1637 pop {r9-r10} 1638 .cfi_adjust_cfa_offset -8 1639 .cfi_restore r9 1640 .cfi_restore r10 1641 bx lr 1642END art_quick_mul_long 1643 1644 /* 1645 * Long integer shift. This is different from the generic 32/64-bit 1646 * binary operations because vAA/vBB are 64-bit but vCC (the shift 1647 * distance) is 32-bit. Also, Dalvik requires us to ignore all but the low 1648 * 6 bits. 1649 * On entry: 1650 * r0: low word 1651 * r1: high word 1652 * r2: shift count 1653 */ 1654 /* shl-long vAA, vBB, vCC */ 1655ARM_ENTRY art_quick_shl_long @ ARM code as thumb code requires spills 1656 and r2, r2, #63 @ r2<- r2 & 0x3f 1657 mov r1, r1, asl r2 @ r1<- r1 << r2 1658 rsb r3, r2, #32 @ r3<- 32 - r2 1659 orr r1, r1, r0, lsr r3 @ r1<- r1 | (r0 << (32-r2)) 1660 subs ip, r2, #32 @ ip<- r2 - 32 1661 movpl r1, r0, asl ip @ if r2 >= 32, r1<- r0 << (r2-32) 1662 mov r0, r0, asl r2 @ r0<- r0 << r2 1663 bx lr 1664END art_quick_shl_long 1665 1666 /* 1667 * Long integer shift. This is different from the generic 32/64-bit 1668 * binary operations because vAA/vBB are 64-bit but vCC (the shift 1669 * distance) is 32-bit. Also, Dalvik requires us to ignore all but the low 1670 * 6 bits. 1671 * On entry: 1672 * r0: low word 1673 * r1: high word 1674 * r2: shift count 1675 */ 1676 /* shr-long vAA, vBB, vCC */ 1677ARM_ENTRY art_quick_shr_long @ ARM code as thumb code requires spills 1678 and r2, r2, #63 @ r0<- r0 & 0x3f 1679 mov r0, r0, lsr r2 @ r0<- r2 >> r2 1680 rsb r3, r2, #32 @ r3<- 32 - r2 1681 orr r0, r0, r1, asl r3 @ r0<- r0 | (r1 << (32-r2)) 1682 subs ip, r2, #32 @ ip<- r2 - 32 1683 movpl r0, r1, asr ip @ if r2 >= 32, r0<-r1 >> (r2-32) 1684 mov r1, r1, asr r2 @ r1<- r1 >> r2 1685 bx lr 1686END art_quick_shr_long 1687 1688 /* 1689 * Long integer shift. This is different from the generic 32/64-bit 1690 * binary operations because vAA/vBB are 64-bit but vCC (the shift 1691 * distance) is 32-bit. Also, Dalvik requires us to ignore all but the low 1692 * 6 bits. 1693 * On entry: 1694 * r0: low word 1695 * r1: high word 1696 * r2: shift count 1697 */ 1698 /* ushr-long vAA, vBB, vCC */ 1699ARM_ENTRY art_quick_ushr_long @ ARM code as thumb code requires spills 1700 and r2, r2, #63 @ r0<- r0 & 0x3f 1701 mov r0, r0, lsr r2 @ r0<- r2 >> r2 1702 rsb r3, r2, #32 @ r3<- 32 - r2 1703 orr r0, r0, r1, asl r3 @ r0<- r0 | (r1 << (32-r2)) 1704 subs ip, r2, #32 @ ip<- r2 - 32 1705 movpl r0, r1, lsr ip @ if r2 >= 32, r0<-r1 >>> (r2-32) 1706 mov r1, r1, lsr r2 @ r1<- r1 >>> r2 1707 bx lr 1708END art_quick_ushr_long 1709 1710 /* 1711 * String's indexOf. 1712 * 1713 * On entry: 1714 * r0: string object (known non-null) 1715 * r1: char to match (known <= 0xFFFF) 1716 * r2: Starting offset in string data 1717 */ 1718ENTRY art_quick_indexof 1719 push {r4, r10-r11, lr} @ 4 words of callee saves 1720 .cfi_adjust_cfa_offset 16 1721 .cfi_rel_offset r4, 0 1722 .cfi_rel_offset r10, 4 1723 .cfi_rel_offset r11, 8 1724 .cfi_rel_offset lr, 12 1725#if (STRING_COMPRESSION_FEATURE) 1726 ldr r4, [r0, #MIRROR_STRING_COUNT_OFFSET] 1727#else 1728 ldr r3, [r0, #MIRROR_STRING_COUNT_OFFSET] 1729#endif 1730 add r0, #MIRROR_STRING_VALUE_OFFSET 1731#if (STRING_COMPRESSION_FEATURE) 1732 /* r4 count (with flag) and r3 holds actual length */ 1733 lsr r3, r4, #1 1734#endif 1735 /* Clamp start to [0..count] */ 1736 cmp r2, #0 1737 it lt 1738 movlt r2, #0 1739 cmp r2, r3 1740 it gt 1741 movgt r2, r3 1742 1743 /* Save a copy in r12 to later compute result */ 1744 mov r12, r0 1745 1746 /* Build pointer to start of data to compare and pre-bias */ 1747#if (STRING_COMPRESSION_FEATURE) 1748 lsrs r4, r4, #1 1749 bcc .Lstring_indexof_compressed 1750#endif 1751 add r0, r0, r2, lsl #1 1752 sub r0, #2 1753 1754 /* Compute iteration count */ 1755 sub r2, r3, r2 1756 1757 /* 1758 * At this point we have: 1759 * r0: start of data to test 1760 * r1: char to compare 1761 * r2: iteration count 1762 * r4: compression style (used temporarily) 1763 * r12: original start of string data 1764 * r3, r4, r10, r11 available for loading string data 1765 */ 1766 1767 subs r2, #4 1768 blt .Lindexof_remainder 1769 1770.Lindexof_loop4: 1771 ldrh r3, [r0, #2]! 1772 ldrh r4, [r0, #2]! 1773 ldrh r10, [r0, #2]! 1774 ldrh r11, [r0, #2]! 1775 cmp r3, r1 1776 beq .Lmatch_0 1777 cmp r4, r1 1778 beq .Lmatch_1 1779 cmp r10, r1 1780 beq .Lmatch_2 1781 cmp r11, r1 1782 beq .Lmatch_3 1783 subs r2, #4 1784 bge .Lindexof_loop4 1785 1786.Lindexof_remainder: 1787 adds r2, #4 1788 beq .Lindexof_nomatch 1789 1790.Lindexof_loop1: 1791 ldrh r3, [r0, #2]! 1792 cmp r3, r1 1793 beq .Lmatch_3 1794 subs r2, #1 1795 bne .Lindexof_loop1 1796 1797.Lindexof_nomatch: 1798 mov r0, #-1 1799 pop {r4, r10-r11, pc} 1800 1801.Lmatch_0: 1802 sub r0, #6 1803 sub r0, r12 1804 asr r0, r0, #1 1805 pop {r4, r10-r11, pc} 1806.Lmatch_1: 1807 sub r0, #4 1808 sub r0, r12 1809 asr r0, r0, #1 1810 pop {r4, r10-r11, pc} 1811.Lmatch_2: 1812 sub r0, #2 1813 sub r0, r12 1814 asr r0, r0, #1 1815 pop {r4, r10-r11, pc} 1816.Lmatch_3: 1817 sub r0, r12 1818 asr r0, r0, #1 1819 pop {r4, r10-r11, pc} 1820#if (STRING_COMPRESSION_FEATURE) 1821.Lstring_indexof_compressed: 1822 add r0, r0, r2 1823 sub r0, #1 1824 sub r2, r3, r2 1825.Lstring_indexof_compressed_loop: 1826 subs r2, #1 1827 blt .Lindexof_nomatch 1828 ldrb r3, [r0, #1]! 1829 cmp r3, r1 1830 beq .Lstring_indexof_compressed_matched 1831 b .Lstring_indexof_compressed_loop 1832.Lstring_indexof_compressed_matched: 1833 sub r0, r12 1834 pop {r4, r10-r11, pc} 1835#endif 1836END art_quick_indexof 1837 1838 /* Assembly routines used to handle ABI differences. */ 1839 1840 /* double fmod(double a, double b) */ 1841 .extern fmod 1842ENTRY art_quick_fmod 1843 push {lr} 1844 .cfi_adjust_cfa_offset 4 1845 .cfi_rel_offset lr, 0 1846 sub sp, #4 1847 .cfi_adjust_cfa_offset 4 1848 vmov r0, r1, d0 1849 vmov r2, r3, d1 1850 bl fmod 1851 vmov d0, r0, r1 1852 add sp, #4 1853 .cfi_adjust_cfa_offset -4 1854 pop {pc} 1855END art_quick_fmod 1856 1857 /* float fmodf(float a, float b) */ 1858 .extern fmodf 1859ENTRY art_quick_fmodf 1860 push {lr} 1861 .cfi_adjust_cfa_offset 4 1862 .cfi_rel_offset lr, 0 1863 sub sp, #4 1864 .cfi_adjust_cfa_offset 4 1865 vmov r0, r1, d0 1866 bl fmodf 1867 vmov s0, r0 1868 add sp, #4 1869 .cfi_adjust_cfa_offset -4 1870 pop {pc} 1871END art_quick_fmodf 1872 1873 /* int64_t art_d2l(double d) */ 1874 .extern art_d2l 1875ENTRY art_quick_d2l 1876 vmov r0, r1, d0 1877 b art_d2l 1878END art_quick_d2l 1879 1880 /* int64_t art_f2l(float f) */ 1881 .extern art_f2l 1882ENTRY art_quick_f2l 1883 vmov r0, s0 1884 b art_f2l 1885END art_quick_f2l 1886 1887 /* float art_l2f(int64_t l) */ 1888 .extern art_l2f 1889ENTRY art_quick_l2f 1890 push {lr} 1891 .cfi_adjust_cfa_offset 4 1892 .cfi_rel_offset lr, 0 1893 sub sp, #4 1894 .cfi_adjust_cfa_offset 4 1895 bl art_l2f 1896 vmov s0, r0 1897 add sp, #4 1898 .cfi_adjust_cfa_offset -4 1899 pop {pc} 1900END art_quick_l2f 1901 1902 .extern artStringBuilderAppend 1903ENTRY art_quick_string_builder_append 1904 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 1905 add r1, sp, #(FRAME_SIZE_SAVE_REFS_ONLY + __SIZEOF_POINTER__) @ pass args 1906 mov r2, rSELF @ pass Thread::Current 1907 bl artStringBuilderAppend @ (uint32_t, const unit32_t*, Thread*) 1908 RESTORE_SAVE_REFS_ONLY_FRAME 1909 REFRESH_MARKING_REGISTER 1910 RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER 1911END art_quick_string_builder_append 1912 1913 /* 1914 * Create a function `name` calling the ReadBarrier::Mark routine, 1915 * getting its argument and returning its result through register 1916 * `reg`, saving and restoring all caller-save registers. 1917 * 1918 * IP is clobbered; `reg` must not be IP. 1919 * 1920 * If `reg` is different from `r0`, the generated function follows a 1921 * non-standard runtime calling convention: 1922 * - register `reg` (which may be different from R0) is used to pass the (sole) argument, 1923 * - register `reg` (which may be different from R0) is used to return the result, 1924 * - all other registers are callee-save (the values they hold are preserved). 1925 */ 1926.macro READ_BARRIER_MARK_REG name, reg 1927ENTRY \name 1928 // Null check so that we can load the lock word. 1929 SMART_CBZ \reg, .Lret_rb_\name 1930 // Check lock word for mark bit, if marked return. Use IP for scratch since it is blocked. 1931 ldr ip, [\reg, MIRROR_OBJECT_LOCK_WORD_OFFSET] 1932 tst ip, #LOCK_WORD_MARK_BIT_MASK_SHIFTED 1933 beq .Lnot_marked_rb_\name 1934 // Already marked, return right away. 1935.Lret_rb_\name: 1936 bx lr 1937 1938.Lnot_marked_rb_\name: 1939 // Test that both the forwarding state bits are 1. 1940#if (LOCK_WORD_STATE_SHIFT != 30) || (LOCK_WORD_STATE_FORWARDING_ADDRESS != 3) 1941 // To use "CMP ip, #modified-immediate; BHS", we need the lock word state in 1942 // the highest bits and the "forwarding address" state to have all bits set. 1943#error "Unexpected lock word state shift or forwarding address state value." 1944#endif 1945 cmp ip, #(LOCK_WORD_STATE_FORWARDING_ADDRESS << LOCK_WORD_STATE_SHIFT) 1946 bhs .Lret_forwarding_address\name 1947 1948.Lslow_rb_\name: 1949 // Save IP: The kSaveEverything entrypoint art_quick_resolve_string used to 1950 // make a tail call here. Currently, it serves only for stack alignment but 1951 // we may reintroduce kSaveEverything calls here in the future. 1952 push {r0-r4, r9, ip, lr} @ save return address, core caller-save registers and ip 1953 .cfi_adjust_cfa_offset 32 1954 .cfi_rel_offset r0, 0 1955 .cfi_rel_offset r1, 4 1956 .cfi_rel_offset r2, 8 1957 .cfi_rel_offset r3, 12 1958 .cfi_rel_offset r4, 16 1959 .cfi_rel_offset r9, 20 1960 .cfi_rel_offset ip, 24 1961 .cfi_rel_offset lr, 28 1962 1963 .ifnc \reg, r0 1964 mov r0, \reg @ pass arg1 - obj from `reg` 1965 .endif 1966 1967 vpush {s0-s15} @ save floating-point caller-save registers 1968 .cfi_adjust_cfa_offset 64 1969 bl artReadBarrierMark @ r0 <- artReadBarrierMark(obj) 1970 vpop {s0-s15} @ restore floating-point registers 1971 .cfi_adjust_cfa_offset -64 1972 1973 .ifc \reg, r0 @ Save result to the stack slot or destination register. 1974 str r0, [sp, #0] 1975 .else 1976 .ifc \reg, r1 1977 str r0, [sp, #4] 1978 .else 1979 .ifc \reg, r2 1980 str r0, [sp, #8] 1981 .else 1982 .ifc \reg, r3 1983 str r0, [sp, #12] 1984 .else 1985 .ifc \reg, r4 1986 str r0, [sp, #16] 1987 .else 1988 .ifc \reg, r9 1989 str r0, [sp, #20] 1990 .else 1991 mov \reg, r0 1992 .endif 1993 .endif 1994 .endif 1995 .endif 1996 .endif 1997 .endif 1998 1999 pop {r0-r4, r9, ip, lr} @ restore caller-save registers 2000 .cfi_adjust_cfa_offset -32 2001 .cfi_restore r0 2002 .cfi_restore r1 2003 .cfi_restore r2 2004 .cfi_restore r3 2005 .cfi_restore r4 2006 .cfi_restore r9 2007 .cfi_restore ip 2008 .cfi_restore lr 2009 bx lr 2010.Lret_forwarding_address\name: 2011 // Shift left by the forwarding address shift. This clears out the state bits since they are 2012 // in the top 2 bits of the lock word. 2013 lsl \reg, ip, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 2014 bx lr 2015END \name 2016.endm 2017 2018READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg00, r0 2019READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, r1 2020READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, r2 2021READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, r3 2022READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg04, r4 2023READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, r5 2024READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, r6 2025READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, r7 2026READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, r8 2027READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, r9 2028READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, r10 2029READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, r11 2030 2031// Helper macros for Baker CC read barrier mark introspection (BRBMI). 2032.macro BRBMI_FOR_REGISTERS macro_for_register, macro_for_reserved_register 2033 \macro_for_register r0 2034 \macro_for_register r1 2035 \macro_for_register r2 2036 \macro_for_register r3 2037 \macro_for_register r4 2038 \macro_for_register r5 2039 \macro_for_register r6 2040 \macro_for_register r7 2041 \macro_for_reserved_register // r8 (rMR) is the marking register. 2042 \macro_for_register r9 2043 \macro_for_register r10 2044 \macro_for_register r11 2045 \macro_for_reserved_register // IP is reserved. 2046 \macro_for_reserved_register // SP is reserved. 2047 \macro_for_reserved_register // LR is reserved. 2048 \macro_for_reserved_register // PC is reserved. 2049.endm 2050 2051.macro BRBMI_RETURN_SWITCH_CASE reg 2052 .balign 8 2053.Lmark_introspection_return_switch_case_\reg: 2054 mov rMR, #1 2055 mov \reg, ip 2056 bx lr 2057.endm 2058 2059.macro BRBMI_RETURN_SWITCH_CASE_OFFSET reg 2060 .byte (.Lmark_introspection_return_switch_case_\reg - .Lmark_introspection_return_table) / 2 2061.endm 2062 2063.macro BRBMI_BAD_RETURN_SWITCH_CASE_OFFSET 2064 .byte (.Lmark_introspection_return_switch_case_bad - .Lmark_introspection_return_table) / 2 2065.endm 2066 2067#if BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET != BAKER_MARK_INTROSPECTION_ARRAY_LDR_OFFSET 2068#error "Array and field introspection code sharing requires same LDR offset." 2069#endif 2070.macro BRBMI_ARRAY_LOAD index_reg 2071 ldr ip, [ip, \index_reg, lsl #2] // 4 bytes. 2072 b art_quick_read_barrier_mark_introspection // Should be 2 bytes, encoding T2. 2073 .balign 8 // Add padding to 8 bytes. 2074.endm 2075 2076.macro BRBMI_BKPT_FILL_4B 2077 bkpt 0 2078 bkpt 0 2079.endm 2080 2081.macro BRBMI_BKPT_FILL_8B 2082 BRBMI_BKPT_FILL_4B 2083 BRBMI_BKPT_FILL_4B 2084.endm 2085 2086.macro BRBMI_RUNTIME_CALL 2087 // Note: This macro generates exactly 22 bytes of code. The core register 2088 // PUSH and the MOVs are 16-bit instructions, the rest is 32-bit instructions. 2089 2090 push {r0-r3, r7, lr} // Save return address and caller-save registers. 2091 .cfi_adjust_cfa_offset 24 2092 .cfi_rel_offset r0, 0 2093 .cfi_rel_offset r1, 4 2094 .cfi_rel_offset r2, 8 2095 .cfi_rel_offset r3, 12 2096 .cfi_rel_offset r7, 16 2097 .cfi_rel_offset lr, 20 2098 2099 mov r0, ip // Pass the reference. 2100 vpush {s0-s15} // save floating-point caller-save registers 2101 .cfi_adjust_cfa_offset 64 2102 bl artReadBarrierMark // r0 <- artReadBarrierMark(obj) 2103 vpop {s0-s15} // restore floating-point registers 2104 .cfi_adjust_cfa_offset -64 2105 mov ip, r0 // Move reference to ip in preparation for return switch. 2106 2107 pop {r0-r3, r7, lr} // Restore registers. 2108 .cfi_adjust_cfa_offset -24 2109 .cfi_restore r0 2110 .cfi_restore r1 2111 .cfi_restore r2 2112 .cfi_restore r3 2113 .cfi_restore r7 2114 .cfi_restore lr 2115.endm 2116 2117.macro BRBMI_CHECK_NULL_AND_MARKED label_suffix 2118 // If reference is null, just return it in the right register. 2119 cmp ip, #0 2120 beq .Lmark_introspection_return\label_suffix 2121 // Use rMR as temp and check the mark bit of the reference. 2122 ldr rMR, [ip, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 2123 tst rMR, #LOCK_WORD_MARK_BIT_MASK_SHIFTED 2124 beq .Lmark_introspection_unmarked\label_suffix 2125.Lmark_introspection_return\label_suffix: 2126.endm 2127 2128.macro BRBMI_UNMARKED_FORWARDING_ADDRESS_CHECK label_suffix 2129.Lmark_introspection_unmarked\label_suffix: 2130 // Check if the top two bits are one, if this is the case it is a forwarding address. 2131#if (LOCK_WORD_STATE_SHIFT != 30) || (LOCK_WORD_STATE_FORWARDING_ADDRESS != 3) 2132 // To use "CMP ip, #modified-immediate; BHS", we need the lock word state in 2133 // the highest bits and the "forwarding address" state to have all bits set. 2134#error "Unexpected lock word state shift or forwarding address state value." 2135#endif 2136 cmp rMR, #(LOCK_WORD_STATE_FORWARDING_ADDRESS << LOCK_WORD_STATE_SHIFT) 2137 bhs .Lmark_introspection_forwarding_address\label_suffix 2138.endm 2139 2140.macro BRBMI_EXTRACT_FORWARDING_ADDRESS label_suffix 2141.Lmark_introspection_forwarding_address\label_suffix: 2142 // Note: This macro generates exactly 22 bytes of code, the branch is near. 2143 2144 // Shift left by the forwarding address shift. This clears out the state bits since they are 2145 // in the top 2 bits of the lock word. 2146 lsl ip, rMR, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 2147 b .Lmark_introspection_return\label_suffix 2148.endm 2149 2150.macro BRBMI_LOAD_RETURN_REG_FROM_CODE_wide ldr_offset 2151 // Load the half of the instruction that contains Rt. Adjust for the thumb state in LR. 2152 ldrh rMR, [lr, #(-1 + \ldr_offset + 2)] 2153.endm 2154 2155.macro BRBMI_LOAD_RETURN_REG_FROM_CODE_narrow ldr_offset 2156 // Load the 16-bit instruction. Adjust for the thumb state in LR. 2157 ldrh rMR, [lr, #(-1 + \ldr_offset)] 2158.endm 2159 2160.macro BRBMI_EXTRACT_RETURN_REG_wide 2161 lsr rMR, rMR, #12 // Extract `ref_reg`. 2162.endm 2163 2164.macro BRBMI_EXTRACT_RETURN_REG_narrow 2165 and rMR, rMR, #7 // Extract `ref_reg`. 2166.endm 2167 2168.macro BRBMI_LOAD_AND_EXTRACT_RETURN_REG ldr_offset, label_suffix 2169 BRBMI_LOAD_RETURN_REG_FROM_CODE\label_suffix \ldr_offset 2170 BRBMI_EXTRACT_RETURN_REG\label_suffix 2171.endm 2172 2173.macro BRBMI_GC_ROOT gc_root_ldr_offset, label_suffix 2174 .balign 32 2175 .thumb_func 2176 .type art_quick_read_barrier_mark_introspection_gc_roots\label_suffix, #function 2177 .hidden art_quick_read_barrier_mark_introspection_gc_roots\label_suffix 2178 .global art_quick_read_barrier_mark_introspection_gc_roots\label_suffix 2179art_quick_read_barrier_mark_introspection_gc_roots\label_suffix: 2180 BRBMI_LOAD_AND_EXTRACT_RETURN_REG \gc_root_ldr_offset, \label_suffix 2181.endm 2182 2183.macro BRBMI_FIELD_SLOW_PATH ldr_offset, label_suffix 2184 .balign 16 2185 // Note: Generates exactly 16 bytes of code. 2186 BRBMI_UNMARKED_FORWARDING_ADDRESS_CHECK \label_suffix 2187 BRBMI_LOAD_AND_EXTRACT_RETURN_REG \ldr_offset, \label_suffix 2188 b .Lmark_introspection_runtime_call 2189.endm 2190 2191 /* 2192 * Use introspection to load a reference from the same address as the LDR 2193 * instruction in generated code would load (unless loaded by the thunk, 2194 * see below), call ReadBarrier::Mark() with that reference if needed 2195 * and return it in the same register as the LDR instruction would load. 2196 * 2197 * The entrypoint is called through a thunk that differs across load kinds. 2198 * For field and array loads the LDR instruction in generated code follows 2199 * the branch to the thunk, i.e. the LDR is (ignoring the heap poisoning) 2200 * at [LR, #(-4 - 1)] (encoding T3) or [LR, #(-2 - 1)] (encoding T1) where 2201 * the -1 is an adjustment for the Thumb mode bit in LR, and the thunk 2202 * knows the holder and performs the gray bit check, returning to the LDR 2203 * instruction if the object is not gray, so this entrypoint no longer 2204 * needs to know anything about the holder. For GC root loads, the LDR 2205 * instruction in generated code precedes the branch to the thunk, i.e. the 2206 * LDR is at [LR, #(-8 - 1)] (encoding T3) or [LR, #(-6 - 1)] (encoding T1) 2207 * where the -1 is again the Thumb mode bit adjustment, and the thunk does 2208 * not do the gray bit check. 2209 * 2210 * For field accesses and array loads with a constant index the thunk loads 2211 * the reference into IP using introspection and calls the main entrypoint 2212 * ("wide", for 32-bit LDR) art_quick_read_barrier_mark_introspection or 2213 * the "narrow" entrypoint (for 16-bit LDR). The latter is at a known 2214 * offset (BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_ENTRYPOINT_OFFSET) 2215 * from the main entrypoint and the thunk adjusts the entrypoint pointer. 2216 * With heap poisoning enabled, the passed reference is poisoned. 2217 * 2218 * For array accesses with non-constant index, the thunk inserts the bits 2219 * 0-5 of the LDR instruction to the entrypoint address, effectively 2220 * calculating a switch case label based on the index register (bits 0-3) 2221 * and adding an extra offset (bits 4-5 hold the shift which is always 2 2222 * for reference loads) to differentiate from the main entrypoint, then 2223 * moves the base register to IP and jumps to the switch case. Therefore 2224 * we need to align the main entrypoint to 512 bytes, accounting for 2225 * a 256-byte offset followed by 16 array entrypoints starting at 2226 * art_quick_read_barrier_mark_introspection_arrays, each containing an LDR 2227 * (register) and a branch to the main entrypoint. 2228 * 2229 * For GC root accesses we cannot use the main entrypoint because of the 2230 * different offset where the LDR instruction in generated code is located. 2231 * (And even with heap poisoning enabled, GC roots are not poisoned.) 2232 * To re-use the same entrypoint pointer in generated code, we make sure 2233 * that the gc root entrypoint (a copy of the entrypoint with a different 2234 * offset for introspection loads) is located at a known offset (0xc0/0xe0 2235 * bytes, or BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_ENTRYPOINT_OFFSET/ 2236 * BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_ENTRYPOINT_OFFSET) from the 2237 * main entrypoint and the GC root thunk adjusts the entrypoint pointer, 2238 * moves the root register to IP and jumps to the customized entrypoint, 2239 * art_quick_read_barrier_mark_introspection_gc_roots_{wide,narrow}. 2240 * The thunk also performs all the fast-path checks, so we need just the 2241 * slow path. 2242 * 2243 * Intrinsic CAS operations (VarHandle*CompareAnd{Set,Exchange}* and 2244 * UnsafeCASObject) use similar code to the GC roots wide load but using 2245 * MOV (register, T3) instead of the LDR (immediate, T3), with destination 2246 * register in bits 8-11 rather than 12-15. Therefore they have their own 2247 * entrypoint, art_quick_read_barrier_mark_introspection_intrinsic_cas 2248 * at the offset BAKER_MARK_INTROSPECTION_INTRINSIC_CAS_ENTRYPOINT_OFFSET. 2249 * This is used only for high registers, low registers reuse the GC roots 2250 * narrow load entrypoint as the low 3 bits of the destination register 2251 * for MOV (register) encoding T1 match the LDR (immediate) encoding T1. 2252 * 2253 * The code structure is 2254 * art_quick_read_barrier_mark_introspection: // @0x00 2255 * Up to 32 bytes code for main entrypoint fast-path code for fields 2256 * (and array elements with constant offset) with LDR encoding T3; 2257 * jumps to the switch in the "narrow" entrypoint. 2258 * art_quick_read_barrier_mark_introspection_narrow: // @0x20 2259 * Up to 48 bytes code for fast path code for fields (and array 2260 * elements with constant offset) with LDR encoding T1, ending in the 2261 * return switch instruction TBB and the table with switch offsets. 2262 * .Lmark_introspection_return_switch_case_r0: // @0x50 2263 * Exactly 88 bytes of code for the return switch cases (8 bytes per 2264 * case, 11 cases; no code for reserved registers). 2265 * .Lmark_introspection_forwarding_address_narrow: // @0xa8 2266 * Exactly 6 bytes to extract the forwarding address and jump to the 2267 * "narrow" entrypoint fast path. 2268 * .Lmark_introspection_return_switch_case_bad: // @0xae 2269 * Exactly 2 bytes, bkpt for unexpected return register. 2270 * .Lmark_introspection_unmarked_narrow: // @0xb0 2271 * Exactly 16 bytes for "narrow" entrypoint slow path. 2272 * art_quick_read_barrier_mark_introspection_gc_roots_wide: // @0xc0 2273 * GC root entrypoint code for LDR encoding T3 (10 bytes); loads and 2274 * extracts the return register and jumps to the runtime call. 2275 * .Lmark_introspection_forwarding_address_wide: // @0xca 2276 * Exactly 6 bytes to extract the forwarding address and jump to the 2277 * "wide" entrypoint fast path. 2278 * .Lmark_introspection_unmarked_wide: // @0xd0 2279 * Exactly 16 bytes for "wide" entrypoint slow path. 2280 * art_quick_read_barrier_mark_introspection_gc_roots_narrow: // @0xe0 2281 * GC root entrypoint code for LDR encoding T1 (8 bytes); loads and 2282 * extracts the return register and falls through to the runtime call. 2283 * .Lmark_introspection_runtime_call: // @0xe8 2284 * Exactly 24 bytes for the runtime call to MarkReg() and jump to the 2285 * return switch. 2286 * art_quick_read_barrier_mark_introspection_arrays: // @0x100 2287 * Exactly 128 bytes for array load switch cases (16x2 instructions). 2288 * art_quick_read_barrier_mark_introspection_intrinsic_cas: // @0x180 2289 * Intrinsic CAS entrypoint for MOV (register) encoding T3 (6 bytes). 2290 * Loads the return register and jumps to the runtime call. 2291 */ 2292#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 2293ENTRY_ALIGNED art_quick_read_barrier_mark_introspection, 512 2294 // At this point, IP contains the reference, rMR is clobbered by the thunk 2295 // and can be freely used as it will be set back to 1 before returning. 2296 // For heap poisoning, the reference is poisoned, so unpoison it first. 2297 UNPOISON_HEAP_REF ip 2298 // Check for null or marked, lock word is loaded into rMR. 2299 BRBMI_CHECK_NULL_AND_MARKED _wide 2300 // Load and extract the return register from the instruction. 2301 BRBMI_LOAD_AND_EXTRACT_RETURN_REG BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET, _wide 2302 b .Lmark_introspection_return_switch 2303 2304 .balign 32 2305 .thumb_func 2306 .type art_quick_read_barrier_mark_introspection_narrow, #function 2307 .hidden art_quick_read_barrier_mark_introspection_narrow 2308 .global art_quick_read_barrier_mark_introspection_narrow 2309art_quick_read_barrier_mark_introspection_narrow: 2310 // At this point, IP contains the reference, rMR is clobbered by the thunk 2311 // and can be freely used as it will be set back to 1 before returning. 2312 // For heap poisoning, the reference is poisoned, so unpoison it first. 2313 UNPOISON_HEAP_REF ip 2314 // Check for null or marked, lock word is loaded into rMR. 2315 BRBMI_CHECK_NULL_AND_MARKED _narrow 2316 // Load and extract the return register from the instruction. 2317 BRBMI_LOAD_AND_EXTRACT_RETURN_REG BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET, _narrow 2318.Lmark_introspection_return_switch: 2319 tbb [pc, rMR] // Jump to the switch case. 2320.Lmark_introspection_return_table: 2321 BRBMI_FOR_REGISTERS BRBMI_RETURN_SWITCH_CASE_OFFSET, BRBMI_BAD_RETURN_SWITCH_CASE_OFFSET 2322 BRBMI_FOR_REGISTERS BRBMI_RETURN_SWITCH_CASE, /* no code */ 2323 2324 .balign 8 2325 BRBMI_EXTRACT_FORWARDING_ADDRESS _narrow // 6 bytes 2326.Lmark_introspection_return_switch_case_bad: 2327 bkpt // 2 bytes 2328 2329 BRBMI_FIELD_SLOW_PATH BAKER_MARK_INTROSPECTION_FIELD_LDR_NARROW_OFFSET, _narrow 2330 2331 // 8 bytes for the loading and extracting of the return register. 2332 BRBMI_GC_ROOT BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_WIDE_OFFSET, _wide 2333 // 2 bytes for near branch to the runtime call. 2334 b .Lmark_introspection_runtime_call 2335 2336 BRBMI_EXTRACT_FORWARDING_ADDRESS _wide // Not even 4-byte aligned. 2337 2338 BRBMI_FIELD_SLOW_PATH BAKER_MARK_INTROSPECTION_FIELD_LDR_WIDE_OFFSET, _wide 2339 2340 // 8 bytes for the loading and extracting of the return register. 2341 BRBMI_GC_ROOT BAKER_MARK_INTROSPECTION_GC_ROOT_LDR_NARROW_OFFSET, _narrow 2342 // And the runtime call and branch to the switch taking exactly 24 bytes 2343 // (22 bytes for BRBMI_RUNTIME_CALL and 2 bytes for the near branch) 2344 // shall take the rest of the 32-byte section (within a cache line). 2345.Lmark_introspection_runtime_call: 2346 BRBMI_RUNTIME_CALL 2347 b .Lmark_introspection_return_switch 2348 2349 .balign 256 2350 .thumb_func 2351 .type art_quick_read_barrier_mark_introspection_arrays, #function 2352 .hidden art_quick_read_barrier_mark_introspection_arrays 2353 .global art_quick_read_barrier_mark_introspection_arrays 2354art_quick_read_barrier_mark_introspection_arrays: 2355 BRBMI_FOR_REGISTERS BRBMI_ARRAY_LOAD, BRBMI_BKPT_FILL_8B 2356 2357 .balign 8 2358 .thumb_func 2359 .type art_quick_read_barrier_mark_introspection_intrinsic_cas, #function 2360 .hidden art_quick_read_barrier_mark_introspection_intrinsic_cas 2361 .global art_quick_read_barrier_mark_introspection_intrinsic_cas 2362art_quick_read_barrier_mark_introspection_intrinsic_cas: 2363 // Load the byte of the MOV instruction that contains Rd. Adjust for the thumb state in LR. 2364 // The MOV (register, T3) is |11101010010|S|1111|(0)000|Rd|0000|Rm|, so the byte we read 2365 // here, i.e. |(0)000|Rd|, contains only the register number, the top 4 bits are 0. 2366 ldrb rMR, [lr, #(-1 + BAKER_MARK_INTROSPECTION_INTRINSIC_CAS_MOV_OFFSET + 3)] 2367 b .Lmark_introspection_runtime_call 2368END art_quick_read_barrier_mark_introspection 2369#else // defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 2370ENTRY art_quick_read_barrier_mark_introspection 2371 bkpt // Unreachable. 2372END art_quick_read_barrier_mark_introspection 2373#endif // defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 2374 2375.extern artInvokePolymorphic 2376ENTRY art_quick_invoke_polymorphic 2377 SETUP_SAVE_REFS_AND_ARGS_FRAME r2 2378 mov r0, r1 @ r0 := receiver 2379 mov r1, rSELF @ r1 := Thread::Current 2380 mov r2, sp @ r2 := SP 2381 bl artInvokePolymorphic @ artInvokePolymorphic(receiver, Thread*, SP) 2382 str r1, [sp, 72] @ r0:r1 := Result. Copy r1 to context. 2383 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2384 REFRESH_MARKING_REGISTER 2385 vmov d0, r0, r1 @ Put result r0:r1 into floating point return register. 2386 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG r2 2387END art_quick_invoke_polymorphic 2388 2389.extern artInvokeCustom 2390ENTRY art_quick_invoke_custom 2391 SETUP_SAVE_REFS_AND_ARGS_FRAME r1 2392 @ r0 := call_site_idx 2393 mov r1, rSELF @ r1 := Thread::Current 2394 mov r2, sp @ r2 := SP 2395 bl artInvokeCustom @ artInvokeCustom(call_site_idx, Thread*, SP) 2396 str r1, [sp, #72] @ Save r1 to context (r0:r1 = result) 2397 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2398 REFRESH_MARKING_REGISTER 2399 vmov d0, r0, r1 @ Put result r0:r1 into floating point return register. 2400 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG r2 2401END art_quick_invoke_custom 2402 2403// Wrap ExecuteSwitchImpl in assembly method which specifies DEX PC for unwinding. 2404// Argument 0: r0: The context pointer for ExecuteSwitchImpl. 2405// Argument 1: r1: Pointer to the templated ExecuteSwitchImpl to call. 2406// Argument 2: r2: The value of DEX PC (memory address of the methods bytecode). 2407ENTRY ExecuteSwitchImplAsm 2408 push {r4, lr} // 2 words of callee saves. 2409 .cfi_adjust_cfa_offset 8 2410 .cfi_rel_offset r4, 0 2411 .cfi_rel_offset lr, 4 2412 mov r4, r2 // r4 = DEX PC 2413 CFI_DEFINE_DEX_PC_WITH_OFFSET(0 /* r0 */, 4 /* r4 */, 0) 2414 blx r1 // Call the wrapped method. 2415 pop {r4, pc} 2416END ExecuteSwitchImplAsm 2417 2418// r0 contains the class, r4 contains the inline cache. We can use ip as temporary. 2419ENTRY art_quick_update_inline_cache 2420#if (INLINE_CACHE_SIZE != 5) 2421#error "INLINE_CACHE_SIZE not as expected." 2422#endif 2423#if defined(USE_READ_BARRIER) && defined(USE_BAKER_READ_BARRIER) 2424 // Don't update the cache if we are marking. 2425 cmp rMR, #0 2426 bne .Ldone 2427#endif 2428.Lentry1: 2429 ldr ip, [r4, #INLINE_CACHE_CLASSES_OFFSET] 2430 cmp ip, r0 2431 beq .Ldone 2432 cmp ip, #0 2433 bne .Lentry2 2434 ldrex ip, [r4, #INLINE_CACHE_CLASSES_OFFSET] 2435 cmp ip, #0 2436 bne .Lentry1 2437 strex ip, r0, [r4, #INLINE_CACHE_CLASSES_OFFSET] 2438 cmp ip, #0 2439 bne .Ldone 2440 b .Lentry1 2441.Lentry2: 2442 ldr ip, [r4, #INLINE_CACHE_CLASSES_OFFSET+4] 2443 cmp ip, r0 2444 beq .Ldone 2445 cmp ip, #0 2446 bne .Lentry3 2447 ldrex ip, [r4, #INLINE_CACHE_CLASSES_OFFSET+4] 2448 cmp ip, #0 2449 bne .Lentry2 2450 strex ip, r0, [r4, #INLINE_CACHE_CLASSES_OFFSET+4] 2451 cmp ip, #0 2452 bne .Ldone 2453 b .Lentry2 2454.Lentry3: 2455 ldr ip, [r4, #INLINE_CACHE_CLASSES_OFFSET+8] 2456 cmp ip, r0 2457 beq .Ldone 2458 cmp ip, #0 2459 bne .Lentry4 2460 ldrex ip, [r4, #INLINE_CACHE_CLASSES_OFFSET+8] 2461 cmp ip, #0 2462 bne .Lentry3 2463 strex ip, r0, [r4, #INLINE_CACHE_CLASSES_OFFSET+8] 2464 cmp ip, #0 2465 bne .Ldone 2466 b .Lentry3 2467.Lentry4: 2468 ldr ip, [r4, #INLINE_CACHE_CLASSES_OFFSET+12] 2469 cmp ip, r0 2470 beq .Ldone 2471 cmp ip, #0 2472 bne .Lentry5 2473 ldrex ip, [r4, #INLINE_CACHE_CLASSES_OFFSET+12] 2474 cmp ip, #0 2475 bne .Lentry4 2476 strex ip, r0, [r4, #INLINE_CACHE_CLASSES_OFFSET+12] 2477 cmp ip, #0 2478 bne .Ldone 2479 b .Lentry4 2480.Lentry5: 2481 // Unconditionally store, the inline cache is megamorphic. 2482 str r0, [r4, #INLINE_CACHE_CLASSES_OFFSET+16] 2483.Ldone: 2484 blx lr 2485END art_quick_update_inline_cache 2486 2487// On entry, method is at the bottom of the stack. 2488ENTRY art_quick_compile_optimized 2489 SETUP_SAVE_EVERYTHING_FRAME r0 2490 ldr r0, [sp, FRAME_SIZE_SAVE_EVERYTHING] @ pass ArtMethod 2491 mov r1, rSELF @ pass Thread::Current 2492 bl artCompileOptimized @ (ArtMethod*, Thread*) 2493 RESTORE_SAVE_EVERYTHING_FRAME 2494 // We don't need to restore the marking register here, as 2495 // artCompileOptimized doesn't allow thread suspension. 2496 blx lr 2497END art_quick_compile_optimized 2498 2499// On entry, method is at the bottom of the stack. 2500ENTRY art_quick_method_entry_hook 2501 SETUP_SAVE_EVERYTHING_FRAME r0 2502 ldr r0, [sp, FRAME_SIZE_SAVE_EVERYTHING] @ pass ArtMethod 2503 mov r1, rSELF @ pass Thread::Current 2504 mov r2, sp @ pass SP 2505 bl artMethodEntryHook @ (ArtMethod*, Thread*, SP) 2506 RESTORE_SAVE_EVERYTHING_FRAME 2507 REFRESH_MARKING_REGISTER 2508 blx lr 2509END art_quick_method_entry_hook 2510 2511ENTRY art_quick_method_exit_hook 2512 SETUP_SAVE_EVERYTHING_FRAME r5 2513 2514 sub sp, #4 @ align stack 2515 push {r2} @ pass frame_size stack 2516 add r3, sp, #(8 + 8) @ store fpr_res pointer, in kSaveEverything frame 2517 add r2, sp, #(136 + 8) @ store gpr_res pointer, in kSaveEverything frame 2518 add r1, sp, #(FRAME_SIZE_SAVE_EVERYTHING + 8) @ pass ArtMethod** 2519 mov r0, rSELF @ pass Thread::Current 2520 blx artMethodExitHook @ (Thread*, ArtMethod**, gpr_res*, fpr_res*, 2521 @ frame_size) 2522 2523 add sp, #8 @ pop arguments on stack 2524 RESTORE_SAVE_EVERYTHING_FRAME 2525 REFRESH_MARKING_REGISTER 2526 blx lr 2527END art_quick_method_exit_hook 2528