1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "asm_support_arm.S" 18 19#include "arch/quick_alloc_entrypoints.S" 20 21 /* Deliver the given exception */ 22 .extern artDeliverExceptionFromCode 23 /* Deliver an exception pending on a thread */ 24 .extern artDeliverPendingException 25 26 /* 27 * Macro to spill the GPRs. 28 */ 29.macro SPILL_ALL_CALLEE_SAVE_GPRS 30 push {r4-r11, lr} @ 9 words (36 bytes) of callee saves. 31 .cfi_adjust_cfa_offset 36 32 .cfi_rel_offset r4, 0 33 .cfi_rel_offset r5, 4 34 .cfi_rel_offset r6, 8 35 .cfi_rel_offset r7, 12 36 .cfi_rel_offset r8, 16 37 .cfi_rel_offset r9, 20 38 .cfi_rel_offset r10, 24 39 .cfi_rel_offset r11, 28 40 .cfi_rel_offset lr, 32 41.endm 42 43 /* 44 * Macro that sets up the callee save frame to conform with 45 * Runtime::CreateCalleeSaveMethod(kSaveAllCalleeSaves) 46 */ 47.macro SETUP_SAVE_ALL_CALLEE_SAVES_FRAME rTemp 48 SPILL_ALL_CALLEE_SAVE_GPRS @ 9 words (36 bytes) of callee saves. 49 vpush {s16-s31} @ 16 words (64 bytes) of floats. 50 .cfi_adjust_cfa_offset 64 51 sub sp, #12 @ 3 words of space, bottom word will hold Method* 52 .cfi_adjust_cfa_offset 12 53 RUNTIME_CURRENT1 \rTemp @ Load Runtime::Current into rTemp. 54 @ Load kSaveAllCalleeSaves Method* into rTemp. 55 ldr \rTemp, [\rTemp, #RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET] 56 str \rTemp, [sp, #0] @ Place Method* at bottom of stack. 57 str sp, [r9, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. 58 59 // Ugly compile-time check, but we only have the preprocessor. 60#if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVES != 36 + 64 + 12) 61#error "FRAME_SIZE_SAVE_ALL_CALLEE_SAVES(ARM) size not as expected." 62#endif 63.endm 64 65 /* 66 * Macro that sets up the callee save frame to conform with 67 * Runtime::CreateCalleeSaveMethod(kSaveRefsOnly). 68 */ 69.macro SETUP_SAVE_REFS_ONLY_FRAME rTemp 70 push {r5-r8, r10-r11, lr} @ 7 words of callee saves 71 .cfi_adjust_cfa_offset 28 72 .cfi_rel_offset r5, 0 73 .cfi_rel_offset r6, 4 74 .cfi_rel_offset r7, 8 75 .cfi_rel_offset r8, 12 76 .cfi_rel_offset r10, 16 77 .cfi_rel_offset r11, 20 78 .cfi_rel_offset lr, 24 79 sub sp, #4 @ bottom word will hold Method* 80 .cfi_adjust_cfa_offset 4 81 RUNTIME_CURRENT2 \rTemp @ Load Runtime::Current into rTemp. 82 @ Load kSaveRefsOnly Method* into rTemp. 83 ldr \rTemp, [\rTemp, #RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET] 84 str \rTemp, [sp, #0] @ Place Method* at bottom of stack. 85 str sp, [r9, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. 86 87 // Ugly compile-time check, but we only have the preprocessor. 88#if (FRAME_SIZE_SAVE_REFS_ONLY != 28 + 4) 89#error "FRAME_SIZE_SAVE_REFS_ONLY(ARM) size not as expected." 90#endif 91.endm 92 93.macro RESTORE_SAVE_REFS_ONLY_FRAME 94 add sp, #4 @ bottom word holds Method* 95 .cfi_adjust_cfa_offset -4 96 pop {r5-r8, r10-r11, lr} @ 7 words of callee saves 97 .cfi_restore r5 98 .cfi_restore r6 99 .cfi_restore r7 100 .cfi_restore r8 101 .cfi_restore r10 102 .cfi_restore r11 103 .cfi_restore lr 104 .cfi_adjust_cfa_offset -28 105.endm 106 107.macro RESTORE_SAVE_REFS_ONLY_FRAME_AND_RETURN 108 RESTORE_SAVE_REFS_ONLY_FRAME 109 bx lr @ return 110.endm 111 112 /* 113 * Macro that sets up the callee save frame to conform with 114 * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs). 115 */ 116.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY 117 push {r1-r3, r5-r8, r10-r11, lr} @ 10 words of callee saves and args. 118 .cfi_adjust_cfa_offset 40 119 .cfi_rel_offset r1, 0 120 .cfi_rel_offset r2, 4 121 .cfi_rel_offset r3, 8 122 .cfi_rel_offset r5, 12 123 .cfi_rel_offset r6, 16 124 .cfi_rel_offset r7, 20 125 .cfi_rel_offset r8, 24 126 .cfi_rel_offset r10, 28 127 .cfi_rel_offset r11, 32 128 .cfi_rel_offset lr, 36 129 vpush {s0-s15} @ 16 words of float args. 130 .cfi_adjust_cfa_offset 64 131 sub sp, #8 @ 2 words of space, alignment padding and Method* 132 .cfi_adjust_cfa_offset 8 133 // Ugly compile-time check, but we only have the preprocessor. 134#if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 40 + 64 + 8) 135#error "FRAME_SIZE_SAVE_REFS_AND_ARGS(ARM) size not as expected." 136#endif 137.endm 138 139.macro SETUP_SAVE_REFS_AND_ARGS_FRAME rTemp 140 SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY 141 RUNTIME_CURRENT3 \rTemp @ Load Runtime::Current into rTemp. 142 @ Load kSaveRefsAndArgs Method* into rTemp. 143 ldr \rTemp, [\rTemp, #RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET] 144 str \rTemp, [sp, #0] @ Place Method* at bottom of stack. 145 str sp, [r9, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. 146.endm 147 148.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_R0 149 SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY 150 str r0, [sp, #0] @ Store ArtMethod* to bottom of stack. 151 str sp, [r9, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. 152.endm 153 154.macro RESTORE_SAVE_REFS_AND_ARGS_FRAME 155 add sp, #8 @ rewind sp 156 .cfi_adjust_cfa_offset -8 157 vpop {s0-s15} 158 .cfi_adjust_cfa_offset -64 159 pop {r1-r3, r5-r8, r10-r11, lr} @ 10 words of callee saves 160 .cfi_restore r1 161 .cfi_restore r2 162 .cfi_restore r3 163 .cfi_restore r5 164 .cfi_restore r6 165 .cfi_restore r7 166 .cfi_restore r8 167 .cfi_restore r10 168 .cfi_restore r11 169 .cfi_restore lr 170 .cfi_adjust_cfa_offset -40 171.endm 172 173 /* 174 * Macro that sets up the callee save frame to conform with 175 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 176 * when core registers are already saved. 177 */ 178.macro SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED rTemp 179 @ 14 words of callee saves and args already saved. 180 vpush {d0-d15} @ 32 words, 2 for each of the 16 saved doubles. 181 .cfi_adjust_cfa_offset 128 182 sub sp, #8 @ 2 words of space, alignment padding and Method* 183 .cfi_adjust_cfa_offset 8 184 RUNTIME_CURRENT1 \rTemp @ Load Runtime::Current into rTemp. 185 @ Load kSaveEverything Method* into rTemp. 186 ldr \rTemp, [\rTemp, #RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET] 187 str \rTemp, [sp, #0] @ Place Method* at bottom of stack. 188 str sp, [r9, #THREAD_TOP_QUICK_FRAME_OFFSET] @ Place sp in Thread::Current()->top_quick_frame. 189 190 // Ugly compile-time check, but we only have the preprocessor. 191#if (FRAME_SIZE_SAVE_EVERYTHING != 56 + 128 + 8) 192#error "FRAME_SIZE_SAVE_EVERYTHING(ARM) size not as expected." 193#endif 194.endm 195 196 /* 197 * Macro that sets up the callee save frame to conform with 198 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 199 */ 200.macro SETUP_SAVE_EVERYTHING_FRAME rTemp 201 push {r0-r12, lr} @ 14 words of callee saves and args. 202 .cfi_adjust_cfa_offset 56 203 .cfi_rel_offset r0, 0 204 .cfi_rel_offset r1, 4 205 .cfi_rel_offset r2, 8 206 .cfi_rel_offset r3, 12 207 .cfi_rel_offset r4, 16 208 .cfi_rel_offset r5, 20 209 .cfi_rel_offset r6, 24 210 .cfi_rel_offset r7, 28 211 .cfi_rel_offset r8, 32 212 .cfi_rel_offset r9, 36 213 .cfi_rel_offset r10, 40 214 .cfi_rel_offset r11, 44 215 .cfi_rel_offset ip, 48 216 .cfi_rel_offset lr, 52 217 SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED \rTemp 218.endm 219 220.macro RESTORE_SAVE_EVERYTHING_FRAME 221 add sp, #8 @ rewind sp 222 .cfi_adjust_cfa_offset -8 223 vpop {d0-d15} 224 .cfi_adjust_cfa_offset -128 225 pop {r0-r12, lr} @ 14 words of callee saves 226 .cfi_restore r0 227 .cfi_restore r1 228 .cfi_restore r2 229 .cfi_restore r3 230 .cfi_restore r5 231 .cfi_restore r6 232 .cfi_restore r7 233 .cfi_restore r8 234 .cfi_restore r9 235 .cfi_restore r10 236 .cfi_restore r11 237 .cfi_restore r12 238 .cfi_restore lr 239 .cfi_adjust_cfa_offset -56 240.endm 241 242.macro RESTORE_SAVE_EVERYTHING_FRAME_KEEP_R0 243 add sp, #8 @ rewind sp 244 .cfi_adjust_cfa_offset -8 245 vpop {d0-d15} 246 .cfi_adjust_cfa_offset -128 247 add sp, #4 @ skip r0 248 .cfi_adjust_cfa_offset -4 249 .cfi_restore r0 @ debugger can no longer restore caller's r0 250 pop {r1-r12, lr} @ 13 words of callee saves 251 .cfi_restore r1 252 .cfi_restore r2 253 .cfi_restore r3 254 .cfi_restore r5 255 .cfi_restore r6 256 .cfi_restore r7 257 .cfi_restore r8 258 .cfi_restore r9 259 .cfi_restore r10 260 .cfi_restore r11 261 .cfi_restore r12 262 .cfi_restore lr 263 .cfi_adjust_cfa_offset -52 264.endm 265 266.macro RETURN_IF_RESULT_IS_ZERO 267 cbnz r0, 1f @ result non-zero branch over 268 bx lr @ return 2691: 270.endm 271 272.macro RETURN_IF_RESULT_IS_NON_ZERO 273 cbz r0, 1f @ result zero branch over 274 bx lr @ return 2751: 276.endm 277 278 /* 279 * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending 280 * exception is Thread::Current()->exception_ when the runtime method frame is ready. 281 */ 282.macro DELIVER_PENDING_EXCEPTION_FRAME_READY 283 mov r0, r9 @ pass Thread::Current 284 bl artDeliverPendingExceptionFromCode @ artDeliverPendingExceptionFromCode(Thread*) 285.endm 286 287 /* 288 * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending 289 * exception is Thread::Current()->exception_. 290 */ 291.macro DELIVER_PENDING_EXCEPTION 292 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r0 @ save callee saves for throw 293 DELIVER_PENDING_EXCEPTION_FRAME_READY 294.endm 295 296.macro NO_ARG_RUNTIME_EXCEPTION c_name, cxx_name 297 .extern \cxx_name 298ENTRY \c_name 299 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r0 @ save all registers as basis for long jump context 300 mov r0, r9 @ pass Thread::Current 301 bl \cxx_name @ \cxx_name(Thread*) 302END \c_name 303.endm 304 305.macro NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name 306 .extern \cxx_name 307ENTRY \c_name 308 SETUP_SAVE_EVERYTHING_FRAME r0 @ save all registers as basis for long jump context 309 mov r0, r9 @ pass Thread::Current 310 bl \cxx_name @ \cxx_name(Thread*) 311END \c_name 312.endm 313 314.macro ONE_ARG_RUNTIME_EXCEPTION c_name, cxx_name 315 .extern \cxx_name 316ENTRY \c_name 317 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r1 @ save all registers as basis for long jump context 318 mov r1, r9 @ pass Thread::Current 319 bl \cxx_name @ \cxx_name(Thread*) 320END \c_name 321.endm 322 323.macro TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING c_name, cxx_name 324 .extern \cxx_name 325ENTRY \c_name 326 SETUP_SAVE_EVERYTHING_FRAME r2 @ save all registers as basis for long jump context 327 mov r2, r9 @ pass Thread::Current 328 bl \cxx_name @ \cxx_name(Thread*) 329END \c_name 330.endm 331 332.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_REG reg 333 ldr \reg, [r9, #THREAD_EXCEPTION_OFFSET] // Get exception field. 334 cbnz \reg, 1f 335 bx lr 3361: 337 DELIVER_PENDING_EXCEPTION 338.endm 339 340.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 341 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG r1 342.endm 343 344.macro RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 345 RETURN_IF_RESULT_IS_ZERO 346 DELIVER_PENDING_EXCEPTION 347.endm 348 349.macro RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 350 RETURN_IF_RESULT_IS_NON_ZERO 351 DELIVER_PENDING_EXCEPTION 352.endm 353 354// Macros taking opportunity of code similarities for downcalls. 355.macro ONE_ARG_REF_DOWNCALL name, entrypoint, return 356 .extern \entrypoint 357ENTRY \name 358 SETUP_SAVE_REFS_ONLY_FRAME r1 @ save callee saves in case of GC 359 mov r1, r9 @ pass Thread::Current 360 bl \entrypoint @ (uint32_t field_idx, Thread*) 361 RESTORE_SAVE_REFS_ONLY_FRAME 362 \return 363END \name 364.endm 365 366.macro TWO_ARG_REF_DOWNCALL name, entrypoint, return 367 .extern \entrypoint 368ENTRY \name 369 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 370 mov r2, r9 @ pass Thread::Current 371 bl \entrypoint @ (field_idx, Object*, Thread*) 372 RESTORE_SAVE_REFS_ONLY_FRAME 373 \return 374END \name 375.endm 376 377.macro THREE_ARG_REF_DOWNCALL name, entrypoint, return 378 .extern \entrypoint 379ENTRY \name 380 SETUP_SAVE_REFS_ONLY_FRAME r3 @ save callee saves in case of GC 381 mov r3, r9 @ pass Thread::Current 382 bl \entrypoint @ (field_idx, Object*, new_val, Thread*) 383 RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here 384 \return 385END \name 386.endm 387 388 /* 389 * Called by managed code, saves callee saves and then calls artThrowException 390 * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception. 391 */ 392ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode 393 394 /* 395 * Called by managed code to create and deliver a NullPointerException. 396 */ 397NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode 398 399 /* 400 * Call installed by a signal handler to create and deliver a NullPointerException. 401 */ 402 .extern art_quick_throw_null_pointer_exception_from_signal 403ENTRY art_quick_throw_null_pointer_exception_from_signal 404 // The fault handler pushes the gc map address, i.e. "return address", to stack 405 // and passes the fault address in LR. So we need to set up the CFI info accordingly. 406 .cfi_def_cfa_offset __SIZEOF_POINTER__ 407 .cfi_rel_offset lr, 0 408 push {r0-r12} @ 13 words of callee saves and args; LR already saved. 409 .cfi_adjust_cfa_offset 52 410 .cfi_rel_offset r0, 0 411 .cfi_rel_offset r1, 4 412 .cfi_rel_offset r2, 8 413 .cfi_rel_offset r3, 12 414 .cfi_rel_offset r4, 16 415 .cfi_rel_offset r5, 20 416 .cfi_rel_offset r6, 24 417 .cfi_rel_offset r7, 28 418 .cfi_rel_offset r8, 32 419 .cfi_rel_offset r9, 36 420 .cfi_rel_offset r10, 40 421 .cfi_rel_offset r11, 44 422 .cfi_rel_offset ip, 48 423 424 @ save all registers as basis for long jump context 425 SETUP_SAVE_EVERYTHING_FRAME_CORE_REGS_SAVED r1 426 mov r0, lr @ pass the fault address stored in LR by the fault handler. 427 mov r1, r9 @ pass Thread::Current 428 bl artThrowNullPointerExceptionFromSignal @ (Thread*) 429END art_quick_throw_null_pointer_exception_from_signal 430 431 /* 432 * Called by managed code to create and deliver an ArithmeticException. 433 */ 434NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_div_zero, artThrowDivZeroFromCode 435 436 /* 437 * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds 438 * index, arg2 holds limit. 439 */ 440TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_array_bounds, artThrowArrayBoundsFromCode 441 442 /* 443 * Called by managed code to create and deliver a StringIndexOutOfBoundsException 444 * as if thrown from a call to String.charAt(). Arg1 holds index, arg2 holds limit. 445 */ 446TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_string_bounds, artThrowStringBoundsFromCode 447 448 /* 449 * Called by managed code to create and deliver a StackOverflowError. 450 */ 451NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode 452 453 /* 454 * All generated callsites for interface invokes and invocation slow paths will load arguments 455 * as usual - except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 456 * the method_idx. This wrapper will save arg1-arg3, and call the appropriate C helper. 457 * NOTE: "this" is first visible argument of the target, and so can be found in arg1/r1. 458 * 459 * The helper will attempt to locate the target and return a 64-bit result in r0/r1 consisting 460 * of the target Method* in r0 and method->code_ in r1. 461 * 462 * If unsuccessful, the helper will return null/null. There will bea pending exception in the 463 * thread and we branch to another stub to deliver it. 464 * 465 * On success this wrapper will restore arguments and *jump* to the target, leaving the lr 466 * pointing back to the original caller. 467 */ 468.macro INVOKE_TRAMPOLINE_BODY cxx_name 469 .extern \cxx_name 470 SETUP_SAVE_REFS_AND_ARGS_FRAME r2 @ save callee saves in case allocation triggers GC 471 mov r2, r9 @ pass Thread::Current 472 mov r3, sp 473 bl \cxx_name @ (method_idx, this, Thread*, SP) 474 mov r12, r1 @ save Method*->code_ 475 RESTORE_SAVE_REFS_AND_ARGS_FRAME 476 cbz r0, 1f @ did we find the target? if not go to exception delivery 477 bx r12 @ tail call to target 4781: 479 DELIVER_PENDING_EXCEPTION 480.endm 481.macro INVOKE_TRAMPOLINE c_name, cxx_name 482ENTRY \c_name 483 INVOKE_TRAMPOLINE_BODY \cxx_name 484END \c_name 485.endm 486 487INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, artInvokeInterfaceTrampolineWithAccessCheck 488 489INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, artInvokeStaticTrampolineWithAccessCheck 490INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, artInvokeDirectTrampolineWithAccessCheck 491INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, artInvokeSuperTrampolineWithAccessCheck 492INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, artInvokeVirtualTrampolineWithAccessCheck 493 494 /* 495 * Quick invocation stub internal. 496 * On entry: 497 * r0 = method pointer 498 * r1 = argument array or null for no argument methods 499 * r2 = size of argument array in bytes 500 * r3 = (managed) thread pointer 501 * [sp] = JValue* result 502 * [sp + 4] = result_in_float 503 * [sp + 8] = core register argument array 504 * [sp + 12] = fp register argument array 505 * +-------------------------+ 506 * | uint32_t* fp_reg_args | 507 * | uint32_t* core_reg_args | 508 * | result_in_float | <- Caller frame 509 * | Jvalue* result | 510 * +-------------------------+ 511 * | lr | 512 * | r11 | 513 * | r9 | 514 * | r4 | <- r11 515 * +-------------------------+ 516 * | uint32_t out[n-1] | 517 * | : : | Outs 518 * | uint32_t out[0] | 519 * | StackRef<ArtMethod> | <- SP value=null 520 * +-------------------------+ 521 */ 522ENTRY art_quick_invoke_stub_internal 523 SPILL_ALL_CALLEE_SAVE_GPRS @ spill regs (9) 524 mov r11, sp @ save the stack pointer 525 .cfi_def_cfa_register r11 526 527 mov r9, r3 @ move managed thread pointer into r9 528 529 add r4, r2, #4 @ create space for method pointer in frame 530 sub r4, sp, r4 @ reserve & align *stack* to 16 bytes: native calling 531 and r4, #0xFFFFFFF0 @ convention only aligns to 8B, so we have to ensure ART 532 mov sp, r4 @ 16B alignment ourselves. 533 534 mov r4, r0 @ save method* 535 add r0, sp, #4 @ pass stack pointer + method ptr as dest for memcpy 536 bl memcpy @ memcpy (dest, src, bytes) 537 mov ip, #0 @ set ip to 0 538 str ip, [sp] @ store null for method* at bottom of frame 539 540 ldr ip, [r11, #48] @ load fp register argument array pointer 541 vldm ip, {s0-s15} @ copy s0 - s15 542 543 ldr ip, [r11, #44] @ load core register argument array pointer 544 mov r0, r4 @ restore method* 545 add ip, ip, #4 @ skip r0 546 ldm ip, {r1-r3} @ copy r1 - r3 547 548#ifdef ARM_R4_SUSPEND_FLAG 549 mov r4, #SUSPEND_CHECK_INTERVAL @ reset r4 to suspend check interval 550#endif 551 552 ldr ip, [r0, #ART_METHOD_QUICK_CODE_OFFSET_32] @ get pointer to the code 553 blx ip @ call the method 554 555 mov sp, r11 @ restore the stack pointer 556 .cfi_def_cfa_register sp 557 558 ldr r4, [sp, #40] @ load result_is_float 559 ldr r9, [sp, #36] @ load the result pointer 560 cmp r4, #0 561 ite eq 562 strdeq r0, [r9] @ store r0/r1 into result pointer 563 vstrne d0, [r9] @ store s0-s1/d0 into result pointer 564 565 pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} @ restore spill regs 566END art_quick_invoke_stub_internal 567 568 /* 569 * On stack replacement stub. 570 * On entry: 571 * r0 = stack to copy 572 * r1 = size of stack 573 * r2 = pc to call 574 * r3 = JValue* result 575 * [sp] = shorty 576 * [sp + 4] = thread 577 */ 578ENTRY art_quick_osr_stub 579 SPILL_ALL_CALLEE_SAVE_GPRS @ Spill regs (9) 580 mov r11, sp @ Save the stack pointer 581 mov r10, r1 @ Save size of stack 582 ldr r9, [r11, #40] @ Move managed thread pointer into r9 583 mov r8, r2 @ Save the pc to call 584 sub r7, sp, #12 @ Reserve space for stack pointer, 585 @ JValue* result, and ArtMethod* slot. 586 and r7, #0xFFFFFFF0 @ Align stack pointer 587 mov sp, r7 @ Update stack pointer 588 str r11, [sp, #4] @ Save old stack pointer 589 str r3, [sp, #8] @ Save JValue* result 590 mov ip, #0 591 str ip, [sp] @ Store null for ArtMethod* at bottom of frame 592 sub sp, sp, r1 @ Reserve space for callee stack 593 mov r2, r1 594 mov r1, r0 595 mov r0, sp 596 bl memcpy @ memcpy (dest r0, src r1, bytes r2) 597 bl .Losr_entry @ Call the method 598 ldr r10, [sp, #8] @ Restore JValue* result 599 ldr sp, [sp, #4] @ Restore saved stack pointer 600 ldr r4, [sp, #36] @ load shorty 601 ldrb r4, [r4, #0] @ load return type 602 cmp r4, #68 @ Test if result type char == 'D'. 603 beq .Losr_fp_result 604 cmp r4, #70 @ Test if result type char == 'F'. 605 beq .Losr_fp_result 606 strd r0, [r10] @ Store r0/r1 into result pointer 607 b .Losr_exit 608.Losr_fp_result: 609 vstr d0, [r10] @ Store s0-s1/d0 into result pointer 610.Losr_exit: 611 pop {r4, r5, r6, r7, r8, r9, r10, r11, pc} 612.Losr_entry: 613 sub r10, r10, #4 614 str lr, [sp, r10] @ Store link register per the compiler ABI 615 bx r8 616END art_quick_osr_stub 617 618 /* 619 * On entry r0 is uint32_t* gprs_ and r1 is uint32_t* fprs_ 620 */ 621ARM_ENTRY art_quick_do_long_jump 622 vldm r1, {s0-s31} @ load all fprs from argument fprs_ 623 ldr r2, [r0, #60] @ r2 = r15 (PC from gprs_ 60=4*15) 624 ldr r14, [r0, #56] @ (LR from gprs_ 56=4*14) 625 add r0, r0, #12 @ increment r0 to skip gprs_[0..2] 12=4*3 626 ldm r0, {r3-r13} @ load remaining gprs from argument gprs_ 627 ldr r0, [r0, #-12] @ load r0 value 628 mov r1, #0 @ clear result register r1 629 bx r2 @ do long jump 630END art_quick_do_long_jump 631 632 /* 633 * Entry from managed code that calls artHandleFillArrayDataFromCode and delivers exception on 634 * failure. 635 */ 636TWO_ARG_REF_DOWNCALL art_quick_handle_fill_data, artHandleFillArrayDataFromCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 637 638 /* 639 * Entry from managed code that calls artLockObjectFromCode, may block for GC. r0 holds the 640 * possibly null object to lock. 641 */ 642 .extern artLockObjectFromCode 643ENTRY art_quick_lock_object 644 cbz r0, .Lslow_lock 645.Lretry_lock: 646 ldr r2, [r9, #THREAD_ID_OFFSET] 647 ldrex r1, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 648 mov r3, r1 649 and r3, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED @ zero the gc bits 650 cbnz r3, .Lnot_unlocked @ already thin locked 651 @ unlocked case - r1: original lock word that's zero except for the read barrier bits. 652 orr r2, r1, r2 @ r2 holds thread id with count of 0 with preserved read barrier bits 653 strex r3, r2, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 654 cbnz r3, .Llock_strex_fail @ store failed, retry 655 dmb ish @ full (LoadLoad|LoadStore) memory barrier 656 bx lr 657.Lnot_unlocked: @ r1: original lock word, r2: thread_id with count of 0 and zero read barrier bits 658 lsr r3, r1, LOCK_WORD_STATE_SHIFT 659 cbnz r3, .Lslow_lock @ if either of the top two bits are set, go slow path 660 eor r2, r1, r2 @ lock_word.ThreadId() ^ self->ThreadId() 661 uxth r2, r2 @ zero top 16 bits 662 cbnz r2, .Lslow_lock @ lock word and self thread id's match -> recursive lock 663 @ else contention, go to slow path 664 mov r3, r1 @ copy the lock word to check count overflow. 665 and r3, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED @ zero the gc bits. 666 add r2, r3, #LOCK_WORD_THIN_LOCK_COUNT_ONE @ increment count in lock word placing in r2 to check overflow 667 lsr r3, r2, #LOCK_WORD_GC_STATE_SHIFT @ if the first gc state bit is set, we overflowed. 668 cbnz r3, .Lslow_lock @ if we overflow the count go slow path 669 add r2, r1, #LOCK_WORD_THIN_LOCK_COUNT_ONE @ increment count for real 670 strex r3, r2, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] @ strex necessary for read barrier bits 671 cbnz r3, .Llock_strex_fail @ strex failed, retry 672 bx lr 673.Llock_strex_fail: 674 b .Lretry_lock @ retry 675.Lslow_lock: 676 SETUP_SAVE_REFS_ONLY_FRAME r1 @ save callee saves in case we block 677 mov r1, r9 @ pass Thread::Current 678 bl artLockObjectFromCode @ (Object* obj, Thread*) 679 RESTORE_SAVE_REFS_ONLY_FRAME 680 RETURN_IF_RESULT_IS_ZERO 681 DELIVER_PENDING_EXCEPTION 682END art_quick_lock_object 683 684ENTRY art_quick_lock_object_no_inline 685 SETUP_SAVE_REFS_ONLY_FRAME r1 @ save callee saves in case we block 686 mov r1, r9 @ pass Thread::Current 687 bl artLockObjectFromCode @ (Object* obj, Thread*) 688 RESTORE_SAVE_REFS_ONLY_FRAME 689 RETURN_IF_RESULT_IS_ZERO 690 DELIVER_PENDING_EXCEPTION 691END art_quick_lock_object_no_inline 692 693 /* 694 * Entry from managed code that calls artUnlockObjectFromCode and delivers exception on failure. 695 * r0 holds the possibly null object to lock. 696 */ 697 .extern artUnlockObjectFromCode 698ENTRY art_quick_unlock_object 699 cbz r0, .Lslow_unlock 700.Lretry_unlock: 701#ifndef USE_READ_BARRIER 702 ldr r1, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 703#else 704 ldrex r1, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] @ Need to use atomic instructions for read barrier 705#endif 706 lsr r2, r1, #LOCK_WORD_STATE_SHIFT 707 cbnz r2, .Lslow_unlock @ if either of the top two bits are set, go slow path 708 ldr r2, [r9, #THREAD_ID_OFFSET] 709 mov r3, r1 @ copy lock word to check thread id equality 710 and r3, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED @ zero the gc bits 711 eor r3, r3, r2 @ lock_word.ThreadId() ^ self->ThreadId() 712 uxth r3, r3 @ zero top 16 bits 713 cbnz r3, .Lslow_unlock @ do lock word and self thread id's match? 714 mov r3, r1 @ copy lock word to detect transition to unlocked 715 and r3, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED @ zero the gc bits 716 cmp r3, #LOCK_WORD_THIN_LOCK_COUNT_ONE 717 bpl .Lrecursive_thin_unlock 718 @ transition to unlocked 719 mov r3, r1 720 and r3, #LOCK_WORD_GC_STATE_MASK_SHIFTED @ r3: zero except for the preserved gc bits 721 dmb ish @ full (LoadStore|StoreStore) memory barrier 722#ifndef USE_READ_BARRIER 723 str r3, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 724#else 725 strex r2, r3, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] @ strex necessary for read barrier bits 726 cbnz r2, .Lunlock_strex_fail @ store failed, retry 727#endif 728 bx lr 729.Lrecursive_thin_unlock: @ r1: original lock word 730 sub r1, r1, #LOCK_WORD_THIN_LOCK_COUNT_ONE @ decrement count 731#ifndef USE_READ_BARRIER 732 str r1, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] 733#else 734 strex r2, r1, [r0, #MIRROR_OBJECT_LOCK_WORD_OFFSET] @ strex necessary for read barrier bits 735 cbnz r2, .Lunlock_strex_fail @ store failed, retry 736#endif 737 bx lr 738.Lunlock_strex_fail: 739 b .Lretry_unlock @ retry 740.Lslow_unlock: 741 @ save callee saves in case exception allocation triggers GC 742 SETUP_SAVE_REFS_ONLY_FRAME r1 743 mov r1, r9 @ pass Thread::Current 744 bl artUnlockObjectFromCode @ (Object* obj, Thread*) 745 RESTORE_SAVE_REFS_ONLY_FRAME 746 RETURN_IF_RESULT_IS_ZERO 747 DELIVER_PENDING_EXCEPTION 748END art_quick_unlock_object 749 750ENTRY art_quick_unlock_object_no_inline 751 @ save callee saves in case exception allocation triggers GC 752 SETUP_SAVE_REFS_ONLY_FRAME r1 753 mov r1, r9 @ pass Thread::Current 754 bl artUnlockObjectFromCode @ (Object* obj, Thread*) 755 RESTORE_SAVE_REFS_ONLY_FRAME 756 RETURN_IF_RESULT_IS_ZERO 757 DELIVER_PENDING_EXCEPTION 758END art_quick_unlock_object_no_inline 759 760 /* 761 * Entry from managed code that calls artInstanceOfFromCode and on failure calls 762 * artThrowClassCastExceptionForObject. 763 */ 764 .extern artInstanceOfFromCode 765 .extern artThrowClassCastExceptionForObject 766ENTRY art_quick_check_instance_of 767 push {r0-r1, lr} @ save arguments, link register and pad 768 .cfi_adjust_cfa_offset 12 769 .cfi_rel_offset r0, 0 770 .cfi_rel_offset r1, 4 771 .cfi_rel_offset lr, 8 772 sub sp, #4 773 .cfi_adjust_cfa_offset 4 774 bl artInstanceOfFromCode 775 cbz r0, .Lthrow_class_cast_exception 776 add sp, #4 777 .cfi_adjust_cfa_offset -4 778 pop {r0-r1, pc} 779 .cfi_adjust_cfa_offset 4 @ Reset unwind info so following code unwinds. 780.Lthrow_class_cast_exception: 781 add sp, #4 782 .cfi_adjust_cfa_offset -4 783 pop {r0-r1, lr} 784 .cfi_adjust_cfa_offset -12 785 .cfi_restore r0 786 .cfi_restore r1 787 .cfi_restore lr 788 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r2 @ save all registers as basis for long jump context 789 mov r2, r9 @ pass Thread::Current 790 bl artThrowClassCastExceptionForObject @ (Object*, Class*, Thread*) 791 bkpt 792END art_quick_check_instance_of 793 794// Restore rReg's value from [sp, #offset] if rReg is not the same as rExclude. 795.macro POP_REG_NE rReg, offset, rExclude 796 .ifnc \rReg, \rExclude 797 ldr \rReg, [sp, #\offset] @ restore rReg 798 .cfi_restore \rReg 799 .endif 800.endm 801 802// Save rReg's value to [sp, #offset]. 803.macro PUSH_REG rReg, offset 804 str \rReg, [sp, #\offset] @ save rReg 805 .cfi_rel_offset \rReg, \offset 806.endm 807 808 /* 809 * Macro to insert read barrier, only used in art_quick_aput_obj. 810 * rObj and rDest are registers, offset is a defined literal such as MIRROR_OBJECT_CLASS_OFFSET. 811 * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. 812 */ 813.macro READ_BARRIER rDest, rObj, offset 814#ifdef USE_READ_BARRIER 815 push {r0-r3, ip, lr} @ 6 words for saved registers (used in art_quick_aput_obj) 816 .cfi_adjust_cfa_offset 24 817 .cfi_rel_offset r0, 0 818 .cfi_rel_offset r1, 4 819 .cfi_rel_offset r2, 8 820 .cfi_rel_offset r3, 12 821 .cfi_rel_offset ip, 16 822 .cfi_rel_offset lr, 20 823 sub sp, #8 @ push padding 824 .cfi_adjust_cfa_offset 8 825 @ mov r0, \rRef @ pass ref in r0 (no-op for now since parameter ref is unused) 826 .ifnc \rObj, r1 827 mov r1, \rObj @ pass rObj 828 .endif 829 mov r2, #\offset @ pass offset 830 bl artReadBarrierSlow @ artReadBarrierSlow(ref, rObj, offset) 831 @ No need to unpoison return value in r0, artReadBarrierSlow() would do the unpoisoning. 832 .ifnc \rDest, r0 833 mov \rDest, r0 @ save return value in rDest 834 .endif 835 add sp, #8 @ pop padding 836 .cfi_adjust_cfa_offset -8 837 POP_REG_NE r0, 0, \rDest @ conditionally restore saved registers 838 POP_REG_NE r1, 4, \rDest 839 POP_REG_NE r2, 8, \rDest 840 POP_REG_NE r3, 12, \rDest 841 POP_REG_NE ip, 16, \rDest 842 add sp, #20 843 .cfi_adjust_cfa_offset -20 844 pop {lr} @ restore lr 845 .cfi_adjust_cfa_offset -4 846 .cfi_restore lr 847#else 848 ldr \rDest, [\rObj, #\offset] 849 UNPOISON_HEAP_REF \rDest 850#endif // USE_READ_BARRIER 851.endm 852 853#ifdef USE_READ_BARRIER 854 .extern artReadBarrierSlow 855#endif 856 .hidden art_quick_aput_obj 857ENTRY art_quick_aput_obj 858#ifdef USE_READ_BARRIER 859 @ The offset to .Ldo_aput_null is too large to use cbz due to expansion from READ_BARRIER macro. 860 tst r2, r2 861 beq .Ldo_aput_null 862#else 863 cbz r2, .Ldo_aput_null 864#endif // USE_READ_BARRIER 865 READ_BARRIER r3, r0, MIRROR_OBJECT_CLASS_OFFSET 866 READ_BARRIER ip, r2, MIRROR_OBJECT_CLASS_OFFSET 867 READ_BARRIER r3, r3, MIRROR_CLASS_COMPONENT_TYPE_OFFSET 868 cmp r3, ip @ value's type == array's component type - trivial assignability 869 bne .Lcheck_assignability 870.Ldo_aput: 871 add r3, r0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 872 POISON_HEAP_REF r2 873 str r2, [r3, r1, lsl #2] 874 ldr r3, [r9, #THREAD_CARD_TABLE_OFFSET] 875 lsr r0, r0, #7 876 strb r3, [r3, r0] 877 blx lr 878.Ldo_aput_null: 879 add r3, r0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 880 str r2, [r3, r1, lsl #2] 881 blx lr 882.Lcheck_assignability: 883 push {r0-r2, lr} @ save arguments 884 .cfi_adjust_cfa_offset 16 885 .cfi_rel_offset r0, 0 886 .cfi_rel_offset r1, 4 887 .cfi_rel_offset r2, 8 888 .cfi_rel_offset lr, 12 889 mov r1, ip 890 mov r0, r3 891 bl artIsAssignableFromCode 892 cbz r0, .Lthrow_array_store_exception 893 pop {r0-r2, lr} 894 .cfi_restore r0 895 .cfi_restore r1 896 .cfi_restore r2 897 .cfi_restore lr 898 .cfi_adjust_cfa_offset -16 899 add r3, r0, #MIRROR_OBJECT_ARRAY_DATA_OFFSET 900 POISON_HEAP_REF r2 901 str r2, [r3, r1, lsl #2] 902 ldr r3, [r9, #THREAD_CARD_TABLE_OFFSET] 903 lsr r0, r0, #7 904 strb r3, [r3, r0] 905 blx lr 906.Lthrow_array_store_exception: 907 pop {r0-r2, lr} 908 /* No need to repeat restore cfi directives, the ones above apply here. */ 909 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r3 910 mov r1, r2 911 mov r2, r9 @ pass Thread::Current 912 bl artThrowArrayStoreException @ (Class*, Class*, Thread*) 913 bkpt @ unreached 914END art_quick_aput_obj 915 916// Macro to facilitate adding new allocation entrypoints. 917.macro ONE_ARG_DOWNCALL name, entrypoint, return 918 .extern \entrypoint 919ENTRY \name 920 SETUP_SAVE_REFS_ONLY_FRAME r1 @ save callee saves in case of GC 921 mov r1, r9 @ pass Thread::Current 922 bl \entrypoint @ (uint32_t type_idx, Method* method, Thread*) 923 RESTORE_SAVE_REFS_ONLY_FRAME 924 \return 925END \name 926.endm 927 928// Macro to facilitate adding new allocation entrypoints. 929.macro TWO_ARG_DOWNCALL name, entrypoint, return 930 .extern \entrypoint 931ENTRY \name 932 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 933 mov r2, r9 @ pass Thread::Current 934 bl \entrypoint @ (uint32_t type_idx, Method* method, Thread*) 935 RESTORE_SAVE_REFS_ONLY_FRAME 936 \return 937END \name 938.endm 939 940// Macro to facilitate adding new array allocation entrypoints. 941.macro THREE_ARG_DOWNCALL name, entrypoint, return 942 .extern \entrypoint 943ENTRY \name 944 SETUP_SAVE_REFS_ONLY_FRAME r3 @ save callee saves in case of GC 945 mov r3, r9 @ pass Thread::Current 946 @ (uint32_t type_idx, Method* method, int32_t component_count, Thread*) 947 bl \entrypoint 948 RESTORE_SAVE_REFS_ONLY_FRAME 949 \return 950END \name 951.endm 952 953// Macro to facilitate adding new allocation entrypoints. 954.macro FOUR_ARG_DOWNCALL name, entrypoint, return 955 .extern \entrypoint 956ENTRY \name 957 SETUP_SAVE_REFS_ONLY_FRAME r12 @ save callee saves in case of GC 958 str r9, [sp, #-16]! @ expand the frame and pass Thread::Current 959 .cfi_adjust_cfa_offset 16 960 bl \entrypoint 961 add sp, #16 @ strip the extra frame 962 .cfi_adjust_cfa_offset -16 963 RESTORE_SAVE_REFS_ONLY_FRAME 964 \return 965END \name 966.endm 967 968// Macro for string and type resolution and initialization. 969.macro ONE_ARG_SAVE_EVERYTHING_DOWNCALL name, entrypoint 970 .extern \entrypoint 971ENTRY \name 972 SETUP_SAVE_EVERYTHING_FRAME r1 @ save everything in case of GC 973 mov r1, r9 @ pass Thread::Current 974 bl \entrypoint @ (uint32_t index, Thread*) 975 cbz r0, 1f @ If result is null, deliver the OOME. 976 .cfi_remember_state 977 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_R0 978 bx lr 979 .cfi_restore_state 9801: 981 DELIVER_PENDING_EXCEPTION_FRAME_READY 982END \name 983.endm 984 985ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode 986ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode 987ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode 988ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode 989 990 /* 991 * Called by managed code to resolve a static field and load a non-wide value. 992 */ 993ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 994ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 995ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 996ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 997ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 998ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 999 /* 1000 * Called by managed code to resolve a static field and load a 64-bit primitive value. 1001 */ 1002 .extern artGet64StaticFromCompiledCode 1003ENTRY art_quick_get64_static 1004 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 1005 mov r1, r9 @ pass Thread::Current 1006 bl artGet64StaticFromCompiledCode @ (uint32_t field_idx, Thread*) 1007 ldr r2, [r9, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ 1008 RESTORE_SAVE_REFS_ONLY_FRAME 1009 cbnz r2, 1f @ success if no exception pending 1010 bx lr @ return on success 10111: 1012 DELIVER_PENDING_EXCEPTION 1013END art_quick_get64_static 1014 1015 /* 1016 * Called by managed code to resolve an instance field and load a non-wide value. 1017 */ 1018TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 1019TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 1020TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 1021TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 1022TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 1023TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DELIVER_PENDING_EXCEPTION_R1 1024 /* 1025 * Called by managed code to resolve an instance field and load a 64-bit primitive value. 1026 */ 1027 .extern artGet64InstanceFromCompiledCode 1028ENTRY art_quick_get64_instance 1029 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 1030 mov r2, r9 @ pass Thread::Current 1031 bl artGet64InstanceFromCompiledCode @ (field_idx, Object*, Thread*) 1032 ldr r2, [r9, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ 1033 RESTORE_SAVE_REFS_ONLY_FRAME 1034 cbnz r2, 1f @ success if no exception pending 1035 bx lr @ return on success 10361: 1037 DELIVER_PENDING_EXCEPTION 1038END art_quick_get64_instance 1039 1040 /* 1041 * Called by managed code to resolve a static field and store a value. 1042 */ 1043TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 1044TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 1045TWO_ARG_REF_DOWNCALL art_quick_set32_static, artSet32StaticFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 1046TWO_ARG_REF_DOWNCALL art_quick_set_obj_static, artSetObjStaticFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 1047 1048 /* 1049 * Called by managed code to resolve an instance field and store a non-wide value. 1050 */ 1051THREE_ARG_REF_DOWNCALL art_quick_set8_instance, artSet8InstanceFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 1052THREE_ARG_REF_DOWNCALL art_quick_set16_instance, artSet16InstanceFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 1053THREE_ARG_REF_DOWNCALL art_quick_set32_instance, artSet32InstanceFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 1054THREE_ARG_REF_DOWNCALL art_quick_set_obj_instance, artSetObjInstanceFromCompiledCode, RETURN_IF_RESULT_IS_ZERO_OR_DELIVER 1055 1056 /* 1057 * Called by managed code to resolve an instance field and store a wide value. 1058 */ 1059 .extern artSet64InstanceFromCompiledCode 1060ENTRY art_quick_set64_instance 1061 SETUP_SAVE_REFS_ONLY_FRAME r12 @ save callee saves in case of GC 1062 @ r2:r3 contain the wide argument 1063 str r9, [sp, #-16]! @ expand the frame and pass Thread::Current 1064 .cfi_adjust_cfa_offset 16 1065 bl artSet64InstanceFromCompiledCode @ (field_idx, Object*, new_val, Thread*) 1066 add sp, #16 @ release out args 1067 .cfi_adjust_cfa_offset -16 1068 RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here 1069 RETURN_IF_RESULT_IS_ZERO 1070 DELIVER_PENDING_EXCEPTION 1071END art_quick_set64_instance 1072 1073 .extern artSet64StaticFromCompiledCode 1074ENTRY art_quick_set64_static 1075 SETUP_SAVE_REFS_ONLY_FRAME r12 @ save callee saves in case of GC 1076 @ r2:r3 contain the wide argument 1077 str r9, [sp, #-16]! @ expand the frame and pass Thread::Current 1078 .cfi_adjust_cfa_offset 16 1079 bl artSet64StaticFromCompiledCode @ (field_idx, new_val, Thread*) 1080 add sp, #16 @ release out args 1081 .cfi_adjust_cfa_offset -16 1082 RESTORE_SAVE_REFS_ONLY_FRAME @ TODO: we can clearly save an add here 1083 RETURN_IF_RESULT_IS_ZERO 1084 DELIVER_PENDING_EXCEPTION 1085END art_quick_set64_static 1086 1087// Generate the allocation entrypoints for each allocator. 1088GENERATE_ALLOC_ENTRYPOINTS_FOR_NON_TLAB_ALLOCATORS 1089// Comment out allocators that have arm specific asm. 1090// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) 1091// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB) 1092GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) 1093// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB) 1094// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_region_tlab, RegionTLAB) 1095// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_region_tlab, RegionTLAB) 1096// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_region_tlab, RegionTLAB) 1097// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_region_tlab, RegionTLAB) 1098GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_region_tlab, RegionTLAB) 1099GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_region_tlab, RegionTLAB) 1100GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_region_tlab, RegionTLAB) 1101 1102// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB) 1103// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB) 1104GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_tlab, TLAB) 1105// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_tlab, TLAB) 1106// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_tlab, TLAB) 1107// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_tlab, TLAB) 1108// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_tlab, TLAB) 1109// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_tlab, TLAB) 1110GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_tlab, TLAB) 1111GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_tlab, TLAB) 1112GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_tlab, TLAB) 1113 1114// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_RESOLVED_OBJECT(_rosalloc, RosAlloc). 1115.macro ART_QUICK_ALLOC_OBJECT_ROSALLOC c_name, cxx_name 1116ENTRY \c_name 1117 // Fast path rosalloc allocation. 1118 // r0: type/return value, r9: Thread::Current 1119 // r1, r2, r3, r12: free. 1120 ldr r3, [r9, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] // Check if the thread local 1121 // allocation stack has room. 1122 // TODO: consider using ldrd. 1123 ldr r12, [r9, #THREAD_LOCAL_ALLOC_STACK_END_OFFSET] 1124 cmp r3, r12 1125 bhs .Lslow_path\c_name 1126 1127 ldr r3, [r0, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (r3) 1128 cmp r3, #ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE // Check if the size is for a thread 1129 // local allocation. Also does the 1130 // initialized and finalizable checks. 1131 bhs .Lslow_path\c_name 1132 // Compute the rosalloc bracket index 1133 // from the size. Since the size is 1134 // already aligned we can combine the 1135 // two shifts together. 1136 add r12, r9, r3, lsr #(ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT - POINTER_SIZE_SHIFT) 1137 // Subtract pointer size since ther 1138 // are no runs for 0 byte allocations 1139 // and the size is already aligned. 1140 // Load the rosalloc run (r12) 1141 ldr r12, [r12, #(THREAD_ROSALLOC_RUNS_OFFSET - __SIZEOF_POINTER__)] 1142 // Load the free list head (r3). This 1143 // will be the return val. 1144 ldr r3, [r12, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1145 cbz r3, .Lslow_path\c_name 1146 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber r0 and r1. 1147 ldr r1, [r3, #ROSALLOC_SLOT_NEXT_OFFSET] // Load the next pointer of the head 1148 // and update the list head with the 1149 // next pointer. 1150 str r1, [r12, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)] 1151 // Store the class pointer in the 1152 // header. This also overwrites the 1153 // next pointer. The offsets are 1154 // asserted to match. 1155#if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET 1156#error "Class pointer needs to overwrite next pointer." 1157#endif 1158 POISON_HEAP_REF r0 1159 str r0, [r3, #MIRROR_OBJECT_CLASS_OFFSET] 1160 // Fence. This is "ish" not "ishst" so 1161 // that it also ensures ordering of 1162 // the class status load with respect 1163 // to later accesses to the class 1164 // object. Alternatively we could use 1165 // "ishst" if we use load-acquire for 1166 // the object size load. 1167 // Needs to be done before pushing on 1168 // allocation since Heap::VisitObjects 1169 // relies on seeing the class pointer. 1170 // b/28790624 1171 dmb ish 1172 // Push the new object onto the thread 1173 // local allocation stack and 1174 // increment the thread local 1175 // allocation stack top. 1176 ldr r1, [r9, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1177 str r3, [r1], #COMPRESSED_REFERENCE_SIZE // (Increment r1 as a side effect.) 1178 str r1, [r9, #THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET] 1179 // Decrement the size of the free list 1180 ldr r1, [r12, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1181 sub r1, #1 1182 // TODO: consider combining this store 1183 // and the list head store above using 1184 // strd. 1185 str r1, [r12, #(ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)] 1186 1187 mov r0, r3 // Set the return value and return. 1188 bx lr 1189 1190.Lslow_path\c_name: 1191 SETUP_SAVE_REFS_ONLY_FRAME r2 @ save callee saves in case of GC 1192 mov r1, r9 @ pass Thread::Current 1193 bl \cxx_name @ (mirror::Class* cls, Thread*) 1194 RESTORE_SAVE_REFS_ONLY_FRAME 1195 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1196END \c_name 1197.endm 1198 1199ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_resolved_rosalloc, artAllocObjectFromCodeResolvedRosAlloc 1200ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_initialized_rosalloc, artAllocObjectFromCodeInitializedRosAlloc 1201 1202// The common fast path code for art_quick_alloc_object_resolved/initialized_tlab 1203// and art_quick_alloc_object_resolved/initialized_region_tlab. 1204// 1205// r0: type r9: Thread::Current, r1, r2, r3, r12: free. 1206// Need to preserve r0 to the slow path. 1207.macro ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH slowPathLabel 1208 // Load thread_local_pos (r12) and 1209 // thread_local_end (r3) with ldrd. 1210 // Check constraints for ldrd. 1211#if !((THREAD_LOCAL_POS_OFFSET + 4 == THREAD_LOCAL_END_OFFSET) && (THREAD_LOCAL_POS_OFFSET % 8 == 0)) 1212#error "Thread::thread_local_pos/end must be consecutive and are 8 byte aligned for performance" 1213#endif 1214 ldrd r12, r3, [r9, #THREAD_LOCAL_POS_OFFSET] 1215 sub r12, r3, r12 // Compute the remaining buf size. 1216 ldr r3, [r0, #MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET] // Load the object size (r3). 1217 cmp r3, r12 // Check if it fits. 1218 bhi \slowPathLabel 1219 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber r0 and r1. 1220 // Reload old thread_local_pos (r0) 1221 // for the return value. 1222 ldr r2, [r9, #THREAD_LOCAL_POS_OFFSET] 1223 add r1, r2, r3 1224 str r1, [r9, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1225 ldr r1, [r9, #THREAD_LOCAL_OBJECTS_OFFSET] // Increment thread_local_objects. 1226 add r1, r1, #1 1227 str r1, [r9, #THREAD_LOCAL_OBJECTS_OFFSET] 1228 POISON_HEAP_REF r0 1229 str r0, [r2, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1230 // Fence. This is "ish" not "ishst" so 1231 // that the code after this allocation 1232 // site will see the right values in 1233 // the fields of the class. 1234 // Alternatively we could use "ishst" 1235 // if we use load-acquire for the 1236 // object size load.) 1237 mov r0, r2 1238 dmb ish 1239 bx lr 1240.endm 1241 1242// The common code for art_quick_alloc_object_*region_tlab 1243.macro GENERATE_ALLOC_OBJECT_RESOLVED_TLAB name, entrypoint 1244ENTRY \name 1245 // Fast path tlab allocation. 1246 // r0: type, r9: Thread::Current 1247 // r1, r2, r3, r12: free. 1248 ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH .Lslow_path\name 1249.Lslow_path\name: 1250 SETUP_SAVE_REFS_ONLY_FRAME r2 // Save callee saves in case of GC. 1251 mov r1, r9 // Pass Thread::Current. 1252 bl \entrypoint // (mirror::Class* klass, Thread*) 1253 RESTORE_SAVE_REFS_ONLY_FRAME 1254 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1255END \name 1256.endm 1257 1258GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_resolved_region_tlab, artAllocObjectFromCodeResolvedRegionTLAB 1259GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_initialized_region_tlab, artAllocObjectFromCodeInitializedRegionTLAB 1260GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_resolved_tlab, artAllocObjectFromCodeResolvedTLAB 1261GENERATE_ALLOC_OBJECT_RESOLVED_TLAB art_quick_alloc_object_initialized_tlab, artAllocObjectFromCodeInitializedTLAB 1262 1263 1264// The common fast path code for art_quick_alloc_array_resolved/initialized_tlab 1265// and art_quick_alloc_array_resolved/initialized_region_tlab. 1266// 1267// r0: type r1: component_count r2: total_size r9: Thread::Current, r3, r12: free. 1268// Need to preserve r0 and r1 to the slow path. 1269.macro ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE slowPathLabel 1270 and r2, r2, #OBJECT_ALIGNMENT_MASK_TOGGLED // Apply alignemnt mask 1271 // (addr + 7) & ~7. 1272 1273 // Load thread_local_pos (r3) and 1274 // thread_local_end (r12) with ldrd. 1275 // Check constraints for ldrd. 1276#if !((THREAD_LOCAL_POS_OFFSET + 4 == THREAD_LOCAL_END_OFFSET) && (THREAD_LOCAL_POS_OFFSET % 8 == 0)) 1277#error "Thread::thread_local_pos/end must be consecutive and are 8 byte aligned for performance" 1278#endif 1279 ldrd r3, r12, [r9, #THREAD_LOCAL_POS_OFFSET] 1280 sub r12, r12, r3 // Compute the remaining buf size. 1281 cmp r2, r12 // Check if the total_size fits. 1282 bhi \slowPathLabel 1283 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber r0 and r1. 1284 add r2, r2, r3 1285 str r2, [r9, #THREAD_LOCAL_POS_OFFSET] // Store new thread_local_pos. 1286 ldr r2, [r9, #THREAD_LOCAL_OBJECTS_OFFSET] // Increment thread_local_objects. 1287 add r2, r2, #1 1288 str r2, [r9, #THREAD_LOCAL_OBJECTS_OFFSET] 1289 POISON_HEAP_REF r0 1290 str r0, [r3, #MIRROR_OBJECT_CLASS_OFFSET] // Store the class pointer. 1291 str r1, [r3, #MIRROR_ARRAY_LENGTH_OFFSET] // Store the array length. 1292 // Fence. This is "ish" not "ishst" so 1293 // that the code after this allocation 1294 // site will see the right values in 1295 // the fields of the class. 1296 // Alternatively we could use "ishst" 1297 // if we use load-acquire for the 1298 // object size load.) 1299 mov r0, r3 1300 dmb ish 1301 bx lr 1302.endm 1303 1304.macro GENERATE_ALLOC_ARRAY_TLAB name, entrypoint, size_setup 1305ENTRY \name 1306 // Fast path array allocation for region tlab allocation. 1307 // r0: mirror::Class* type 1308 // r1: int32_t component_count 1309 // r9: thread 1310 // r2, r3, r12: free. 1311 \size_setup .Lslow_path\name 1312 ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE .Lslow_path\name 1313.Lslow_path\name: 1314 // r0: mirror::Class* klass 1315 // r1: int32_t component_count 1316 // r2: Thread* self 1317 SETUP_SAVE_REFS_ONLY_FRAME r2 // save callee saves in case of GC 1318 mov r2, r9 // pass Thread::Current 1319 bl \entrypoint 1320 RESTORE_SAVE_REFS_ONLY_FRAME 1321 RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER 1322END \name 1323.endm 1324 1325.macro COMPUTE_ARRAY_SIZE_UNKNOWN slow_path 1326 bkpt // We should never enter here. 1327 // Code below is for reference. 1328 // Possibly a large object, go slow. 1329 // Also does negative array size check. 1330 movw r2, #((MIN_LARGE_OBJECT_THRESHOLD - MIRROR_WIDE_ARRAY_DATA_OFFSET) / 8) 1331 cmp r1, r2 1332 bhi \slow_path 1333 // Array classes are never finalizable 1334 // or uninitialized, no need to check. 1335 ldr r3, [r0, #MIRROR_CLASS_COMPONENT_TYPE_OFFSET] // Load component type 1336 UNPOISON_HEAP_REF r3 1337 ldr r3, [r3, #MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET] 1338 lsr r3, r3, #PRIMITIVE_TYPE_SIZE_SHIFT_SHIFT // Component size shift is in high 16 1339 // bits. 1340 lsl r2, r1, r3 // Calculate data size 1341 // Add array data offset and alignment. 1342 add r2, r2, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1343#if MIRROR_WIDE_ARRAY_DATA_OFFSET != MIRROR_INT_ARRAY_DATA_OFFSET + 4 1344#error Long array data offset must be 4 greater than int array data offset. 1345#endif 1346 1347 add r3, r3, #1 // Add 4 to the length only if the 1348 // component size shift is 3 1349 // (for 64 bit alignment). 1350 and r3, r3, #4 1351 add r2, r2, r3 1352.endm 1353 1354.macro COMPUTE_ARRAY_SIZE_8 slow_path 1355 // Possibly a large object, go slow. 1356 // Also does negative array size check. 1357 movw r2, #(MIN_LARGE_OBJECT_THRESHOLD - MIRROR_INT_ARRAY_DATA_OFFSET) 1358 cmp r1, r2 1359 bhi \slow_path 1360 // Add array data offset and alignment. 1361 add r2, r1, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1362.endm 1363 1364.macro COMPUTE_ARRAY_SIZE_16 slow_path 1365 // Possibly a large object, go slow. 1366 // Also does negative array size check. 1367 movw r2, #((MIN_LARGE_OBJECT_THRESHOLD - MIRROR_INT_ARRAY_DATA_OFFSET) / 2) 1368 cmp r1, r2 1369 bhi \slow_path 1370 lsl r2, r1, #1 1371 // Add array data offset and alignment. 1372 add r2, r2, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1373.endm 1374 1375.macro COMPUTE_ARRAY_SIZE_32 slow_path 1376 // Possibly a large object, go slow. 1377 // Also does negative array size check. 1378 movw r2, #((MIN_LARGE_OBJECT_THRESHOLD - MIRROR_INT_ARRAY_DATA_OFFSET) / 4) 1379 cmp r1, r2 1380 bhi \slow_path 1381 lsl r2, r1, #2 1382 // Add array data offset and alignment. 1383 add r2, r2, #(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1384.endm 1385 1386.macro COMPUTE_ARRAY_SIZE_64 slow_path 1387 // Possibly a large object, go slow. 1388 // Also does negative array size check. 1389 movw r2, #((MIN_LARGE_OBJECT_THRESHOLD - MIRROR_LONG_ARRAY_DATA_OFFSET) / 8) 1390 cmp r1, r2 1391 bhi \slow_path 1392 lsl r2, r1, #3 1393 // Add array data offset and alignment. 1394 add r2, r2, #(MIRROR_WIDE_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK) 1395.endm 1396 1397# TODO(ngeoffray): art_quick_alloc_array_resolved_region_tlab is not used for arm, remove 1398# the entrypoint once all backends have been updated to use the size variants. 1399GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN 1400GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_8 1401GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_16 1402GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_32 1403GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_64 1404GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN 1405GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_8 1406GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_16 1407GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_32 1408GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_64 1409 1410 /* 1411 * Called by managed code when the value in rSUSPEND has been decremented to 0. 1412 */ 1413 .extern artTestSuspendFromCode 1414ENTRY art_quick_test_suspend 1415#ifdef ARM_R4_SUSPEND_FLAG 1416 ldrh rSUSPEND, [rSELF, #THREAD_FLAGS_OFFSET] 1417 cbnz rSUSPEND, 1f @ check Thread::Current()->suspend_count_ == 0 1418 mov rSUSPEND, #SUSPEND_CHECK_INTERVAL @ reset rSUSPEND to SUSPEND_CHECK_INTERVAL 1419 bx lr @ return if suspend_count_ == 0 14201: 1421 mov rSUSPEND, #SUSPEND_CHECK_INTERVAL @ reset rSUSPEND to SUSPEND_CHECK_INTERVAL 1422#endif 1423 SETUP_SAVE_EVERYTHING_FRAME r0 @ save everything for GC stack crawl 1424 mov r0, rSELF 1425 bl artTestSuspendFromCode @ (Thread*) 1426 RESTORE_SAVE_EVERYTHING_FRAME 1427 bx lr 1428END art_quick_test_suspend 1429 1430ENTRY art_quick_implicit_suspend 1431 mov r0, rSELF 1432 SETUP_SAVE_REFS_ONLY_FRAME r1 @ save callee saves for stack crawl 1433 bl artTestSuspendFromCode @ (Thread*) 1434 RESTORE_SAVE_REFS_ONLY_FRAME_AND_RETURN 1435END art_quick_implicit_suspend 1436 1437 /* 1438 * Called by managed code that is attempting to call a method on a proxy class. On entry 1439 * r0 holds the proxy method and r1 holds the receiver; r2 and r3 may contain arguments. The 1440 * frame size of the invoked proxy method agrees with a ref and args callee save frame. 1441 */ 1442 .extern artQuickProxyInvokeHandler 1443ENTRY art_quick_proxy_invoke_handler 1444 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_R0 1445 mov r2, r9 @ pass Thread::Current 1446 mov r3, sp @ pass SP 1447 blx artQuickProxyInvokeHandler @ (Method* proxy method, receiver, Thread*, SP) 1448 ldr r2, [r9, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ 1449 // Tear down the callee-save frame. Skip arg registers. 1450 add sp, #(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) 1451 .cfi_adjust_cfa_offset -(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) 1452 RESTORE_SAVE_REFS_ONLY_FRAME 1453 cbnz r2, 1f @ success if no exception is pending 1454 vmov d0, r0, r1 @ store into fpr, for when it's a fpr return... 1455 bx lr @ return on success 14561: 1457 DELIVER_PENDING_EXCEPTION 1458END art_quick_proxy_invoke_handler 1459 1460 /* 1461 * Called to resolve an imt conflict. 1462 * r0 is the conflict ArtMethod. 1463 * r12 is a hidden argument that holds the target interface method's dex method index. 1464 * 1465 * Note that this stub writes to r0, r4, and r12. 1466 */ 1467ENTRY art_quick_imt_conflict_trampoline 1468 ldr r4, [sp, #0] // Load referrer 1469 ldr r4, [r4, #ART_METHOD_DEX_CACHE_METHODS_OFFSET_32] // Load dex cache methods array 1470 ldr r12, [r4, r12, lsl #POINTER_SIZE_SHIFT] // Load interface method 1471 ldr r0, [r0, #ART_METHOD_JNI_OFFSET_32] // Load ImtConflictTable 1472 ldr r4, [r0] // Load first entry in ImtConflictTable. 1473.Limt_table_iterate: 1474 cmp r4, r12 1475 // Branch if found. Benchmarks have shown doing a branch here is better. 1476 beq .Limt_table_found 1477 // If the entry is null, the interface method is not in the ImtConflictTable. 1478 cbz r4, .Lconflict_trampoline 1479 // Iterate over the entries of the ImtConflictTable. 1480 ldr r4, [r0, #(2 * __SIZEOF_POINTER__)]! 1481 b .Limt_table_iterate 1482.Limt_table_found: 1483 // We successfully hit an entry in the table. Load the target method 1484 // and jump to it. 1485 ldr r0, [r0, #__SIZEOF_POINTER__] 1486 ldr pc, [r0, #ART_METHOD_QUICK_CODE_OFFSET_32] 1487.Lconflict_trampoline: 1488 // Call the runtime stub to populate the ImtConflictTable and jump to the 1489 // resolved method. 1490 mov r0, r12 // Load interface method 1491 INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline 1492END art_quick_imt_conflict_trampoline 1493 1494 .extern artQuickResolutionTrampoline 1495ENTRY art_quick_resolution_trampoline 1496 SETUP_SAVE_REFS_AND_ARGS_FRAME r2 1497 mov r2, r9 @ pass Thread::Current 1498 mov r3, sp @ pass SP 1499 blx artQuickResolutionTrampoline @ (Method* called, receiver, Thread*, SP) 1500 cbz r0, 1f @ is code pointer null? goto exception 1501 mov r12, r0 1502 ldr r0, [sp, #0] @ load resolved method in r0 1503 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1504 bx r12 @ tail-call into actual code 15051: 1506 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1507 DELIVER_PENDING_EXCEPTION 1508END art_quick_resolution_trampoline 1509 1510 /* 1511 * Called to do a generic JNI down-call 1512 */ 1513ENTRY art_quick_generic_jni_trampoline 1514 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_R0 1515 1516 // Save rSELF 1517 mov r11, rSELF 1518 // Save SP , so we can have static CFI info. r10 is saved in ref_and_args. 1519 mov r10, sp 1520 .cfi_def_cfa_register r10 1521 1522 sub sp, sp, #5120 1523 1524 // prepare for artQuickGenericJniTrampoline call 1525 // (Thread*, SP) 1526 // r0 r1 <= C calling convention 1527 // rSELF r10 <= where they are 1528 1529 mov r0, rSELF // Thread* 1530 mov r1, r10 1531 blx artQuickGenericJniTrampoline // (Thread*, sp) 1532 1533 // The C call will have registered the complete save-frame on success. 1534 // The result of the call is: 1535 // r0: pointer to native code, 0 on error. 1536 // r1: pointer to the bottom of the used area of the alloca, can restore stack till there. 1537 1538 // Check for error = 0. 1539 cbz r0, .Lexception_in_native 1540 1541 // Release part of the alloca. 1542 mov sp, r1 1543 1544 // Save the code pointer 1545 mov r12, r0 1546 1547 // Load parameters from frame into registers. 1548 pop {r0-r3} 1549 1550 // Softfloat. 1551 // TODO: Change to hardfloat when supported. 1552 1553 blx r12 // native call. 1554 1555 // result sign extension is handled in C code 1556 // prepare for artQuickGenericJniEndTrampoline call 1557 // (Thread*, result, result_f) 1558 // r0 r2,r3 stack <= C calling convention 1559 // r11 r0,r1 r0,r1 <= where they are 1560 sub sp, sp, #8 // Stack alignment. 1561 1562 push {r0-r1} 1563 mov r3, r1 1564 mov r2, r0 1565 mov r0, r11 1566 1567 blx artQuickGenericJniEndTrampoline 1568 1569 // Restore self pointer. 1570 mov r9, r11 1571 1572 // Pending exceptions possible. 1573 ldr r2, [r9, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ 1574 cbnz r2, .Lexception_in_native 1575 1576 // Tear down the alloca. 1577 mov sp, r10 1578 .cfi_def_cfa_register sp 1579 1580 // Tear down the callee-save frame. Skip arg registers. 1581 add sp, #FRAME_SIZE_SAVE_REFS_AND_ARGS-FRAME_SIZE_SAVE_REFS_ONLY 1582 .cfi_adjust_cfa_offset -(FRAME_SIZE_SAVE_REFS_AND_ARGS-FRAME_SIZE_SAVE_REFS_ONLY) 1583 RESTORE_SAVE_REFS_ONLY_FRAME 1584 1585 // store into fpr, for when it's a fpr return... 1586 vmov d0, r0, r1 1587 bx lr // ret 1588 // Undo the unwinding information from above since it doesn't apply below. 1589 .cfi_def_cfa_register r10 1590 .cfi_adjust_cfa_offset FRAME_SIZE_SAVE_REFS_AND_ARGS-FRAME_SIZE_SAVE_REFS_ONLY 1591 1592.Lexception_in_native: 1593 ldr sp, [r9, #THREAD_TOP_QUICK_FRAME_OFFSET] 1594 .cfi_def_cfa_register sp 1595 # This will create a new save-all frame, required by the runtime. 1596 DELIVER_PENDING_EXCEPTION 1597END art_quick_generic_jni_trampoline 1598 1599 .extern artQuickToInterpreterBridge 1600ENTRY art_quick_to_interpreter_bridge 1601 SETUP_SAVE_REFS_AND_ARGS_FRAME r1 1602 mov r1, r9 @ pass Thread::Current 1603 mov r2, sp @ pass SP 1604 blx artQuickToInterpreterBridge @ (Method* method, Thread*, SP) 1605 ldr r2, [r9, #THREAD_EXCEPTION_OFFSET] @ load Thread::Current()->exception_ 1606 // Tear down the callee-save frame. Skip arg registers. 1607 add sp, #(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) 1608 .cfi_adjust_cfa_offset -(FRAME_SIZE_SAVE_REFS_AND_ARGS - FRAME_SIZE_SAVE_REFS_ONLY) 1609 RESTORE_SAVE_REFS_ONLY_FRAME 1610 cbnz r2, 1f @ success if no exception is pending 1611 vmov d0, r0, r1 @ store into fpr, for when it's a fpr return... 1612 bx lr @ return on success 16131: 1614 DELIVER_PENDING_EXCEPTION 1615END art_quick_to_interpreter_bridge 1616 1617/* 1618 * Called to attempt to execute an obsolete method. 1619 */ 1620ONE_ARG_RUNTIME_EXCEPTION art_invoke_obsolete_method_stub, artInvokeObsoleteMethod 1621 1622 /* 1623 * Routine that intercepts method calls and returns. 1624 */ 1625 .extern artInstrumentationMethodEntryFromCode 1626 .extern artInstrumentationMethodExitFromCode 1627ENTRY art_quick_instrumentation_entry 1628 @ Make stack crawlable and clobber r2 and r3 (post saving) 1629 SETUP_SAVE_REFS_AND_ARGS_FRAME r2 1630 @ preserve r0 (not normally an arg) knowing there is a spare slot in kSaveRefsAndArgs. 1631 str r0, [sp, #4] 1632 mov r2, r9 @ pass Thread::Current 1633 mov r3, lr @ pass LR 1634 blx artInstrumentationMethodEntryFromCode @ (Method*, Object*, Thread*, LR) 1635 mov r12, r0 @ r12 holds reference to code 1636 ldr r0, [sp, #4] @ restore r0 1637 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1638 blx r12 @ call method with lr set to art_quick_instrumentation_exit 1639@ Deliberate fall-through into art_quick_instrumentation_exit. 1640 .type art_quick_instrumentation_exit, #function 1641 .global art_quick_instrumentation_exit 1642art_quick_instrumentation_exit: 1643 mov lr, #0 @ link register is to here, so clobber with 0 for later checks 1644 SETUP_SAVE_REFS_ONLY_FRAME r2 @ set up frame knowing r2 and r3 must be dead on exit 1645 mov r12, sp @ remember bottom of caller's frame 1646 push {r0-r1} @ save return value 1647 .cfi_adjust_cfa_offset 8 1648 .cfi_rel_offset r0, 0 1649 .cfi_rel_offset r1, 4 1650 vpush {d0} @ save fp return value 1651 .cfi_adjust_cfa_offset 8 1652 sub sp, #8 @ space for return value argument. Note: AAPCS stack alignment is 8B, no 1653 @ need to align by 16. 1654 .cfi_adjust_cfa_offset 8 1655 vstr d0, [sp] @ d0 -> [sp] for fpr_res 1656 mov r2, r0 @ pass return value as gpr_res 1657 mov r3, r1 1658 mov r0, r9 @ pass Thread::Current 1659 mov r1, r12 @ pass SP 1660 blx artInstrumentationMethodExitFromCode @ (Thread*, SP, gpr_res, fpr_res) 1661 add sp, #8 1662 .cfi_adjust_cfa_offset -8 1663 1664 mov r2, r0 @ link register saved by instrumentation 1665 mov lr, r1 @ r1 is holding link register if we're to bounce to deoptimize 1666 vpop {d0} @ restore fp return value 1667 .cfi_adjust_cfa_offset -8 1668 pop {r0, r1} @ restore return value 1669 .cfi_adjust_cfa_offset -8 1670 .cfi_restore r0 1671 .cfi_restore r1 1672 add sp, #32 @ remove callee save frame 1673 .cfi_adjust_cfa_offset -32 1674 bx r2 @ return 1675END art_quick_instrumentation_entry 1676 1677 /* 1678 * Instrumentation has requested that we deoptimize into the interpreter. The deoptimization 1679 * will long jump to the upcall with a special exception of -1. 1680 */ 1681 .extern artDeoptimize 1682ENTRY art_quick_deoptimize 1683 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME r0 1684 mov r0, r9 @ pass Thread::Current 1685 blx artDeoptimize @ (Thread*) 1686END art_quick_deoptimize 1687 1688 /* 1689 * Compiled code has requested that we deoptimize into the interpreter. The deoptimization 1690 * will long jump to the interpreter bridge. 1691 */ 1692 .extern artDeoptimizeFromCompiledCode 1693ENTRY art_quick_deoptimize_from_compiled_code 1694 SETUP_SAVE_EVERYTHING_FRAME r1 1695 mov r1, r9 @ pass Thread::Current 1696 blx artDeoptimizeFromCompiledCode @ (DeoptimizationKind, Thread*) 1697END art_quick_deoptimize_from_compiled_code 1698 1699 /* 1700 * Signed 64-bit integer multiply. 1701 * 1702 * Consider WXxYZ (r1r0 x r3r2) with a long multiply: 1703 * WX 1704 * x YZ 1705 * -------- 1706 * ZW ZX 1707 * YW YX 1708 * 1709 * The low word of the result holds ZX, the high word holds 1710 * (ZW+YX) + (the high overflow from ZX). YW doesn't matter because 1711 * it doesn't fit in the low 64 bits. 1712 * 1713 * Unlike most ARM math operations, multiply instructions have 1714 * restrictions on using the same register more than once (Rd and Rm 1715 * cannot be the same). 1716 */ 1717 /* mul-long vAA, vBB, vCC */ 1718ENTRY art_quick_mul_long 1719 push {r9 - r10} 1720 .cfi_adjust_cfa_offset 8 1721 .cfi_rel_offset r9, 0 1722 .cfi_rel_offset r10, 4 1723 mul ip, r2, r1 @ ip<- ZxW 1724 umull r9, r10, r2, r0 @ r9/r10 <- ZxX 1725 mla r2, r0, r3, ip @ r2<- YxX + (ZxW) 1726 add r10, r2, r10 @ r10<- r10 + low(ZxW + (YxX)) 1727 mov r0,r9 1728 mov r1,r10 1729 pop {r9 - r10} 1730 .cfi_adjust_cfa_offset -8 1731 .cfi_restore r9 1732 .cfi_restore r10 1733 bx lr 1734END art_quick_mul_long 1735 1736 /* 1737 * Long integer shift. This is different from the generic 32/64-bit 1738 * binary operations because vAA/vBB are 64-bit but vCC (the shift 1739 * distance) is 32-bit. Also, Dalvik requires us to ignore all but the low 1740 * 6 bits. 1741 * On entry: 1742 * r0: low word 1743 * r1: high word 1744 * r2: shift count 1745 */ 1746 /* shl-long vAA, vBB, vCC */ 1747ARM_ENTRY art_quick_shl_long @ ARM code as thumb code requires spills 1748 and r2, r2, #63 @ r2<- r2 & 0x3f 1749 mov r1, r1, asl r2 @ r1<- r1 << r2 1750 rsb r3, r2, #32 @ r3<- 32 - r2 1751 orr r1, r1, r0, lsr r3 @ r1<- r1 | (r0 << (32-r2)) 1752 subs ip, r2, #32 @ ip<- r2 - 32 1753 movpl r1, r0, asl ip @ if r2 >= 32, r1<- r0 << (r2-32) 1754 mov r0, r0, asl r2 @ r0<- r0 << r2 1755 bx lr 1756END art_quick_shl_long 1757 1758 /* 1759 * Long integer shift. This is different from the generic 32/64-bit 1760 * binary operations because vAA/vBB are 64-bit but vCC (the shift 1761 * distance) is 32-bit. Also, Dalvik requires us to ignore all but the low 1762 * 6 bits. 1763 * On entry: 1764 * r0: low word 1765 * r1: high word 1766 * r2: shift count 1767 */ 1768 /* shr-long vAA, vBB, vCC */ 1769ARM_ENTRY art_quick_shr_long @ ARM code as thumb code requires spills 1770 and r2, r2, #63 @ r0<- r0 & 0x3f 1771 mov r0, r0, lsr r2 @ r0<- r2 >> r2 1772 rsb r3, r2, #32 @ r3<- 32 - r2 1773 orr r0, r0, r1, asl r3 @ r0<- r0 | (r1 << (32-r2)) 1774 subs ip, r2, #32 @ ip<- r2 - 32 1775 movpl r0, r1, asr ip @ if r2 >= 32, r0<-r1 >> (r2-32) 1776 mov r1, r1, asr r2 @ r1<- r1 >> r2 1777 bx lr 1778END art_quick_shr_long 1779 1780 /* 1781 * Long integer shift. This is different from the generic 32/64-bit 1782 * binary operations because vAA/vBB are 64-bit but vCC (the shift 1783 * distance) is 32-bit. Also, Dalvik requires us to ignore all but the low 1784 * 6 bits. 1785 * On entry: 1786 * r0: low word 1787 * r1: high word 1788 * r2: shift count 1789 */ 1790 /* ushr-long vAA, vBB, vCC */ 1791ARM_ENTRY art_quick_ushr_long @ ARM code as thumb code requires spills 1792 and r2, r2, #63 @ r0<- r0 & 0x3f 1793 mov r0, r0, lsr r2 @ r0<- r2 >> r2 1794 rsb r3, r2, #32 @ r3<- 32 - r2 1795 orr r0, r0, r1, asl r3 @ r0<- r0 | (r1 << (32-r2)) 1796 subs ip, r2, #32 @ ip<- r2 - 32 1797 movpl r0, r1, lsr ip @ if r2 >= 32, r0<-r1 >>> (r2-32) 1798 mov r1, r1, lsr r2 @ r1<- r1 >>> r2 1799 bx lr 1800END art_quick_ushr_long 1801 1802 /* 1803 * String's indexOf. 1804 * 1805 * On entry: 1806 * r0: string object (known non-null) 1807 * r1: char to match (known <= 0xFFFF) 1808 * r2: Starting offset in string data 1809 */ 1810ENTRY art_quick_indexof 1811 push {r4, r10-r11, lr} @ 4 words of callee saves 1812 .cfi_adjust_cfa_offset 16 1813 .cfi_rel_offset r4, 0 1814 .cfi_rel_offset r10, 4 1815 .cfi_rel_offset r11, 8 1816 .cfi_rel_offset lr, 12 1817#if (STRING_COMPRESSION_FEATURE) 1818 ldr r4, [r0, #MIRROR_STRING_COUNT_OFFSET] 1819#else 1820 ldr r3, [r0, #MIRROR_STRING_COUNT_OFFSET] 1821#endif 1822 add r0, #MIRROR_STRING_VALUE_OFFSET 1823#if (STRING_COMPRESSION_FEATURE) 1824 /* r4 count (with flag) and r3 holds actual length */ 1825 lsr r3, r4, #1 1826#endif 1827 /* Clamp start to [0..count] */ 1828 cmp r2, #0 1829 it lt 1830 movlt r2, #0 1831 cmp r2, r3 1832 it gt 1833 movgt r2, r3 1834 1835 /* Save a copy in r12 to later compute result */ 1836 mov r12, r0 1837 1838 /* Build pointer to start of data to compare and pre-bias */ 1839#if (STRING_COMPRESSION_FEATURE) 1840 lsrs r4, r4, #1 1841 bcc .Lstring_indexof_compressed 1842#endif 1843 add r0, r0, r2, lsl #1 1844 sub r0, #2 1845 1846 /* Compute iteration count */ 1847 sub r2, r3, r2 1848 1849 /* 1850 * At this point we have: 1851 * r0: start of data to test 1852 * r1: char to compare 1853 * r2: iteration count 1854 * r4: compression style (used temporarily) 1855 * r12: original start of string data 1856 * r3, r4, r10, r11 available for loading string data 1857 */ 1858 1859 subs r2, #4 1860 blt .Lindexof_remainder 1861 1862.Lindexof_loop4: 1863 ldrh r3, [r0, #2]! 1864 ldrh r4, [r0, #2]! 1865 ldrh r10, [r0, #2]! 1866 ldrh r11, [r0, #2]! 1867 cmp r3, r1 1868 beq .Lmatch_0 1869 cmp r4, r1 1870 beq .Lmatch_1 1871 cmp r10, r1 1872 beq .Lmatch_2 1873 cmp r11, r1 1874 beq .Lmatch_3 1875 subs r2, #4 1876 bge .Lindexof_loop4 1877 1878.Lindexof_remainder: 1879 adds r2, #4 1880 beq .Lindexof_nomatch 1881 1882.Lindexof_loop1: 1883 ldrh r3, [r0, #2]! 1884 cmp r3, r1 1885 beq .Lmatch_3 1886 subs r2, #1 1887 bne .Lindexof_loop1 1888 1889.Lindexof_nomatch: 1890 mov r0, #-1 1891 pop {r4, r10-r11, pc} 1892 1893.Lmatch_0: 1894 sub r0, #6 1895 sub r0, r12 1896 asr r0, r0, #1 1897 pop {r4, r10-r11, pc} 1898.Lmatch_1: 1899 sub r0, #4 1900 sub r0, r12 1901 asr r0, r0, #1 1902 pop {r4, r10-r11, pc} 1903.Lmatch_2: 1904 sub r0, #2 1905 sub r0, r12 1906 asr r0, r0, #1 1907 pop {r4, r10-r11, pc} 1908.Lmatch_3: 1909 sub r0, r12 1910 asr r0, r0, #1 1911 pop {r4, r10-r11, pc} 1912#if (STRING_COMPRESSION_FEATURE) 1913.Lstring_indexof_compressed: 1914 add r0, r0, r2 1915 sub r0, #1 1916 sub r2, r3, r2 1917.Lstring_indexof_compressed_loop: 1918 subs r2, #1 1919 blt .Lindexof_nomatch 1920 ldrb r3, [r0, #1]! 1921 cmp r3, r1 1922 beq .Lstring_indexof_compressed_matched 1923 b .Lstring_indexof_compressed_loop 1924.Lstring_indexof_compressed_matched: 1925 sub r0, r12 1926 pop {r4, r10-r11, pc} 1927#endif 1928END art_quick_indexof 1929 1930 /* Assembly routines used to handle ABI differences. */ 1931 1932 /* double fmod(double a, double b) */ 1933 .extern fmod 1934ENTRY art_quick_fmod 1935 push {lr} 1936 .cfi_adjust_cfa_offset 4 1937 .cfi_rel_offset lr, 0 1938 sub sp, #4 1939 .cfi_adjust_cfa_offset 4 1940 vmov r0, r1, d0 1941 vmov r2, r3, d1 1942 bl fmod 1943 vmov d0, r0, r1 1944 add sp, #4 1945 .cfi_adjust_cfa_offset -4 1946 pop {pc} 1947END art_quick_fmod 1948 1949 /* float fmodf(float a, float b) */ 1950 .extern fmodf 1951ENTRY art_quick_fmodf 1952 push {lr} 1953 .cfi_adjust_cfa_offset 4 1954 .cfi_rel_offset lr, 0 1955 sub sp, #4 1956 .cfi_adjust_cfa_offset 4 1957 vmov r0, r1, d0 1958 bl fmodf 1959 vmov s0, r0 1960 add sp, #4 1961 .cfi_adjust_cfa_offset -4 1962 pop {pc} 1963END art_quick_fmodf 1964 1965 /* int64_t art_d2l(double d) */ 1966 .extern art_d2l 1967ENTRY art_quick_d2l 1968 vmov r0, r1, d0 1969 b art_d2l 1970END art_quick_d2l 1971 1972 /* int64_t art_f2l(float f) */ 1973 .extern art_f2l 1974ENTRY art_quick_f2l 1975 vmov r0, s0 1976 b art_f2l 1977END art_quick_f2l 1978 1979 /* float art_l2f(int64_t l) */ 1980 .extern art_l2f 1981ENTRY art_quick_l2f 1982 push {lr} 1983 .cfi_adjust_cfa_offset 4 1984 .cfi_rel_offset lr, 0 1985 sub sp, #4 1986 .cfi_adjust_cfa_offset 4 1987 bl art_l2f 1988 vmov s0, r0 1989 add sp, #4 1990 .cfi_adjust_cfa_offset -4 1991 pop {pc} 1992END art_quick_l2f 1993 1994.macro CONDITIONAL_CBZ reg, reg_if, dest 1995.ifc \reg, \reg_if 1996 cbz \reg, \dest 1997.endif 1998.endm 1999 2000.macro CONDITIONAL_CMPBZ reg, reg_if, dest 2001.ifc \reg, \reg_if 2002 cmp \reg, #0 2003 beq \dest 2004.endif 2005.endm 2006 2007// Use CBZ if the register is in {r0, r7} otherwise compare and branch. 2008.macro SMART_CBZ reg, dest 2009 CONDITIONAL_CBZ \reg, r0, \dest 2010 CONDITIONAL_CBZ \reg, r1, \dest 2011 CONDITIONAL_CBZ \reg, r2, \dest 2012 CONDITIONAL_CBZ \reg, r3, \dest 2013 CONDITIONAL_CBZ \reg, r4, \dest 2014 CONDITIONAL_CBZ \reg, r5, \dest 2015 CONDITIONAL_CBZ \reg, r6, \dest 2016 CONDITIONAL_CBZ \reg, r7, \dest 2017 CONDITIONAL_CMPBZ \reg, r8, \dest 2018 CONDITIONAL_CMPBZ \reg, r9, \dest 2019 CONDITIONAL_CMPBZ \reg, r10, \dest 2020 CONDITIONAL_CMPBZ \reg, r11, \dest 2021 CONDITIONAL_CMPBZ \reg, r12, \dest 2022 CONDITIONAL_CMPBZ \reg, r13, \dest 2023 CONDITIONAL_CMPBZ \reg, r14, \dest 2024 CONDITIONAL_CMPBZ \reg, r15, \dest 2025.endm 2026 2027 /* 2028 * Create a function `name` calling the ReadBarrier::Mark routine, 2029 * getting its argument and returning its result through register 2030 * `reg`, saving and restoring all caller-save registers. 2031 * 2032 * IP is clobbered; `reg` must not be IP. 2033 * 2034 * If `reg` is different from `r0`, the generated function follows a 2035 * non-standard runtime calling convention: 2036 * - register `reg` is used to pass the (sole) argument of this 2037 * function (instead of R0); 2038 * - register `reg` is used to return the result of this function 2039 * (instead of R0); 2040 * - R0 is treated like a normal (non-argument) caller-save register; 2041 * - everything else is the same as in the standard runtime calling 2042 * convention (e.g. standard callee-save registers are preserved). 2043 */ 2044.macro READ_BARRIER_MARK_REG name, reg 2045ENTRY \name 2046 // Null check so that we can load the lock word. 2047 SMART_CBZ \reg, .Lret_rb_\name 2048 // Check lock word for mark bit, if marked return. Use IP for scratch since it is blocked. 2049 ldr ip, [\reg, MIRROR_OBJECT_LOCK_WORD_OFFSET] 2050 tst ip, #LOCK_WORD_MARK_BIT_MASK_SHIFTED 2051 beq .Lnot_marked_rb_\name 2052 // Already marked, return right away. 2053.Lret_rb_\name: 2054 bx lr 2055 2056.Lnot_marked_rb_\name: 2057 // Test that both the forwarding state bits are 1. 2058#if (LOCK_WORD_STATE_SHIFT != 30) || (LOCK_WORD_STATE_FORWARDING_ADDRESS != 3) 2059 // To use "CMP ip, #modified-immediate; BHS", we need the lock word state in 2060 // the highest bits and the "forwarding address" state to have all bits set. 2061#error "Unexpected lock word state shift or forwarding address state value." 2062#endif 2063 cmp ip, #(LOCK_WORD_STATE_FORWARDING_ADDRESS << LOCK_WORD_STATE_SHIFT) 2064 bhs .Lret_forwarding_address\name 2065 2066.Lslow_rb_\name: 2067 // Save IP: The kSaveEverything entrypoint art_quick_resolve_string used to 2068 // make a tail call here. Currently, it serves only for stack alignment but 2069 // we may reintroduce kSaveEverything calls here in the future. 2070 push {r0-r4, r9, ip, lr} @ save return address, core caller-save registers and ip 2071 .cfi_adjust_cfa_offset 32 2072 .cfi_rel_offset r0, 0 2073 .cfi_rel_offset r1, 4 2074 .cfi_rel_offset r2, 8 2075 .cfi_rel_offset r3, 12 2076 .cfi_rel_offset r4, 16 2077 .cfi_rel_offset r9, 20 2078 .cfi_rel_offset ip, 24 2079 .cfi_rel_offset lr, 28 2080 2081 .ifnc \reg, r0 2082 mov r0, \reg @ pass arg1 - obj from `reg` 2083 .endif 2084 2085 vpush {s0-s15} @ save floating-point caller-save registers 2086 .cfi_adjust_cfa_offset 64 2087 bl artReadBarrierMark @ r0 <- artReadBarrierMark(obj) 2088 vpop {s0-s15} @ restore floating-point registers 2089 .cfi_adjust_cfa_offset -64 2090 2091 .ifc \reg, r0 @ Save result to the stack slot or destination register. 2092 str r0, [sp, #0] 2093 .else 2094 .ifc \reg, r1 2095 str r0, [sp, #4] 2096 .else 2097 .ifc \reg, r2 2098 str r0, [sp, #8] 2099 .else 2100 .ifc \reg, r3 2101 str r0, [sp, #12] 2102 .else 2103 .ifc \reg, r4 2104 str r0, [sp, #16] 2105 .else 2106 .ifc \reg, r9 2107 str r0, [sp, #20] 2108 .else 2109 mov \reg, r0 2110 .endif 2111 .endif 2112 .endif 2113 .endif 2114 .endif 2115 .endif 2116 2117 pop {r0-r4, r9, ip, lr} @ restore caller-save registers 2118 .cfi_adjust_cfa_offset -32 2119 .cfi_restore r0 2120 .cfi_restore r1 2121 .cfi_restore r2 2122 .cfi_restore r3 2123 .cfi_restore r4 2124 .cfi_restore r9 2125 .cfi_restore ip 2126 .cfi_restore lr 2127 bx lr 2128.Lret_forwarding_address\name: 2129 // Shift left by the forwarding address shift. This clears out the state bits since they are 2130 // in the top 2 bits of the lock word. 2131 lsl \reg, ip, #LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 2132 bx lr 2133END \name 2134.endm 2135 2136READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg00, r0 2137READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, r1 2138READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, r2 2139READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, r3 2140READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg04, r4 2141READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, r5 2142READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, r6 2143READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, r7 2144READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, r8 2145READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, r9 2146READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, r10 2147READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, r11 2148 2149.extern artInvokePolymorphic 2150ENTRY art_quick_invoke_polymorphic 2151 SETUP_SAVE_REFS_AND_ARGS_FRAME r2 2152 mov r2, r9 @ pass Thread::Current 2153 mov r3, sp @ pass SP 2154 mov r0, #0 @ initialize 64-bit JValue as zero. 2155 str r0, [sp, #-4]! 2156 .cfi_adjust_cfa_offset 4 2157 str r0, [sp, #-4]! 2158 .cfi_adjust_cfa_offset 4 2159 mov r0, sp @ pass JValue for return result as first argument. 2160 bl artInvokePolymorphic @ artInvokePolymorphic(JValue, receiver, Thread*, SP) 2161 sub r0, 'A' @ return value is descriptor of handle's return type. 2162 cmp r0, 'Z' - 'A' @ check if value is in bounds of handler table 2163 bgt .Lcleanup_and_return @ and clean-up if not. 2164 adr r1, .Lhandler_table 2165 tbb [r0, r1] @ branch to handler for return value based on return type. 2166 2167.Lstart_of_handlers: 2168.Lstore_boolean_result: 2169 ldrb r0, [sp] @ Copy boolean value to return value of this function. 2170 b .Lcleanup_and_return 2171.Lstore_char_result: 2172 ldrh r0, [sp] @ Copy char value to return value of this function. 2173 b .Lcleanup_and_return 2174.Lstore_float_result: 2175 vldr s0, [sp] @ Copy float value from JValue result to the context restored by 2176 vstr s0, [sp, #16] @ RESTORE_SAVE_REFS_AND_ARGS_FRAME. 2177 b .Lcleanup_and_return 2178.Lstore_double_result: 2179 vldr d0, [sp] @ Copy double value from JValue result to the context restored by 2180 vstr d0, [sp, #16] @ RESTORE_SAVE_REFS_AND_ARGS_FRAME. 2181 b .Lcleanup_and_return 2182.Lstore_long_result: 2183 ldr r1, [sp, #4] @ Copy the upper bits from JValue result to the context restored by 2184 str r1, [sp, #80] @ RESTORE_SAVE_REFS_AND_ARGS_FRAME. 2185 // Fall-through for lower bits. 2186.Lstore_int_result: 2187 ldr r0, [sp] @ Copy int value to return value of this function. 2188 // Fall-through to clean up and return. 2189.Lcleanup_and_return: 2190 add sp, #8 2191 .cfi_adjust_cfa_offset -8 2192 RESTORE_SAVE_REFS_AND_ARGS_FRAME 2193 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG r2 2194 2195.macro HANDLER_TABLE_OFFSET handler_label 2196 .byte (\handler_label - .Lstart_of_handlers) / 2 2197.endm 2198 2199.Lhandler_table: 2200 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // A 2201 HANDLER_TABLE_OFFSET(.Lstore_int_result) // B (byte) 2202 HANDLER_TABLE_OFFSET(.Lstore_char_result) // C (char) 2203 HANDLER_TABLE_OFFSET(.Lstore_double_result) // D (double) 2204 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // E 2205 HANDLER_TABLE_OFFSET(.Lstore_float_result) // F (float) 2206 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // G 2207 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // H 2208 HANDLER_TABLE_OFFSET(.Lstore_int_result) // I (int) 2209 HANDLER_TABLE_OFFSET(.Lstore_long_result) // J (long) 2210 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // K 2211 HANDLER_TABLE_OFFSET(.Lstore_int_result) // L (object) 2212 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // M 2213 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // N 2214 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // O 2215 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // P 2216 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // Q 2217 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // R 2218 HANDLER_TABLE_OFFSET(.Lstore_int_result) // S (short) 2219 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // T 2220 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // U 2221 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // V (void) 2222 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // W 2223 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // X 2224 HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // Y 2225 HANDLER_TABLE_OFFSET(.Lstore_boolean_result) // Z (boolean) 2226.purgem HANDLER_TABLE_OFFSET 2227END art_quick_invoke_polymorphic 2228