1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "asm_support_x86_64.S" 18#include "interpreter/cfi_asm_support.h" 19 20#include "arch/quick_alloc_entrypoints.S" 21 22MACRO0(ASSERT_USE_READ_BARRIER) 23#if !defined(USE_READ_BARRIER) 24 int3 25 int3 26#endif 27END_MACRO 28 29// For x86, the CFA is esp+4, the address above the pushed return address on the stack. 30 31 32 /* 33 * Macro that sets up the callee save frame to conform with 34 * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs) 35 */ 36MACRO0(SETUP_SAVE_REFS_AND_ARGS_FRAME) 37#if defined(__APPLE__) 38 int3 39 int3 40#else 41 SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY 42 // R10 := Runtime::Current() 43 LOAD_RUNTIME_INSTANCE r10 44 // R10 := ArtMethod* for ref and args callee save frame method. 45 movq RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET(%r10), %r10 46 // Store ArtMethod* to bottom of stack. 47 movq %r10, 0(%rsp) 48 // Store rsp as the top quick frame. 49 movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET 50#endif // __APPLE__ 51END_MACRO 52 53MACRO0(SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_RDI) 54 SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY 55 // Store ArtMethod to bottom of stack. 56 movq %rdi, 0(%rsp) 57 // Store rsp as the stop quick frame. 58 movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET 59END_MACRO 60 61 /* 62 * Macro that sets up the callee save frame to conform with 63 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 64 * when R14 and R15 are already saved. 65 */ 66MACRO1(SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) 67#if defined(__APPLE__) 68 int3 69 int3 70#else 71 // Save core registers from highest to lowest to agree with core spills bitmap. 72 // R14 and R15, or at least placeholders for them, are already on the stack. 73 PUSH r13 74 PUSH r12 75 PUSH r11 76 PUSH r10 77 PUSH r9 78 PUSH r8 79 PUSH rdi 80 PUSH rsi 81 PUSH rbp 82 PUSH rbx 83 PUSH rdx 84 PUSH rcx 85 PUSH rax 86 // Create space for FPRs and stack alignment padding. 87 subq MACRO_LITERAL(8 + 16 * 8), %rsp 88 CFI_ADJUST_CFA_OFFSET(8 + 16 * 8) 89 // R10 := Runtime::Current() 90 LOAD_RUNTIME_INSTANCE r10 91 // Save FPRs. 92 movq %xmm0, 8(%rsp) 93 movq %xmm1, 16(%rsp) 94 movq %xmm2, 24(%rsp) 95 movq %xmm3, 32(%rsp) 96 movq %xmm4, 40(%rsp) 97 movq %xmm5, 48(%rsp) 98 movq %xmm6, 56(%rsp) 99 movq %xmm7, 64(%rsp) 100 movq %xmm8, 72(%rsp) 101 movq %xmm9, 80(%rsp) 102 movq %xmm10, 88(%rsp) 103 movq %xmm11, 96(%rsp) 104 movq %xmm12, 104(%rsp) 105 movq %xmm13, 112(%rsp) 106 movq %xmm14, 120(%rsp) 107 movq %xmm15, 128(%rsp) 108 // Push ArtMethod* for save everything frame method. 109 pushq \runtime_method_offset(%r10) 110 CFI_ADJUST_CFA_OFFSET(8) 111 // Store rsp as the top quick frame. 112 movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET 113 114 // Ugly compile-time check, but we only have the preprocessor. 115 // Last +8: implicit return address pushed on stack when caller made call. 116#if (FRAME_SIZE_SAVE_EVERYTHING != 15 * 8 + 16 * 8 + 16 + 8) 117#error "FRAME_SIZE_SAVE_EVERYTHING(X86_64) size not as expected." 118#endif 119#endif // __APPLE__ 120END_MACRO 121 122 /* 123 * Macro that sets up the callee save frame to conform with 124 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 125 * when R15 is already saved. 126 */ 127MACRO1(SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) 128 PUSH r14 129 SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED \runtime_method_offset 130END_MACRO 131 132 /* 133 * Macro that sets up the callee save frame to conform with 134 * Runtime::CreateCalleeSaveMethod(kSaveEverything) 135 */ 136MACRO1(SETUP_SAVE_EVERYTHING_FRAME, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) 137 PUSH r15 138 SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED \runtime_method_offset 139END_MACRO 140 141MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_FRPS) 142 // Restore FPRs. Method and padding is still on the stack. 143 movq 16(%rsp), %xmm0 144 movq 24(%rsp), %xmm1 145 movq 32(%rsp), %xmm2 146 movq 40(%rsp), %xmm3 147 movq 48(%rsp), %xmm4 148 movq 56(%rsp), %xmm5 149 movq 64(%rsp), %xmm6 150 movq 72(%rsp), %xmm7 151 movq 80(%rsp), %xmm8 152 movq 88(%rsp), %xmm9 153 movq 96(%rsp), %xmm10 154 movq 104(%rsp), %xmm11 155 movq 112(%rsp), %xmm12 156 movq 120(%rsp), %xmm13 157 movq 128(%rsp), %xmm14 158 movq 136(%rsp), %xmm15 159END_MACRO 160 161MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_GPRS_EXCEPT_RAX) 162 // Restore callee and GPR args (except RAX), mixed together to agree with core spills bitmap. 163 POP rcx 164 POP rdx 165 POP rbx 166 POP rbp 167 POP rsi 168 POP rdi 169 POP r8 170 POP r9 171 POP r10 172 POP r11 173 POP r12 174 POP r13 175 POP r14 176 POP r15 177END_MACRO 178 179MACRO0(RESTORE_SAVE_EVERYTHING_FRAME) 180 RESTORE_SAVE_EVERYTHING_FRAME_FRPS 181 182 // Remove save everything callee save method, stack alignment padding and FPRs. 183 addq MACRO_LITERAL(16 + 16 * 8), %rsp 184 CFI_ADJUST_CFA_OFFSET(-(16 + 16 * 8)) 185 186 POP rax 187 RESTORE_SAVE_EVERYTHING_FRAME_GPRS_EXCEPT_RAX 188END_MACRO 189 190MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_KEEP_RAX) 191 RESTORE_SAVE_EVERYTHING_FRAME_FRPS 192 193 // Remove save everything callee save method, stack alignment padding and FPRs, skip RAX. 194 addq MACRO_LITERAL(16 + 16 * 8 + 8), %rsp 195 CFI_ADJUST_CFA_OFFSET(-(16 + 16 * 8 + 8)) 196 197 RESTORE_SAVE_EVERYTHING_FRAME_GPRS_EXCEPT_RAX 198END_MACRO 199 200MACRO2(NO_ARG_RUNTIME_EXCEPTION, c_name, cxx_name) 201 DEFINE_FUNCTION VAR(c_name) 202 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 203 // Outgoing argument set up 204 movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current() 205 call CALLVAR(cxx_name) // cxx_name(Thread*) 206 UNREACHABLE 207 END_FUNCTION VAR(c_name) 208END_MACRO 209 210MACRO2(NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING, c_name, cxx_name) 211 DEFINE_FUNCTION VAR(c_name) 212 SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context 213 // Outgoing argument set up 214 movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current() 215 call CALLVAR(cxx_name) // cxx_name(Thread*) 216 UNREACHABLE 217 END_FUNCTION VAR(c_name) 218END_MACRO 219 220MACRO2(ONE_ARG_RUNTIME_EXCEPTION, c_name, cxx_name) 221 DEFINE_FUNCTION VAR(c_name) 222 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 223 // Outgoing argument set up 224 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 225 call CALLVAR(cxx_name) // cxx_name(arg1, Thread*) 226 UNREACHABLE 227 END_FUNCTION VAR(c_name) 228END_MACRO 229 230MACRO2(TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING, c_name, cxx_name) 231 DEFINE_FUNCTION VAR(c_name) 232 SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context 233 // Outgoing argument set up 234 movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() 235 call CALLVAR(cxx_name) // cxx_name(Thread*) 236 UNREACHABLE 237 END_FUNCTION VAR(c_name) 238END_MACRO 239 240 /* 241 * Called by managed code to create and deliver a NullPointerException. 242 */ 243NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode 244 245 /* 246 * Call installed by a signal handler to create and deliver a NullPointerException. 247 */ 248DEFINE_FUNCTION_CUSTOM_CFA art_quick_throw_null_pointer_exception_from_signal, 2 * __SIZEOF_POINTER__ 249 // Fault address and return address were saved by the fault handler. 250 // Save all registers as basis for long jump context; R15 will replace fault address later. 251 SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED 252 // Retrieve fault address and save R15. 253 movq (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__)(%rsp), %rdi 254 movq %r15, (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__)(%rsp) 255 CFI_REL_OFFSET(%r15, (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__)) 256 // Outgoing argument set up; RDI already contains the fault address. 257 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 258 call SYMBOL(artThrowNullPointerExceptionFromSignal) // (addr, self) 259 UNREACHABLE 260END_FUNCTION art_quick_throw_null_pointer_exception_from_signal 261 262 /* 263 * Called by managed code to create and deliver an ArithmeticException. 264 */ 265NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_div_zero, artThrowDivZeroFromCode 266 267 /* 268 * Called by managed code to create and deliver a StackOverflowError. 269 */ 270NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode 271 272 /* 273 * Called by managed code, saves callee saves and then calls artThrowException 274 * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception. 275 */ 276ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode 277 278 /* 279 * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds 280 * index, arg2 holds limit. 281 */ 282TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_array_bounds, artThrowArrayBoundsFromCode 283 284 /* 285 * Called by managed code to create and deliver a StringIndexOutOfBoundsException 286 * as if thrown from a call to String.charAt(). Arg1 holds index, arg2 holds limit. 287 */ 288TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_string_bounds, artThrowStringBoundsFromCode 289 290 /* 291 * All generated callsites for interface invokes and invocation slow paths will load arguments 292 * as usual - except instead of loading arg0/rdi with the target Method*, arg0/rdi will contain 293 * the method_idx. This wrapper will save arg1-arg3, and call the appropriate C helper. 294 * NOTE: "this" is first visible argument of the target, and so can be found in arg1/rsi. 295 * 296 * The helper will attempt to locate the target and return a 128-bit result in rax/rdx consisting 297 * of the target Method* in rax and method->code_ in rdx. 298 * 299 * If unsuccessful, the helper will return null/????. There will be a pending exception in the 300 * thread and we branch to another stub to deliver it. 301 * 302 * On success this wrapper will restore arguments and *jump* to the target, leaving the return 303 * location on the stack. 304 * 305 * Adapted from x86 code. 306 */ 307MACRO1(INVOKE_TRAMPOLINE_BODY, cxx_name) 308 SETUP_SAVE_REFS_AND_ARGS_FRAME // save callee saves in case allocation triggers GC 309 // Helper signature is always 310 // (method_idx, *this_object, *caller_method, *self, sp) 311 312 movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread 313 movq %rsp, %rcx // pass SP 314 315 call CALLVAR(cxx_name) // cxx_name(arg1, arg2, Thread*, SP) 316 // save the code pointer 317 movq %rax, %rdi 318 movq %rdx, %rax 319 RESTORE_SAVE_REFS_AND_ARGS_FRAME 320 321 testq %rdi, %rdi 322 jz 1f 323 324 // Tail call to intended method. 325 jmp *%rax 3261: 327 DELIVER_PENDING_EXCEPTION 328END_MACRO 329MACRO2(INVOKE_TRAMPOLINE, c_name, cxx_name) 330 DEFINE_FUNCTION VAR(c_name) 331 INVOKE_TRAMPOLINE_BODY RAW_VAR(cxx_name) 332 END_FUNCTION VAR(c_name) 333END_MACRO 334 335INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, artInvokeInterfaceTrampolineWithAccessCheck 336 337INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, artInvokeStaticTrampolineWithAccessCheck 338INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, artInvokeDirectTrampolineWithAccessCheck 339INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, artInvokeSuperTrampolineWithAccessCheck 340INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, artInvokeVirtualTrampolineWithAccessCheck 341 342 343 /* 344 * Helper for quick invocation stub to set up XMM registers. Assumes r10 == shorty, 345 * r11 == arg_array. Clobbers r10, r11 and al. Branches to xmm_setup_finished if it encounters 346 * the end of the shorty. 347 */ 348MACRO2(LOOP_OVER_SHORTY_LOADING_XMMS, xmm_reg, finished) 3491: // LOOP 350 movb (%r10), %al // al := *shorty 351 addq MACRO_LITERAL(1), %r10 // shorty++ 352 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto xmm_setup_finished 353 je VAR(finished) 354 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE 355 je 2f 356 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT 357 je 3f 358 addq MACRO_LITERAL(4), %r11 // arg_array++ 359 // Handle extra space in arg array taken by a long. 360 cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP 361 jne 1b 362 addq MACRO_LITERAL(4), %r11 // arg_array++ 363 jmp 1b // goto LOOP 3642: // FOUND_DOUBLE 365 movsd (%r11), REG_VAR(xmm_reg) 366 addq MACRO_LITERAL(8), %r11 // arg_array+=2 367 jmp 4f 3683: // FOUND_FLOAT 369 movss (%r11), REG_VAR(xmm_reg) 370 addq MACRO_LITERAL(4), %r11 // arg_array++ 3714: 372END_MACRO 373 374 /* 375 * Helper for quick invocation stub to set up GPR registers. Assumes r10 == shorty, 376 * r11 == arg_array. Clobbers r10, r11 and al. Branches to gpr_setup_finished if it encounters 377 * the end of the shorty. 378 */ 379MACRO3(LOOP_OVER_SHORTY_LOADING_GPRS, gpr_reg64, gpr_reg32, finished) 3801: // LOOP 381 movb (%r10), %al // al := *shorty 382 addq MACRO_LITERAL(1), %r10 // shorty++ 383 cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto gpr_setup_finished 384 je VAR(finished) 385 cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG 386 je 2f 387 cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT 388 je 3f 389 cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE 390 je 4f 391 movl (%r11), REG_VAR(gpr_reg32) 392 addq MACRO_LITERAL(4), %r11 // arg_array++ 393 jmp 5f 3942: // FOUND_LONG 395 movq (%r11), REG_VAR(gpr_reg64) 396 addq MACRO_LITERAL(8), %r11 // arg_array+=2 397 jmp 5f 3983: // SKIP_FLOAT 399 addq MACRO_LITERAL(4), %r11 // arg_array++ 400 jmp 1b 4014: // SKIP_DOUBLE 402 addq MACRO_LITERAL(8), %r11 // arg_array+=2 403 jmp 1b 4045: 405END_MACRO 406 407 /* 408 * Quick invocation stub. 409 * On entry: 410 * [sp] = return address 411 * rdi = method pointer 412 * rsi = argument array that must at least contain the this pointer. 413 * rdx = size of argument array in bytes 414 * rcx = (managed) thread pointer 415 * r8 = JValue* result 416 * r9 = char* shorty 417 */ 418DEFINE_FUNCTION art_quick_invoke_stub 419#if defined(__APPLE__) 420 int3 421 int3 422#else 423 // Set up argument XMM registers. 424 leaq 1(%r9), %r10 // R10 := shorty + 1 ; ie skip return arg character. 425 leaq 4(%rsi), %r11 // R11 := arg_array + 4 ; ie skip this pointer. 426 LOOP_OVER_SHORTY_LOADING_XMMS xmm0, .Lxmm_setup_finished 427 LOOP_OVER_SHORTY_LOADING_XMMS xmm1, .Lxmm_setup_finished 428 LOOP_OVER_SHORTY_LOADING_XMMS xmm2, .Lxmm_setup_finished 429 LOOP_OVER_SHORTY_LOADING_XMMS xmm3, .Lxmm_setup_finished 430 LOOP_OVER_SHORTY_LOADING_XMMS xmm4, .Lxmm_setup_finished 431 LOOP_OVER_SHORTY_LOADING_XMMS xmm5, .Lxmm_setup_finished 432 LOOP_OVER_SHORTY_LOADING_XMMS xmm6, .Lxmm_setup_finished 433 LOOP_OVER_SHORTY_LOADING_XMMS xmm7, .Lxmm_setup_finished 434 .balign 16 435.Lxmm_setup_finished: 436 PUSH rbp // Save rbp. 437 PUSH r8 // Save r8/result*. 438 PUSH r9 // Save r9/shorty*. 439 PUSH rbx // Save native callee save rbx 440 PUSH r12 // Save native callee save r12 441 PUSH r13 // Save native callee save r13 442 PUSH r14 // Save native callee save r14 443 PUSH r15 // Save native callee save r15 444 movq %rsp, %rbp // Copy value of stack pointer into base pointer. 445 CFI_DEF_CFA_REGISTER(rbp) 446 447 movl %edx, %r10d 448 addl LITERAL(100), %edx // Reserve space for return addr, StackReference<method>, rbp, 449 // r8, r9, rbx, r12, r13, r14, and r15 in frame. 450 andl LITERAL(0xFFFFFFF0), %edx // Align frame size to 16 bytes. 451 subl LITERAL(72), %edx // Remove space for return address, rbp, r8, r9, rbx, r12, 452 // r13, r14, and r15 453 subq %rdx, %rsp // Reserve stack space for argument array. 454 455#if (STACK_REFERENCE_SIZE != 4) 456#error "STACK_REFERENCE_SIZE(X86_64) size not as expected." 457#endif 458 movq LITERAL(0), (%rsp) // Store null for method* 459 460 movl %r10d, %ecx // Place size of args in rcx. 461 movq %rdi, %rax // rax := method to be called 462 movq %rsi, %r11 // r11 := arg_array 463 leaq 8(%rsp), %rdi // rdi is pointing just above the ArtMethod* in the stack 464 // arguments. 465 // Copy arg array into stack. 466 rep movsb // while (rcx--) { *rdi++ = *rsi++ } 467 leaq 1(%r9), %r10 // r10 := shorty + 1 ; ie skip return arg character 468 movq %rax, %rdi // rdi := method to be called 469 movl (%r11), %esi // rsi := this pointer 470 addq LITERAL(4), %r11 // arg_array++ 471 LOOP_OVER_SHORTY_LOADING_GPRS rdx, edx, .Lgpr_setup_finished 472 LOOP_OVER_SHORTY_LOADING_GPRS rcx, ecx, .Lgpr_setup_finished 473 LOOP_OVER_SHORTY_LOADING_GPRS r8, r8d, .Lgpr_setup_finished 474 LOOP_OVER_SHORTY_LOADING_GPRS r9, r9d, .Lgpr_setup_finished 475.Lgpr_setup_finished: 476 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 477 movq %rbp, %rsp // Restore stack pointer. 478 POP r15 // Pop r15 479 POP r14 // Pop r14 480 POP r13 // Pop r13 481 POP r12 // Pop r12 482 POP rbx // Pop rbx 483 POP r9 // Pop r9 - shorty* 484 POP r8 // Pop r8 - result*. 485 POP rbp // Pop rbp 486 cmpb LITERAL(68), (%r9) // Test if result type char == 'D'. 487 je .Lreturn_double_quick 488 cmpb LITERAL(70), (%r9) // Test if result type char == 'F'. 489 je .Lreturn_float_quick 490 movq %rax, (%r8) // Store the result assuming its a long, int or Object* 491 ret 492.Lreturn_double_quick: 493 movsd %xmm0, (%r8) // Store the double floating point result. 494 ret 495.Lreturn_float_quick: 496 movss %xmm0, (%r8) // Store the floating point result. 497 ret 498#endif // __APPLE__ 499END_FUNCTION art_quick_invoke_stub 500 501 /* 502 * Quick invocation stub. 503 * On entry: 504 * [sp] = return address 505 * rdi = method pointer 506 * rsi = argument array or null if no arguments. 507 * rdx = size of argument array in bytes 508 * rcx = (managed) thread pointer 509 * r8 = JValue* result 510 * r9 = char* shorty 511 */ 512DEFINE_FUNCTION art_quick_invoke_static_stub 513#if defined(__APPLE__) 514 int3 515 int3 516#else 517 // Set up argument XMM registers. 518 leaq 1(%r9), %r10 // R10 := shorty + 1 ; ie skip return arg character 519 movq %rsi, %r11 // R11 := arg_array 520 LOOP_OVER_SHORTY_LOADING_XMMS xmm0, .Lxmm_setup_finished2 521 LOOP_OVER_SHORTY_LOADING_XMMS xmm1, .Lxmm_setup_finished2 522 LOOP_OVER_SHORTY_LOADING_XMMS xmm2, .Lxmm_setup_finished2 523 LOOP_OVER_SHORTY_LOADING_XMMS xmm3, .Lxmm_setup_finished2 524 LOOP_OVER_SHORTY_LOADING_XMMS xmm4, .Lxmm_setup_finished2 525 LOOP_OVER_SHORTY_LOADING_XMMS xmm5, .Lxmm_setup_finished2 526 LOOP_OVER_SHORTY_LOADING_XMMS xmm6, .Lxmm_setup_finished2 527 LOOP_OVER_SHORTY_LOADING_XMMS xmm7, .Lxmm_setup_finished2 528 .balign 16 529.Lxmm_setup_finished2: 530 PUSH rbp // Save rbp. 531 PUSH r8 // Save r8/result*. 532 PUSH r9 // Save r9/shorty*. 533 PUSH rbx // Save rbx 534 PUSH r12 // Save r12 535 PUSH r13 // Save r13 536 PUSH r14 // Save r14 537 PUSH r15 // Save r15 538 movq %rsp, %rbp // Copy value of stack pointer into base pointer. 539 CFI_DEF_CFA_REGISTER(rbp) 540 541 movl %edx, %r10d 542 addl LITERAL(100), %edx // Reserve space for return addr, StackReference<method>, rbp, 543 // r8, r9, r12, r13, r14, and r15 in frame. 544 andl LITERAL(0xFFFFFFF0), %edx // Align frame size to 16 bytes. 545 subl LITERAL(72), %edx // Remove space for return address, rbp, r8, r9, rbx, r12, 546 // r13, r14, and r15. 547 subq %rdx, %rsp // Reserve stack space for argument array. 548 549#if (STACK_REFERENCE_SIZE != 4) 550#error "STACK_REFERENCE_SIZE(X86_64) size not as expected." 551#endif 552 movq LITERAL(0), (%rsp) // Store null for method* 553 554 movl %r10d, %ecx // Place size of args in rcx. 555 movq %rdi, %rax // rax := method to be called 556 movq %rsi, %r11 // r11 := arg_array 557 leaq 8(%rsp), %rdi // rdi is pointing just above the ArtMethod* in the 558 // stack arguments. 559 // Copy arg array into stack. 560 rep movsb // while (rcx--) { *rdi++ = *rsi++ } 561 leaq 1(%r9), %r10 // r10 := shorty + 1 ; ie skip return arg character 562 movq %rax, %rdi // rdi := method to be called 563 LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, .Lgpr_setup_finished2 564 LOOP_OVER_SHORTY_LOADING_GPRS rdx, edx, .Lgpr_setup_finished2 565 LOOP_OVER_SHORTY_LOADING_GPRS rcx, ecx, .Lgpr_setup_finished2 566 LOOP_OVER_SHORTY_LOADING_GPRS r8, r8d, .Lgpr_setup_finished2 567 LOOP_OVER_SHORTY_LOADING_GPRS r9, r9d, .Lgpr_setup_finished2 568.Lgpr_setup_finished2: 569 call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. 570 movq %rbp, %rsp // Restore stack pointer. 571 POP r15 // Pop r15 572 POP r14 // Pop r14 573 POP r13 // Pop r13 574 POP r12 // Pop r12 575 POP rbx // Pop rbx 576 POP r9 // Pop r9 - shorty*. 577 POP r8 // Pop r8 - result*. 578 POP rbp // Pop rbp 579 cmpb LITERAL(68), (%r9) // Test if result type char == 'D'. 580 je .Lreturn_double_quick2 581 cmpb LITERAL(70), (%r9) // Test if result type char == 'F'. 582 je .Lreturn_float_quick2 583 movq %rax, (%r8) // Store the result assuming its a long, int or Object* 584 ret 585.Lreturn_double_quick2: 586 movsd %xmm0, (%r8) // Store the double floating point result. 587 ret 588.Lreturn_float_quick2: 589 movss %xmm0, (%r8) // Store the floating point result. 590 ret 591#endif // __APPLE__ 592END_FUNCTION art_quick_invoke_static_stub 593 594 /* 595 * Long jump stub. 596 * On entry: 597 * rdi = gprs 598 * rsi = fprs 599 */ 600DEFINE_FUNCTION art_quick_do_long_jump 601#if defined(__APPLE__) 602 int3 603 int3 604#else 605 // Restore FPRs. 606 movq 0(%rsi), %xmm0 607 movq 8(%rsi), %xmm1 608 movq 16(%rsi), %xmm2 609 movq 24(%rsi), %xmm3 610 movq 32(%rsi), %xmm4 611 movq 40(%rsi), %xmm5 612 movq 48(%rsi), %xmm6 613 movq 56(%rsi), %xmm7 614 movq 64(%rsi), %xmm8 615 movq 72(%rsi), %xmm9 616 movq 80(%rsi), %xmm10 617 movq 88(%rsi), %xmm11 618 movq 96(%rsi), %xmm12 619 movq 104(%rsi), %xmm13 620 movq 112(%rsi), %xmm14 621 movq 120(%rsi), %xmm15 622 // Restore FPRs. 623 movq %rdi, %rsp // RSP points to gprs. 624 // Load all registers except RSP and RIP with values in gprs. 625 popq %r15 626 popq %r14 627 popq %r13 628 popq %r12 629 popq %r11 630 popq %r10 631 popq %r9 632 popq %r8 633 popq %rdi 634 popq %rsi 635 popq %rbp 636 addq LITERAL(8), %rsp // Skip rsp 637 popq %rbx 638 popq %rdx 639 popq %rcx 640 popq %rax 641 popq %rsp // Load stack pointer. 642 ret // From higher in the stack pop rip. 643#endif // __APPLE__ 644END_FUNCTION art_quick_do_long_jump 645 646MACRO3(ONE_ARG_DOWNCALL, c_name, cxx_name, return_macro) 647 DEFINE_FUNCTION VAR(c_name) 648 SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC 649 // Outgoing argument set up 650 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 651 call CALLVAR(cxx_name) // cxx_name(arg0, Thread*) 652 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 653 CALL_MACRO(return_macro) // return or deliver exception 654 END_FUNCTION VAR(c_name) 655END_MACRO 656 657MACRO3(TWO_ARG_DOWNCALL, c_name, cxx_name, return_macro) 658 DEFINE_FUNCTION VAR(c_name) 659 SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC 660 // Outgoing argument set up 661 movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() 662 call CALLVAR(cxx_name) // cxx_name(arg0, arg1, Thread*) 663 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 664 CALL_MACRO(return_macro) // return or deliver exception 665 END_FUNCTION VAR(c_name) 666END_MACRO 667 668MACRO3(THREE_ARG_DOWNCALL, c_name, cxx_name, return_macro) 669 DEFINE_FUNCTION VAR(c_name) 670 SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC 671 // Outgoing argument set up 672 movq %gs:THREAD_SELF_OFFSET, %rcx // pass Thread::Current() 673 call CALLVAR(cxx_name) // cxx_name(arg0, arg1, arg2, Thread*) 674 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 675 CALL_MACRO(return_macro) // return or deliver exception 676 END_FUNCTION VAR(c_name) 677END_MACRO 678 679MACRO3(FOUR_ARG_DOWNCALL, c_name, cxx_name, return_macro) 680 DEFINE_FUNCTION VAR(c_name) 681 SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC 682 // Outgoing argument set up 683 movq %gs:THREAD_SELF_OFFSET, %r8 // pass Thread::Current() 684 call CALLVAR(cxx_name) // cxx_name(arg1, arg2, arg3, arg4, Thread*) 685 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 686 CALL_MACRO(return_macro) // return or deliver exception 687 END_FUNCTION VAR(c_name) 688END_MACRO 689 690MACRO3(ONE_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro) 691 DEFINE_FUNCTION VAR(c_name) 692 SETUP_SAVE_REFS_ONLY_FRAME 693 // arg0 is in rdi 694 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 695 call CALLVAR(cxx_name) // cxx_name(arg0, Thread*) 696 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 697 CALL_MACRO(return_macro) 698 END_FUNCTION VAR(c_name) 699END_MACRO 700 701MACRO3(TWO_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro) 702 DEFINE_FUNCTION VAR(c_name) 703 SETUP_SAVE_REFS_ONLY_FRAME 704 // arg0 and arg1 are in rdi/rsi 705 movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() 706 call CALLVAR(cxx_name) // (arg0, arg1, Thread*) 707 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 708 CALL_MACRO(return_macro) 709 END_FUNCTION VAR(c_name) 710END_MACRO 711 712MACRO3(THREE_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro) 713 DEFINE_FUNCTION VAR(c_name) 714 SETUP_SAVE_REFS_ONLY_FRAME 715 // arg0, arg1, and arg2 are in rdi/rsi/rdx 716 movq %gs:THREAD_SELF_OFFSET, %rcx // pass Thread::Current() 717 call CALLVAR(cxx_name) // cxx_name(arg0, arg1, arg2, Thread*) 718 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 719 CALL_MACRO(return_macro) // return or deliver exception 720 END_FUNCTION VAR(c_name) 721END_MACRO 722 723 /* 724 * Macro for resolution and initialization of indexed DEX file 725 * constants such as classes and strings. 726 */ 727MACRO3(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET) 728 DEFINE_FUNCTION VAR(c_name) 729 SETUP_SAVE_EVERYTHING_FRAME \runtime_method_offset // save everything for GC 730 // Outgoing argument set up 731 movl %eax, %edi // pass the index of the constant as arg0 732 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 733 call CALLVAR(cxx_name) // cxx_name(arg0, Thread*) 734 testl %eax, %eax // If result is null, deliver pending exception. 735 jz 1f 736 DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_RAX /*is_ref=*/1 7371: 738 DELIVER_PENDING_EXCEPTION_FRAME_READY 739 END_FUNCTION VAR(c_name) 740END_MACRO 741 742MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT, c_name, cxx_name) 743 ONE_ARG_SAVE_EVERYTHING_DOWNCALL \c_name, \cxx_name, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET 744END_MACRO 745 746MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER) 747 testq %rax, %rax // rax == 0 ? 748 jz 1f // if rax == 0 goto 1 749 DEOPT_OR_RETURN /*is_ref=*/1 // Check if deopt is required 7501: // deliver exception on current thread 751 DELIVER_PENDING_EXCEPTION 752END_MACRO 753 754 755MACRO0(RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION) 756 movq %gs:THREAD_EXCEPTION_OFFSET, %rcx // get exception field 757 testq %rcx, %rcx // rcx == 0 ? 758 jnz 1f // if rcx != 0 goto 1 759 DEOPT_OR_RETURN // Check if deopt is required 7601: // deliver exception on current thread 761 DELIVER_PENDING_EXCEPTION 762END_MACRO 763 764MACRO1(DEOPT_OR_RETURN, is_ref = 0) 765 cmpl LITERAL(0), %gs:THREAD_DEOPT_CHECK_REQUIRED_OFFSET 766 jne 2f 767 ret 7682: 769 SETUP_SAVE_EVERYTHING_FRAME 770 movq LITERAL(\is_ref), %rdx // pass if result is a reference 771 movq %rax, %rsi // pass the result 772 movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current 773 call SYMBOL(artDeoptimizeIfNeeded) 774 CFI_REMEMBER_STATE 775 RESTORE_SAVE_EVERYTHING_FRAME 776 ret 777 CFI_RESTORE_STATE_AND_DEF_CFA rsp, FRAME_SIZE_SAVE_EVERYTHING 778END_MACRO 779 780MACRO1(DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_RAX, is_ref = 0) 781 cmpl LITERAL(0), %gs:THREAD_DEOPT_CHECK_REQUIRED_OFFSET 782 jne 2f 783 CFI_REMEMBER_STATE 784 RESTORE_SAVE_EVERYTHING_FRAME_KEEP_RAX 785 ret 786 CFI_RESTORE_STATE_AND_DEF_CFA rsp, FRAME_SIZE_SAVE_EVERYTHING 7872: 788 movq %rax, SAVE_EVERYTHING_FRAME_RAX_OFFSET(%rsp) // update result in the frame 789 movq LITERAL(\is_ref), %rdx // pass if result is a reference 790 movq %rax, %rsi // pass the result 791 movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current 792 call SYMBOL(artDeoptimizeIfNeeded) 793 CFI_REMEMBER_STATE 794 RESTORE_SAVE_EVERYTHING_FRAME 795 ret 796 CFI_RESTORE_STATE_AND_DEF_CFA rsp, FRAME_SIZE_SAVE_EVERYTHING 797END_MACRO 798 799 800 801MACRO0(RETURN_IF_EAX_ZERO) 802 testl %eax, %eax // eax == 0 ? 803 jnz 1f // if eax != 0 goto 1 804 DEOPT_OR_RETURN // Check if we need a deopt 8051: // deliver exception on current thread 806 DELIVER_PENDING_EXCEPTION 807END_MACRO 808 809// Generate the allocation entrypoints for each allocator. 810GENERATE_ALLOC_ENTRYPOINTS_FOR_NON_TLAB_ALLOCATORS 811 812// Comment out allocators that have x86_64 specific asm. 813// Region TLAB: 814// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) 815// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB) 816GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) 817GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_region_tlab, RegionTLAB) 818// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB) 819// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_region_tlab, RegionTLAB) 820// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_region_tlab, RegionTLAB) 821// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_region_tlab, RegionTLAB) 822// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_region_tlab, RegionTLAB) 823GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_region_tlab, RegionTLAB) 824GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_region_tlab, RegionTLAB) 825GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_region_tlab, RegionTLAB) 826// Normal TLAB: 827// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB) 828// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB) 829GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_tlab, TLAB) 830GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_tlab, TLAB) 831// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_tlab, TLAB) 832// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_tlab, TLAB) 833// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_tlab, TLAB) 834// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_tlab, TLAB) 835// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_tlab, TLAB) 836GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_tlab, TLAB) 837GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_tlab, TLAB) 838GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_tlab, TLAB) 839 840 841// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_rosalloc, RosAlloc). 842MACRO2(ART_QUICK_ALLOC_OBJECT_ROSALLOC, c_name, cxx_name) 843 DEFINE_FUNCTION VAR(c_name) 844 // Fast path rosalloc allocation. 845 // RDI: mirror::Class*, RAX: return value 846 // RSI, RDX, RCX, R8, R9: free. 847 // Check if the thread local 848 // allocation stack has room. 849 movq %gs:THREAD_SELF_OFFSET, %r8 // r8 = thread 850 movq THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(%r8), %rcx // rcx = alloc stack top. 851 cmpq THREAD_LOCAL_ALLOC_STACK_END_OFFSET(%r8), %rcx 852 jae .Lslow_path\c_name 853 // Load the object size 854 movl MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET(%rdi), %eax 855 // Check if the size is for a thread 856 // local allocation. Also does the 857 // initialized and finalizable checks. 858 cmpl LITERAL(ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE), %eax 859 ja .Lslow_path\c_name 860 // Compute the rosalloc bracket index 861 // from the size. 862 shrq LITERAL(ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT), %rax 863 // Load the rosalloc run (r9) 864 // Subtract __SIZEOF_POINTER__ to 865 // subtract one from edi as there is no 866 // 0 byte run and the size is already 867 // aligned. 868 movq (THREAD_ROSALLOC_RUNS_OFFSET - __SIZEOF_POINTER__)(%r8, %rax, __SIZEOF_POINTER__), %r9 869 // Load the free list head (rax). This 870 // will be the return val. 871 movq (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)(%r9), %rax 872 testq %rax, %rax 873 jz .Lslow_path\c_name 874 // "Point of no slow path". Won't go to the slow path from here on. OK to clobber rdi and rsi. 875 // Push the new object onto the thread 876 // local allocation stack and 877 // increment the thread local 878 // allocation stack top. 879 movl %eax, (%rcx) 880 addq LITERAL(COMPRESSED_REFERENCE_SIZE), %rcx 881 movq %rcx, THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(%r8) 882 // Load the next pointer of the head 883 // and update the list head with the 884 // next pointer. 885 movq ROSALLOC_SLOT_NEXT_OFFSET(%rax), %rcx 886 movq %rcx, (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)(%r9) 887 // Store the class pointer in the 888 // header. This also overwrites the 889 // next pointer. The offsets are 890 // asserted to match. 891#if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET 892#error "Class pointer needs to overwrite next pointer." 893#endif 894 POISON_HEAP_REF edi 895 movl %edi, MIRROR_OBJECT_CLASS_OFFSET(%rax) 896 // Decrement the size of the free list 897 decl (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)(%r9) 898 // No fence necessary for x86. 899 ret 900.Lslow_path\c_name: 901 SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC 902 // Outgoing argument set up 903 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 904 call CALLVAR(cxx_name) // cxx_name(arg0, Thread*) 905 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 906 RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deliver exception 907 END_FUNCTION VAR(c_name) 908END_MACRO 909 910ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_resolved_rosalloc, artAllocObjectFromCodeResolvedRosAlloc 911ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_initialized_rosalloc, artAllocObjectFromCodeInitializedRosAlloc 912 913// The common fast path code for art_quick_alloc_object_resolved_region_tlab. 914// TODO: delete ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH since it is the same as 915// ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH. 916// 917// RDI: the class, RAX: return value. 918// RCX, RSI, RDX: scratch, r8: Thread::Current(). 919MACRO1(ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH, slowPathLabel) 920 ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH(RAW_VAR(slowPathLabel)) 921END_MACRO 922 923// The fast path code for art_quick_alloc_object_initialized_region_tlab. 924// 925// RDI: the class, RSI: ArtMethod*, RAX: return value. 926// RCX, RSI, RDX: scratch, r8: Thread::Current(). 927MACRO1(ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH, slowPathLabel) 928 movq %gs:THREAD_SELF_OFFSET, %r8 // r8 = thread 929 movl MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET(%rdi), %ecx // Load the object size. 930 movq THREAD_LOCAL_POS_OFFSET(%r8), %rax 931 addq %rax, %rcx // Add size to pos, note that these 932 // are both 32 bit ints, overflow 933 // will cause the add to be past the 934 // end of the thread local region. 935 cmpq THREAD_LOCAL_END_OFFSET(%r8), %rcx // Check if it fits. 936 ja RAW_VAR(slowPathLabel) 937 movq %rcx, THREAD_LOCAL_POS_OFFSET(%r8) // Update thread_local_pos. 938 incq THREAD_LOCAL_OBJECTS_OFFSET(%r8) // Increase thread_local_objects. 939 // Store the class pointer in the 940 // header. 941 // No fence needed for x86. 942 POISON_HEAP_REF edi 943 movl %edi, MIRROR_OBJECT_CLASS_OFFSET(%rax) 944 ret // Fast path succeeded. 945END_MACRO 946 947// The fast path code for art_quick_alloc_array_region_tlab. 948// Inputs: RDI: the class, RSI: int32_t component_count, R9: total_size 949// Free temps: RCX, RDX, R8 950// Output: RAX: return value. 951MACRO1(ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE, slowPathLabel) 952 movq %gs:THREAD_SELF_OFFSET, %rcx // rcx = thread 953 // Mask out the unaligned part to make sure we are 8 byte aligned. 954 andq LITERAL(OBJECT_ALIGNMENT_MASK_TOGGLED64), %r9 955 movq THREAD_LOCAL_POS_OFFSET(%rcx), %rax 956 addq %rax, %r9 957 cmpq THREAD_LOCAL_END_OFFSET(%rcx), %r9 // Check if it fits. 958 ja RAW_VAR(slowPathLabel) 959 movq %r9, THREAD_LOCAL_POS_OFFSET(%rcx) // Update thread_local_pos. 960 addq LITERAL(1), THREAD_LOCAL_OBJECTS_OFFSET(%rcx) // Increase thread_local_objects. 961 // Store the class pointer in the 962 // header. 963 // No fence needed for x86. 964 POISON_HEAP_REF edi 965 movl %edi, MIRROR_OBJECT_CLASS_OFFSET(%rax) 966 movl %esi, MIRROR_ARRAY_LENGTH_OFFSET(%rax) 967 ret // Fast path succeeded. 968END_MACRO 969 970// The common slow path code for art_quick_alloc_object_{resolved, initialized}_tlab 971// and art_quick_alloc_object_{resolved, initialized}_region_tlab. 972MACRO1(ALLOC_OBJECT_TLAB_SLOW_PATH, cxx_name) 973 SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC 974 // Outgoing argument set up 975 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 976 call CALLVAR(cxx_name) // cxx_name(arg0, Thread*) 977 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 978 RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deliver exception 979END_MACRO 980 981// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB). May be 982// called with CC if the GC is not active. 983DEFINE_FUNCTION art_quick_alloc_object_resolved_tlab 984 // RDI: mirror::Class* klass 985 // RDX, RSI, RCX, R8, R9: free. RAX: return val. 986 ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH .Lart_quick_alloc_object_resolved_tlab_slow_path 987.Lart_quick_alloc_object_resolved_tlab_slow_path: 988 ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeResolvedTLAB 989END_FUNCTION art_quick_alloc_object_resolved_tlab 990 991// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB). 992// May be called with CC if the GC is not active. 993DEFINE_FUNCTION art_quick_alloc_object_initialized_tlab 994 // RDI: mirror::Class* klass 995 // RDX, RSI, RCX, R8, R9: free. RAX: return val. 996 ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH .Lart_quick_alloc_object_initialized_tlab_slow_path 997.Lart_quick_alloc_object_initialized_tlab_slow_path: 998 ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeInitializedTLAB 999END_FUNCTION art_quick_alloc_object_initialized_tlab 1000 1001MACRO0(COMPUTE_ARRAY_SIZE_UNKNOWN) 1002 movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rdi), %ecx // Load component type. 1003 UNPOISON_HEAP_REF ecx 1004 movl MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET(%rcx), %ecx // Load primitive type. 1005 shrq MACRO_LITERAL(PRIMITIVE_TYPE_SIZE_SHIFT_SHIFT), %rcx // Get component size shift. 1006 movq %rsi, %r9 1007 salq %cl, %r9 // Calculate array count shifted. 1008 // Add array header + alignment rounding. 1009 addq MACRO_LITERAL(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9 1010 // Add 4 extra bytes if we are doing a long array. 1011 addq MACRO_LITERAL(1), %rcx 1012 andq MACRO_LITERAL(4), %rcx 1013#if MIRROR_LONG_ARRAY_DATA_OFFSET != MIRROR_INT_ARRAY_DATA_OFFSET + 4 1014#error Long array data offset must be 4 greater than int array data offset. 1015#endif 1016 addq %rcx, %r9 1017END_MACRO 1018 1019MACRO0(COMPUTE_ARRAY_SIZE_8) 1020 // RDI: mirror::Class* klass, RSI: int32_t component_count 1021 // RDX, RCX, R8, R9: free. RAX: return val. 1022 movq %rsi, %r9 1023 // Add array header + alignment rounding. 1024 addq MACRO_LITERAL(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9 1025END_MACRO 1026 1027MACRO0(COMPUTE_ARRAY_SIZE_16) 1028 // RDI: mirror::Class* klass, RSI: int32_t component_count 1029 // RDX, RCX, R8, R9: free. RAX: return val. 1030 movq %rsi, %r9 1031 salq MACRO_LITERAL(1), %r9 1032 // Add array header + alignment rounding. 1033 addq MACRO_LITERAL(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9 1034END_MACRO 1035 1036MACRO0(COMPUTE_ARRAY_SIZE_32) 1037 // RDI: mirror::Class* klass, RSI: int32_t component_count 1038 // RDX, RCX, R8, R9: free. RAX: return val. 1039 movq %rsi, %r9 1040 salq MACRO_LITERAL(2), %r9 1041 // Add array header + alignment rounding. 1042 addq MACRO_LITERAL(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9 1043END_MACRO 1044 1045MACRO0(COMPUTE_ARRAY_SIZE_64) 1046 // RDI: mirror::Class* klass, RSI: int32_t component_count 1047 // RDX, RCX, R8, R9: free. RAX: return val. 1048 movq %rsi, %r9 1049 salq MACRO_LITERAL(3), %r9 1050 // Add array header + alignment rounding. 1051 addq MACRO_LITERAL(MIRROR_WIDE_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9 1052END_MACRO 1053 1054MACRO3(GENERATE_ALLOC_ARRAY_TLAB, c_entrypoint, cxx_name, size_setup) 1055 DEFINE_FUNCTION VAR(c_entrypoint) 1056 // RDI: mirror::Class* klass, RSI: int32_t component_count 1057 // RDX, RCX, R8, R9: free. RAX: return val. 1058 CALL_MACRO(size_setup) 1059 ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE .Lslow_path\c_entrypoint 1060.Lslow_path\c_entrypoint: 1061 SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC 1062 // Outgoing argument set up 1063 movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() 1064 call CALLVAR(cxx_name) // cxx_name(arg0, arg1, Thread*) 1065 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 1066 RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deliver exception 1067 END_FUNCTION VAR(c_entrypoint) 1068END_MACRO 1069 1070 1071GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN 1072GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_8 1073GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_16 1074GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_32 1075GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_64 1076 1077GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN 1078GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_8 1079GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_16 1080GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_32 1081GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_64 1082 1083// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB). 1084DEFINE_FUNCTION art_quick_alloc_object_resolved_region_tlab 1085 // Fast path region tlab allocation. 1086 // RDI: mirror::Class* klass 1087 // RDX, RSI, RCX, R8, R9: free. RAX: return val. 1088 ASSERT_USE_READ_BARRIER 1089 ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH .Lart_quick_alloc_object_resolved_region_tlab_slow_path 1090.Lart_quick_alloc_object_resolved_region_tlab_slow_path: 1091 ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeResolvedRegionTLAB 1092END_FUNCTION art_quick_alloc_object_resolved_region_tlab 1093 1094// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB). 1095DEFINE_FUNCTION art_quick_alloc_object_initialized_region_tlab 1096 // Fast path region tlab allocation. 1097 // RDI: mirror::Class* klass 1098 // RDX, RSI, RCX, R8, R9: free. RAX: return val. 1099 ASSERT_USE_READ_BARRIER 1100 // No read barrier since the caller is responsible for that. 1101 ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH .Lart_quick_alloc_object_initialized_region_tlab_slow_path 1102.Lart_quick_alloc_object_initialized_region_tlab_slow_path: 1103 ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeInitializedRegionTLAB 1104END_FUNCTION art_quick_alloc_object_initialized_region_tlab 1105 1106ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode 1107ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_resolve_type, artResolveTypeFromCode 1108ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_type_and_verify_access, artResolveTypeAndVerifyAccessFromCode 1109ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_handle, artResolveMethodHandleFromCode 1110ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_type, artResolveMethodTypeFromCode 1111ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode 1112 1113TWO_ARG_REF_DOWNCALL art_quick_handle_fill_data, artHandleFillArrayDataFromCode, RETURN_IF_EAX_ZERO 1114 1115 /* 1116 * Entry from managed code that tries to lock the object in a fast path and 1117 * calls `artLockObjectFromCode()` for the difficult cases, may block for GC. 1118 * RDI holds the possibly null object to lock. 1119 */ 1120DEFINE_FUNCTION art_quick_lock_object 1121 testq %rdi, %rdi // Null check object. 1122 jz art_quick_lock_object_no_inline 1123 LOCK_OBJECT_FAST_PATH rdi, ecx, art_quick_lock_object_no_inline 1124END_FUNCTION art_quick_lock_object 1125 1126 /* 1127 * Entry from managed code that calls `artLockObjectFromCode()`, may block for GC. 1128 * RDI holds the possibly null object to lock. 1129 */ 1130DEFINE_FUNCTION art_quick_lock_object_no_inline 1131 // This is also the slow path for art_quick_lock_object. 1132 SETUP_SAVE_REFS_ONLY_FRAME 1133 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 1134 call SYMBOL(artLockObjectFromCode) // artLockObjectFromCode(object, Thread*) 1135 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 1136 RETURN_IF_EAX_ZERO 1137END_FUNCTION art_quick_lock_object_no_inline 1138 1139 /* 1140 * Entry from managed code that tries to unlock the object in a fast path and calls 1141 * `artUnlockObjectFromCode()` for the difficult cases and delivers exception on failure. 1142 * RDI holds the possibly null object to unlock. 1143 */ 1144DEFINE_FUNCTION art_quick_unlock_object 1145 testq %rdi, %rdi // Null check object. 1146 jz art_quick_lock_object_no_inline 1147 UNLOCK_OBJECT_FAST_PATH rdi, ecx, /*saved_rax*/ none, art_quick_unlock_object_no_inline 1148END_FUNCTION art_quick_unlock_object 1149 1150 /* 1151 * Entry from managed code that calls `artUnlockObjectFromCode()` 1152 * and delivers exception on failure. 1153 * RDI holds the possibly null object to unlock. 1154 */ 1155DEFINE_FUNCTION art_quick_unlock_object_no_inline 1156 // This is also the slow path for art_quick_unlock_object. 1157 SETUP_SAVE_REFS_ONLY_FRAME 1158 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 1159 call SYMBOL(artUnlockObjectFromCode) // artUnlockObjectFromCode(object, Thread*) 1160 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 1161 RETURN_IF_EAX_ZERO 1162END_FUNCTION art_quick_unlock_object_no_inline 1163 1164DEFINE_FUNCTION art_quick_check_instance_of 1165 // Type check using the bit string passes null as the target class. In that case just throw. 1166 testl %esi, %esi 1167 jz .Lthrow_class_cast_exception_for_bitstring_check 1168 1169 // We could check the super classes here but that is usually already checked in the caller. 1170 PUSH rdi // Save args for exc 1171 PUSH rsi 1172 subq LITERAL(8), %rsp // Alignment padding. 1173 CFI_ADJUST_CFA_OFFSET(8) 1174 SETUP_FP_CALLEE_SAVE_FRAME 1175 call SYMBOL(artInstanceOfFromCode) // (Object* obj, Class* ref_klass) 1176 testq %rax, %rax 1177 jz .Lthrow_class_cast_exception // jump forward if not assignable 1178 CFI_REMEMBER_STATE 1179 RESTORE_FP_CALLEE_SAVE_FRAME 1180 addq LITERAL(24), %rsp // pop arguments 1181 CFI_ADJUST_CFA_OFFSET(-24) 1182 ret 1183 CFI_RESTORE_STATE_AND_DEF_CFA rsp, 64 // Reset unwind info so following code unwinds. 1184 1185.Lthrow_class_cast_exception: 1186 RESTORE_FP_CALLEE_SAVE_FRAME 1187 addq LITERAL(8), %rsp // pop padding 1188 CFI_ADJUST_CFA_OFFSET(-8) 1189 POP rsi // Pop arguments 1190 POP rdi 1191 1192.Lthrow_class_cast_exception_for_bitstring_check: 1193 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context 1194 mov %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() 1195 call SYMBOL(artThrowClassCastExceptionForObject) // (Object* src, Class* dest, Thread*) 1196 UNREACHABLE 1197END_FUNCTION art_quick_check_instance_of 1198 1199 1200// Restore reg's value if reg is not the same as exclude_reg, otherwise just adjust stack. 1201MACRO2(POP_REG_NE, reg, exclude_reg) 1202 .ifc RAW_VAR(reg), RAW_VAR(exclude_reg) 1203 addq MACRO_LITERAL(8), %rsp 1204 CFI_ADJUST_CFA_OFFSET(-8) 1205 .else 1206 POP RAW_VAR(reg) 1207 .endif 1208END_MACRO 1209 1210DEFINE_FUNCTION art_quick_aput_obj 1211 test %edx, %edx // store of null 1212 jz .Laput_obj_null 1213 movl MIRROR_OBJECT_CLASS_OFFSET(%rdi), %ecx 1214 UNPOISON_HEAP_REF ecx 1215#ifdef USE_READ_BARRIER 1216 cmpl LITERAL(0), %gs:THREAD_IS_GC_MARKING_OFFSET 1217 jnz .Laput_obj_gc_marking 1218#endif // USE_READ_BARRIER 1219 movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rcx), %ecx 1220 cmpl MIRROR_OBJECT_CLASS_OFFSET(%rdx), %ecx // Both poisoned if heap poisoning is enabled. 1221 jne .Laput_obj_check_assignability 1222.Laput_obj_store: 1223 POISON_HEAP_REF edx 1224 movl %edx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rdi, %rsi, 4) 1225 movq %gs:THREAD_CARD_TABLE_OFFSET, %rdx 1226 shrl LITERAL(CARD_TABLE_CARD_SHIFT), %edi 1227 movb %dl, (%rdx, %rdi) 1228 ret 1229 1230.Laput_obj_null: 1231 movl %edx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rdi, %rsi, 4) 1232 ret 1233 1234.Laput_obj_check_assignability: 1235 UNPOISON_HEAP_REF ecx // Unpoison array component type if poisoning is enabled. 1236 PUSH_ARG rdi // Save arguments. 1237 PUSH_ARG rsi 1238 PUSH_ARG rdx 1239 movl MIRROR_OBJECT_CLASS_OFFSET(%rdx), %esi // Pass arg2 = value's class. 1240 UNPOISON_HEAP_REF esi 1241.Laput_obj_check_assignability_call: 1242 movl %ecx, %edi // Pass arg1 = array's component type. 1243 SETUP_FP_CALLEE_SAVE_FRAME 1244 call SYMBOL(artIsAssignableFromCode) // (Class* a, Class* b) 1245 RESTORE_FP_CALLEE_SAVE_FRAME // Resore FP registers. 1246 POP_ARG rdx // Restore arguments. 1247 POP_ARG rsi 1248 POP_ARG rdi 1249 testq %rax, %rax // Check for exception. 1250 jz .Laput_obj_throw_array_store_exception 1251 POISON_HEAP_REF edx 1252 movl %edx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rdi, %rsi, 4) 1253 movq %gs:THREAD_CARD_TABLE_OFFSET, %rdx 1254 shrl LITERAL(CARD_TABLE_CARD_SHIFT), %edi 1255 movb %dl, (%rdx, %rdi) 1256 ret 1257 1258.Laput_obj_throw_array_store_exception: 1259#ifdef USE_READ_BARRIER 1260 CFI_REMEMBER_STATE 1261#endif // USE_READ_BARRIER 1262 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // Save all registers as basis for long jump context. 1263 // Outgoing argument set up. 1264 movq %rdx, %rsi // Pass arg 2 = value. 1265 movq %gs:THREAD_SELF_OFFSET, %rdx // Pass arg 3 = Thread::Current(). 1266 // Pass arg 1 = array. 1267 call SYMBOL(artThrowArrayStoreException) // (array, value, Thread*) 1268 UNREACHABLE 1269 1270#ifdef USE_READ_BARRIER 1271 CFI_RESTORE_STATE_AND_DEF_CFA rsp, 4 1272.Laput_obj_gc_marking: 1273 // We need to align stack for `art_quick_read_barrier_mark_regNN`. 1274 INCREASE_FRAME 8 // Stack alignment. 1275 call SYMBOL(art_quick_read_barrier_mark_reg01) // Mark ECX 1276 movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rcx), %ecx 1277 UNPOISON_HEAP_REF ecx 1278 call SYMBOL(art_quick_read_barrier_mark_reg01) // Mark ECX 1279 movl MIRROR_OBJECT_CLASS_OFFSET(%rdx), %eax 1280 UNPOISON_HEAP_REF eax 1281 call SYMBOL(art_quick_read_barrier_mark_reg00) // Mark EAX 1282 DECREASE_FRAME 8 // Remove stack alignment. 1283 cmpl %eax, %ecx 1284 je .Laput_obj_store 1285 // Prepare arguments in line with `.Laput_obj_check_assignability_call` and jump there. 1286 PUSH_ARG rdi // Save arguments. 1287 PUSH_ARG rsi 1288 PUSH_ARG rdx 1289 movl %eax, %esi // Pass arg2 - type of the value to be stored. 1290 // The arg1 shall be moved at `.Ldo_assignability_check_call`. 1291 jmp .Laput_obj_check_assignability_call 1292#endif // USE_READ_BARRIER 1293END_FUNCTION art_quick_aput_obj 1294 1295// TODO: This is quite silly on X86_64 now. 1296DEFINE_FUNCTION art_quick_memcpy 1297 call PLT_SYMBOL(memcpy) // (void*, const void*, size_t) 1298 ret 1299END_FUNCTION art_quick_memcpy 1300 1301DEFINE_FUNCTION art_quick_test_suspend 1302 SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET // save everything for GC 1303 // Outgoing argument set up 1304 movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current() 1305 call SYMBOL(artTestSuspendFromCode) // (Thread*) 1306 RESTORE_SAVE_EVERYTHING_FRAME // restore frame up to return address 1307 ret 1308END_FUNCTION art_quick_test_suspend 1309 1310UNIMPLEMENTED art_quick_ldiv 1311UNIMPLEMENTED art_quick_lmod 1312UNIMPLEMENTED art_quick_lmul 1313UNIMPLEMENTED art_quick_lshl 1314UNIMPLEMENTED art_quick_lshr 1315UNIMPLEMENTED art_quick_lushr 1316 1317// Note: Functions `art{Get,Set}<Kind>{Static,Instance}FromCompiledCode` are 1318// defined with a macro in runtime/entrypoints/quick/quick_field_entrypoints.cc. 1319 1320THREE_ARG_REF_DOWNCALL art_quick_set8_instance, artSet8InstanceFromCompiledCode, RETURN_IF_EAX_ZERO 1321THREE_ARG_REF_DOWNCALL art_quick_set16_instance, artSet16InstanceFromCompiledCode, RETURN_IF_EAX_ZERO 1322THREE_ARG_REF_DOWNCALL art_quick_set32_instance, artSet32InstanceFromCompiledCode, RETURN_IF_EAX_ZERO 1323THREE_ARG_REF_DOWNCALL art_quick_set64_instance, artSet64InstanceFromCompiledCode, RETURN_IF_EAX_ZERO 1324THREE_ARG_REF_DOWNCALL art_quick_set_obj_instance, artSetObjInstanceFromCompiledCode, RETURN_IF_EAX_ZERO 1325 1326TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1327TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1328TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1329TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1330TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1331TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1332TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1333 1334TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCompiledCode, RETURN_IF_EAX_ZERO 1335TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCompiledCode, RETURN_IF_EAX_ZERO 1336TWO_ARG_REF_DOWNCALL art_quick_set32_static, artSet32StaticFromCompiledCode, RETURN_IF_EAX_ZERO 1337TWO_ARG_REF_DOWNCALL art_quick_set64_static, artSet64StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1338TWO_ARG_REF_DOWNCALL art_quick_set_obj_static, artSetObjStaticFromCompiledCode, RETURN_IF_EAX_ZERO 1339 1340ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1341ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1342ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1343ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1344ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1345ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1346ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION 1347 1348DEFINE_FUNCTION art_quick_proxy_invoke_handler 1349 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_RDI 1350 1351 movq %gs:THREAD_SELF_OFFSET, %rdx // Pass Thread::Current(). 1352 movq %rsp, %rcx // Pass SP. 1353 call SYMBOL(artQuickProxyInvokeHandler) // (proxy method, receiver, Thread*, SP) 1354 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1355 movq %rax, %xmm0 // Copy return value in case of float returns. 1356 RETURN_OR_DELIVER_PENDING_EXCEPTION 1357END_FUNCTION art_quick_proxy_invoke_handler 1358 1359 /* 1360 * Called to resolve an imt conflict. 1361 * rdi is the conflict ArtMethod. 1362 * rax is a hidden argument that holds the target interface method. 1363 * 1364 * Note that this stub writes to rdi. 1365 */ 1366DEFINE_FUNCTION art_quick_imt_conflict_trampoline 1367#if defined(__APPLE__) 1368 int3 1369 int3 1370#else 1371 movq ART_METHOD_JNI_OFFSET_64(%rdi), %rdi // Load ImtConflictTable 1372.Limt_table_iterate: 1373 cmpq %rax, 0(%rdi) 1374 jne .Limt_table_next_entry 1375 // We successfully hit an entry in the table. Load the target method 1376 // and jump to it. 1377 movq __SIZEOF_POINTER__(%rdi), %rdi 1378 jmp *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) 1379.Limt_table_next_entry: 1380 // If the entry is null, the interface method is not in the ImtConflictTable. 1381 cmpq LITERAL(0), 0(%rdi) 1382 jz .Lconflict_trampoline 1383 // Iterate over the entries of the ImtConflictTable. 1384 addq LITERAL(2 * __SIZEOF_POINTER__), %rdi 1385 jmp .Limt_table_iterate 1386.Lconflict_trampoline: 1387 // Call the runtime stub to populate the ImtConflictTable and jump to the 1388 // resolved method. 1389 movq %rax, %rdi // Load interface method 1390 INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline 1391#endif // __APPLE__ 1392END_FUNCTION art_quick_imt_conflict_trampoline 1393 1394DEFINE_FUNCTION art_quick_resolution_trampoline 1395 SETUP_SAVE_REFS_AND_ARGS_FRAME 1396 movq %gs:THREAD_SELF_OFFSET, %rdx 1397 movq %rsp, %rcx 1398 call SYMBOL(artQuickResolutionTrampoline) // (called, receiver, Thread*, SP) 1399 movq %rax, %r10 // Remember returned code pointer in R10. 1400 movq (%rsp), %rdi // Load called method into RDI. 1401 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1402 testq %r10, %r10 // If code pointer is null goto deliver pending exception. 1403 jz 1f 1404 jmp *%r10 // Tail call into method. 14051: 1406 DELIVER_PENDING_EXCEPTION 1407END_FUNCTION art_quick_resolution_trampoline 1408 1409/* Generic JNI frame layout: 1410 * 1411 * #-------------------# 1412 * | | 1413 * | caller method... | 1414 * #-------------------# <--- SP on entry 1415 * 1416 * | 1417 * V 1418 * 1419 * #-------------------# 1420 * | caller method... | 1421 * #-------------------# 1422 * | Return | 1423 * | R15 | callee save 1424 * | R14 | callee save 1425 * | R13 | callee save 1426 * | R12 | callee save 1427 * | R9 | arg5 1428 * | R8 | arg4 1429 * | RSI/R6 | arg1 1430 * | RBP/R5 | callee save 1431 * | RBX/R3 | callee save 1432 * | RDX/R2 | arg2 1433 * | RCX/R1 | arg3 1434 * | XMM7 | float arg 8 1435 * | XMM6 | float arg 7 1436 * | XMM5 | float arg 6 1437 * | XMM4 | float arg 5 1438 * | XMM3 | float arg 4 1439 * | XMM2 | float arg 3 1440 * | XMM1 | float arg 2 1441 * | XMM0 | float arg 1 1442 * | RDI/Method* | <- sp 1443 * #-------------------# 1444 * | Scratch Alloca | 5K scratch space 1445 * #---------#---------# 1446 * | | sp* | 1447 * | Tramp. #---------# 1448 * | args | thread | 1449 * | Tramp. #---------# 1450 * | | method | 1451 * #-------------------# <--- SP on artQuickGenericJniTrampoline 1452 * 1453 * | 1454 * v artQuickGenericJniTrampoline 1455 * 1456 * #-------------------# 1457 * | caller method... | 1458 * #-------------------# 1459 * | Return PC | 1460 * | Callee-Saves | 1461 * | padding | // 8B 1462 * | Method* | <--- (1) 1463 * #-------------------# 1464 * | local ref cookie | // 4B 1465 * | padding | // 0B or 4B to align handle scope on 8B address 1466 * | handle scope | // Size depends on number of references; multiple of 4B. 1467 * #-------------------# 1468 * | JNI Stack Args | // Empty if all args fit into registers. 1469 * #-------------------# <--- SP on native call (1) 1470 * | Free scratch | 1471 * #-------------------# 1472 * | SP for JNI call | // Pointer to (1). 1473 * #-------------------# 1474 * | Hidden arg | // For @CriticalNative 1475 * #-------------------# 1476 * | | 1477 * | Stack for Regs | The trampoline assembly will pop these values 1478 * | | into registers for native call 1479 * #-------------------# 1480 */ 1481 /* 1482 * Called to do a generic JNI down-call 1483 */ 1484DEFINE_FUNCTION art_quick_generic_jni_trampoline 1485 SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_RDI 1486 1487 movq %rsp, %rbp // save SP at (old) callee-save frame 1488 CFI_DEF_CFA_REGISTER(rbp) 1489 1490 subq LITERAL(GENERIC_JNI_TRAMPOLINE_RESERVED_AREA), %rsp 1491 // prepare for artQuickGenericJniTrampoline call 1492 // (Thread*, managed_sp, reserved_area) 1493 // rdi rsi rdx <= C calling convention 1494 // gs:... rbp rsp <= where they are 1495 movq %gs:THREAD_SELF_OFFSET, %rdi // Pass Thread::Current(). 1496 movq %rbp, %rsi // Pass managed frame SP. 1497 movq %rsp, %rdx // Pass reserved area. 1498 call SYMBOL(artQuickGenericJniTrampoline) // (Thread*, sp) 1499 1500 // The C call will have registered the complete save-frame on success. 1501 // The result of the call is: 1502 // %rax: pointer to native code, 0 on error. 1503 // The bottom of the reserved area contains values for arg registers, 1504 // hidden arg register and SP for out args for the call. 1505 1506 // Check for error (class init check or locking for synchronized native method can throw). 1507 test %rax, %rax 1508 jz .Lexception_in_native 1509 1510 // pop from the register-passing alloca region 1511 // what's the right layout? 1512 popq %rdi 1513 popq %rsi 1514 popq %rdx 1515 popq %rcx 1516 popq %r8 1517 popq %r9 1518 // TODO: skip floating point if unused, some flag. 1519 movq 0(%rsp), %xmm0 1520 movq 8(%rsp), %xmm1 1521 movq 16(%rsp), %xmm2 1522 movq 24(%rsp), %xmm3 1523 movq 32(%rsp), %xmm4 1524 movq 40(%rsp), %xmm5 1525 movq 48(%rsp), %xmm6 1526 movq 56(%rsp), %xmm7 1527 1528 // Save call target in scratch register. 1529 movq %rax, %r11 1530 1531 // Load hidden arg (rax) for @CriticalNative. 1532 movq 64(%rsp), %rax 1533 // Load SP for out args, releasing unneeded reserved area. 1534 movq 72(%rsp), %rsp 1535 1536 // native call 1537 call *%r11 1538 1539 // result sign extension is handled in C code 1540 // prepare for artQuickGenericJniEndTrampoline call 1541 // (Thread*, result, result_f) 1542 // rdi rsi rdx <= C calling convention 1543 // gs:... rax xmm0 <= where they are 1544 movq %gs:THREAD_SELF_OFFSET, %rdi 1545 movq %rax, %rsi 1546 movq %xmm0, %rdx 1547 call SYMBOL(artQuickGenericJniEndTrampoline) 1548 1549 // Pending exceptions possible. 1550 // TODO: use cmpq, needs direct encoding because of gas bug 1551 movq %gs:THREAD_EXCEPTION_OFFSET, %rcx 1552 test %rcx, %rcx 1553 jnz .Lexception_in_native 1554 1555 // Tear down the alloca. 1556 movq %rbp, %rsp 1557 1558 // store into fpr, for when it's a fpr return... 1559 movq %rax, %xmm0 1560 1561 LOAD_RUNTIME_INSTANCE rcx 1562 cmpb MACRO_LITERAL(0), RUN_EXIT_HOOKS_OFFSET_FROM_RUNTIME_INSTANCE(%rcx) 1563 jne .Lcall_method_exit_hook 1564.Lcall_method_exit_hook_done: 1565 1566 // Tear down the callee-save frame. 1567 CFI_REMEMBER_STATE 1568 CFI_DEF_CFA_REGISTER(rsp) 1569 // Load callee-save FPRs. Skip FP args. 1570 movq 80(%rsp), %xmm12 1571 movq 88(%rsp), %xmm13 1572 movq 96(%rsp), %xmm14 1573 movq 104(%rsp), %xmm15 1574 // Pop method, padding, FP args and two GRP args (rcx, rdx). 1575 DECREASE_FRAME 16 + 12*8 + 2*8 1576 // Load callee-save GPRs and skip args, mixed together to agree with core spills bitmap. 1577 POP rbx // Callee save. 1578 POP rbp // Callee save. 1579 DECREASE_FRAME 3*8 // Skip three args (RSI, R8, R9). 1580 POP r12 // Callee save. 1581 POP r13 // Callee save. 1582 POP r14 // Callee save. 1583 POP r15 // Callee save. 1584 ret 1585 1586 // Undo the unwinding information from above since it doesn't apply below. 1587 CFI_RESTORE_STATE_AND_DEF_CFA rbp, 208 1588 1589.Lcall_method_exit_hook: 1590 movq LITERAL(FRAME_SIZE_SAVE_REFS_AND_ARGS), %r8 1591 call art_quick_method_exit_hook 1592 jmp .Lcall_method_exit_hook_done 1593 1594.Lexception_in_native: 1595 pushq %gs:THREAD_TOP_QUICK_FRAME_OFFSET 1596 addq LITERAL(-1), (%rsp) // Remove the GenericJNI tag. 1597 movq (%rsp), %rsp 1598 call art_deliver_pending_exception 1599END_FUNCTION art_quick_generic_jni_trampoline 1600 1601DEFINE_FUNCTION art_deliver_pending_exception 1602 // This will create a new save-all frame, required by the runtime. 1603 DELIVER_PENDING_EXCEPTION 1604END_FUNCTION art_deliver_pending_exception 1605 1606 /* 1607 * Called to bridge from the quick to interpreter ABI. On entry the arguments match those 1608 * of a quick call: 1609 * RDI = method being called / to bridge to. 1610 * RSI, RDX, RCX, R8, R9 are arguments to that method. 1611 */ 1612DEFINE_FUNCTION art_quick_to_interpreter_bridge 1613 SETUP_SAVE_REFS_AND_ARGS_FRAME // Set up frame and save arguments. 1614 movq %gs:THREAD_SELF_OFFSET, %rsi // RSI := Thread::Current() 1615 movq %rsp, %rdx // RDX := sp 1616 call SYMBOL(artQuickToInterpreterBridge) // (method, Thread*, SP) 1617 RESTORE_SAVE_REFS_AND_ARGS_FRAME // TODO: no need to restore arguments in this case. 1618 movq %rax, %xmm0 // Place return value also into floating point return value. 1619 RETURN_OR_DELIVER_PENDING_EXCEPTION // return or deliver exception 1620END_FUNCTION art_quick_to_interpreter_bridge 1621 1622 /* 1623 * Called to catch an attempt to invoke an obsolete method. 1624 * RDI = method being called. 1625 */ 1626ONE_ARG_RUNTIME_EXCEPTION art_invoke_obsolete_method_stub, artInvokeObsoleteMethod 1627 1628 /* 1629 * Compiled code has requested that we deoptimize into the interpreter. The deoptimization 1630 * will long jump to the interpreter bridge. 1631 */ 1632DEFINE_FUNCTION art_quick_deoptimize_from_compiled_code 1633 SETUP_SAVE_EVERYTHING_FRAME 1634 // Stack should be aligned now. 1635 movq %gs:THREAD_SELF_OFFSET, %rsi // Pass Thread. 1636 call SYMBOL(artDeoptimizeFromCompiledCode) // (DeoptimizationKind, Thread*) 1637 UNREACHABLE 1638END_FUNCTION art_quick_deoptimize_from_compiled_code 1639 1640 /* 1641 * String's compareTo. 1642 * 1643 * On entry: 1644 * rdi: this string object (known non-null) 1645 * rsi: comp string object (known non-null) 1646 */ 1647DEFINE_FUNCTION art_quick_string_compareto 1648 movl MIRROR_STRING_COUNT_OFFSET(%edi), %r8d 1649 movl MIRROR_STRING_COUNT_OFFSET(%esi), %r9d 1650 /* Build pointers to the start of string data */ 1651 leal MIRROR_STRING_VALUE_OFFSET(%edi), %edi 1652 leal MIRROR_STRING_VALUE_OFFSET(%esi), %esi 1653#if (STRING_COMPRESSION_FEATURE) 1654 /* Differ cases */ 1655 shrl LITERAL(1), %r8d 1656 jnc .Lstring_compareto_this_is_compressed 1657 shrl LITERAL(1), %r9d 1658 jnc .Lstring_compareto_that_is_compressed 1659 jmp .Lstring_compareto_both_not_compressed 1660.Lstring_compareto_this_is_compressed: 1661 shrl LITERAL(1), %r9d 1662 jnc .Lstring_compareto_both_compressed 1663 /* Comparison this (8-bit) and that (16-bit) */ 1664 mov %r8d, %eax 1665 subl %r9d, %eax 1666 mov %r8d, %ecx 1667 cmovg %r9d, %ecx 1668 /* Going into loop to compare each character */ 1669 jecxz .Lstring_compareto_keep_length1 // check loop counter (if 0 then stop) 1670.Lstring_compareto_loop_comparison_this_compressed: 1671 movzbl (%edi), %r8d // move *(this_cur_char) byte to long 1672 movzwl (%esi), %r9d // move *(that_cur_char) word to long 1673 addl LITERAL(1), %edi // ++this_cur_char (8-bit) 1674 addl LITERAL(2), %esi // ++that_cur_char (16-bit) 1675 subl %r9d, %r8d 1676 loope .Lstring_compareto_loop_comparison_this_compressed 1677 cmovne %r8d, %eax // return eax = *(this_cur_char) - *(that_cur_char) 1678.Lstring_compareto_keep_length1: 1679 ret 1680.Lstring_compareto_that_is_compressed: 1681 movl %r8d, %eax 1682 subl %r9d, %eax 1683 mov %r8d, %ecx 1684 cmovg %r9d, %ecx 1685 /* Comparison this (8-bit) and that (16-bit) */ 1686 jecxz .Lstring_compareto_keep_length2 // check loop counter (if 0, don't compare) 1687.Lstring_compareto_loop_comparison_that_compressed: 1688 movzwl (%edi), %r8d // move *(this_cur_char) word to long 1689 movzbl (%esi), %r9d // move *(that_cur_chat) byte to long 1690 addl LITERAL(2), %edi // ++this_cur_char (16-bit) 1691 addl LITERAL(1), %esi // ++that_cur_char (8-bit) 1692 subl %r9d, %r8d 1693 loope .Lstring_compareto_loop_comparison_that_compressed 1694 cmovne %r8d, %eax // return eax = *(this_cur_char) - *(that_cur_char) 1695.Lstring_compareto_keep_length2: 1696 ret 1697.Lstring_compareto_both_compressed: 1698 /* Calculate min length and count diff */ 1699 movl %r8d, %ecx 1700 movl %r8d, %eax 1701 subl %r9d, %eax 1702 cmovg %r9d, %ecx 1703 jecxz .Lstring_compareto_keep_length3 1704 repe cmpsb 1705 je .Lstring_compareto_keep_length3 1706 movzbl -1(%edi), %eax // get last compared char from this string (8-bit) 1707 movzbl -1(%esi), %ecx // get last compared char from comp string (8-bit) 1708 jmp .Lstring_compareto_count_difference 1709#endif // STRING_COMPRESSION_FEATURE 1710.Lstring_compareto_both_not_compressed: 1711 /* Calculate min length and count diff */ 1712 movl %r8d, %ecx 1713 movl %r8d, %eax 1714 subl %r9d, %eax 1715 cmovg %r9d, %ecx 1716 /* 1717 * At this point we have: 1718 * eax: value to return if first part of strings are equal 1719 * ecx: minimum among the lengths of the two strings 1720 * esi: pointer to comp string data 1721 * edi: pointer to this string data 1722 */ 1723 jecxz .Lstring_compareto_keep_length3 1724 repe cmpsw // find nonmatching chars in [%esi] and [%edi], up to length %ecx 1725 je .Lstring_compareto_keep_length3 1726 movzwl -2(%edi), %eax // get last compared char from this string (16-bit) 1727 movzwl -2(%esi), %ecx // get last compared char from comp string (16-bit) 1728.Lstring_compareto_count_difference: 1729 subl %ecx, %eax // return the difference 1730.Lstring_compareto_keep_length3: 1731 ret 1732END_FUNCTION art_quick_string_compareto 1733 1734UNIMPLEMENTED art_quick_memcmp16 1735 1736DEFINE_FUNCTION art_quick_instance_of 1737 SETUP_FP_CALLEE_SAVE_FRAME 1738 subq LITERAL(8), %rsp // Alignment padding. 1739 CFI_ADJUST_CFA_OFFSET(8) 1740 call SYMBOL(artInstanceOfFromCode) // (mirror::Object*, mirror::Class*) 1741 addq LITERAL(8), %rsp 1742 CFI_ADJUST_CFA_OFFSET(-8) 1743 RESTORE_FP_CALLEE_SAVE_FRAME 1744 ret 1745END_FUNCTION art_quick_instance_of 1746 1747DEFINE_FUNCTION art_quick_string_builder_append 1748 SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC 1749 // Outgoing argument set up 1750 leaq FRAME_SIZE_SAVE_REFS_ONLY + __SIZEOF_POINTER__(%rsp), %rsi // pass args 1751 movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() 1752 call artStringBuilderAppend // (uint32_t, const unit32_t*, Thread*) 1753 RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address 1754 RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER // return or deopt or deliver exception 1755END_FUNCTION art_quick_string_builder_append 1756 1757// Create a function `name` calling the ReadBarrier::Mark routine, 1758// getting its argument and returning its result through register 1759// `reg`, saving and restoring all caller-save registers. 1760// 1761// The generated function follows a non-standard runtime calling 1762// convention: 1763// - register `reg` (which may be different from RDI) is used to pass the (sole) argument, 1764// - register `reg` (which may be different from RAX) is used to return the result, 1765// - all other registers are callee-save (the values they hold are preserved). 1766MACRO2(READ_BARRIER_MARK_REG, name, reg) 1767 DEFINE_FUNCTION VAR(name) 1768 // Null check so that we can load the lock word. 1769 testq REG_VAR(reg), REG_VAR(reg) 1770 jz .Lret_rb_\name 1771.Lnot_null_\name: 1772 // Check the mark bit, if it is 1 return. 1773 testl LITERAL(LOCK_WORD_MARK_BIT_MASK_SHIFTED), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(reg)) 1774 jz .Lslow_rb_\name 1775 ret 1776.Lslow_rb_\name: 1777 PUSH rax 1778 movl MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(reg)), %eax 1779 addl LITERAL(LOCK_WORD_STATE_FORWARDING_ADDRESS_OVERFLOW), %eax 1780 // Jump if the addl caused eax to unsigned overflow. The only case where it overflows is the 1781 // forwarding address one. 1782 // Taken ~25% of the time. 1783 jnae .Lret_forwarding_address\name 1784 1785 // Save all potentially live caller-save core registers. 1786 movq 0(%rsp), %rax 1787 PUSH rcx 1788 PUSH rdx 1789 PUSH rsi 1790 PUSH rdi 1791 PUSH r8 1792 PUSH r9 1793 PUSH r10 1794 PUSH r11 1795 // Create space for caller-save floating-point registers. 1796 subq MACRO_LITERAL(12 * 8), %rsp 1797 CFI_ADJUST_CFA_OFFSET(12 * 8) 1798 // Save all potentially live caller-save floating-point registers. 1799 movq %xmm0, 0(%rsp) 1800 movq %xmm1, 8(%rsp) 1801 movq %xmm2, 16(%rsp) 1802 movq %xmm3, 24(%rsp) 1803 movq %xmm4, 32(%rsp) 1804 movq %xmm5, 40(%rsp) 1805 movq %xmm6, 48(%rsp) 1806 movq %xmm7, 56(%rsp) 1807 movq %xmm8, 64(%rsp) 1808 movq %xmm9, 72(%rsp) 1809 movq %xmm10, 80(%rsp) 1810 movq %xmm11, 88(%rsp) 1811 SETUP_FP_CALLEE_SAVE_FRAME 1812 1813 .ifnc RAW_VAR(reg), rdi 1814 movq REG_VAR(reg), %rdi // Pass arg1 - obj from `reg`. 1815 .endif 1816 call SYMBOL(artReadBarrierMark) // artReadBarrierMark(obj) 1817 .ifnc RAW_VAR(reg), rax 1818 movq %rax, REG_VAR(reg) // Return result into `reg`. 1819 .endif 1820 1821 RESTORE_FP_CALLEE_SAVE_FRAME 1822 // Restore floating-point registers. 1823 movq 0(%rsp), %xmm0 1824 movq 8(%rsp), %xmm1 1825 movq 16(%rsp), %xmm2 1826 movq 24(%rsp), %xmm3 1827 movq 32(%rsp), %xmm4 1828 movq 40(%rsp), %xmm5 1829 movq 48(%rsp), %xmm6 1830 movq 56(%rsp), %xmm7 1831 movq 64(%rsp), %xmm8 1832 movq 72(%rsp), %xmm9 1833 movq 80(%rsp), %xmm10 1834 movq 88(%rsp), %xmm11 1835 // Remove floating-point registers. 1836 addq MACRO_LITERAL(12 * 8), %rsp 1837 CFI_ADJUST_CFA_OFFSET(-(12 * 8)) 1838 // Restore core regs, except `reg`, as it is used to return the 1839 // result of this function (simply remove it from the stack instead). 1840 POP_REG_NE r11, RAW_VAR(reg) 1841 POP_REG_NE r10, RAW_VAR(reg) 1842 POP_REG_NE r9, RAW_VAR(reg) 1843 POP_REG_NE r8, RAW_VAR(reg) 1844 POP_REG_NE rdi, RAW_VAR(reg) 1845 POP_REG_NE rsi, RAW_VAR(reg) 1846 POP_REG_NE rdx, RAW_VAR(reg) 1847 POP_REG_NE rcx, RAW_VAR(reg) 1848 POP_REG_NE rax, RAW_VAR(reg) 1849.Lret_rb_\name: 1850 ret 1851.Lret_forwarding_address\name: 1852 // The overflow cleared the top bits. 1853 sall LITERAL(LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT), %eax 1854 movq %rax, REG_VAR(reg) 1855 POP_REG_NE rax, RAW_VAR(reg) 1856 ret 1857 END_FUNCTION VAR(name) 1858END_MACRO 1859 1860READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg00, rax 1861READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, rcx 1862READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, rdx 1863READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, rbx 1864// Note: There is no art_quick_read_barrier_mark_reg04, as register 4 (RSP) 1865// cannot be used to pass arguments. 1866READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, rbp 1867READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, rsi 1868READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, rdi 1869READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, r8 1870READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, r9 1871READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, r10 1872READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, r11 1873READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, r12 1874READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, r13 1875READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, r14 1876READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg15, r15 1877 1878DEFINE_FUNCTION art_quick_read_barrier_slow 1879 SETUP_FP_CALLEE_SAVE_FRAME 1880 subq LITERAL(8), %rsp // Alignment padding. 1881 CFI_ADJUST_CFA_OFFSET(8) 1882 call SYMBOL(artReadBarrierSlow) // artReadBarrierSlow(ref, obj, offset) 1883 addq LITERAL(8), %rsp 1884 CFI_ADJUST_CFA_OFFSET(-8) 1885 RESTORE_FP_CALLEE_SAVE_FRAME 1886 ret 1887END_FUNCTION art_quick_read_barrier_slow 1888 1889DEFINE_FUNCTION art_quick_read_barrier_for_root_slow 1890 SETUP_FP_CALLEE_SAVE_FRAME 1891 subq LITERAL(8), %rsp // Alignment padding. 1892 CFI_ADJUST_CFA_OFFSET(8) 1893 call SYMBOL(artReadBarrierForRootSlow) // artReadBarrierForRootSlow(root) 1894 addq LITERAL(8), %rsp 1895 CFI_ADJUST_CFA_OFFSET(-8) 1896 RESTORE_FP_CALLEE_SAVE_FRAME 1897 ret 1898END_FUNCTION art_quick_read_barrier_for_root_slow 1899 1900 /* 1901 * On stack replacement stub. 1902 * On entry: 1903 * [sp] = return address 1904 * rdi = stack to copy 1905 * rsi = size of stack 1906 * rdx = pc to call 1907 * rcx = JValue* result 1908 * r8 = shorty 1909 * r9 = thread 1910 * 1911 * Note that the native C ABI already aligned the stack to 16-byte. 1912 */ 1913DEFINE_FUNCTION art_quick_osr_stub 1914 // Save the non-volatiles. 1915 PUSH rbp // Save rbp. 1916 PUSH rcx // Save rcx/result*. 1917 PUSH r8 // Save r8/shorty*. 1918 1919 // Save callee saves. 1920 PUSH rbx 1921 PUSH r12 1922 PUSH r13 1923 PUSH r14 1924 PUSH r15 1925 1926 pushq LITERAL(0) // Push null for ArtMethod*. 1927 CFI_ADJUST_CFA_OFFSET(8) 1928 movl %esi, %ecx // rcx := size of stack 1929 movq %rdi, %rsi // rsi := stack to copy 1930 movq %rsp, %rbp // Save stack pointer to RBP for CFI use in .Losr_entry. 1931 call .Losr_entry 1932 CFI_REMEMBER_STATE 1933 1934 // Restore stack and callee-saves. 1935 addq LITERAL(8), %rsp 1936 CFI_ADJUST_CFA_OFFSET(-8) 1937 POP r15 1938 POP r14 1939 POP r13 1940 POP r12 1941 POP rbx 1942 POP r8 1943 POP rcx 1944 POP rbp 1945 movq %rax, (%rcx) // Store the result. 1946 ret 1947.Losr_entry: 1948 CFI_RESTORE_STATE_AND_DEF_CFA rsp, 80 1949 // Since the call has pushed the return address we need to switch the CFA register to RBP. 1950 CFI_DEF_CFA_REGISTER(rbp) 1951 1952 subl LITERAL(8), %ecx // Given stack size contains pushed frame pointer, substract it. 1953 subq %rcx, %rsp 1954 movq %rsp, %rdi // rdi := beginning of stack 1955 rep movsb // while (rcx--) { *rdi++ = *rsi++ } 1956 jmp *%rdx 1957END_FUNCTION art_quick_osr_stub 1958 1959DEFINE_FUNCTION art_quick_invoke_polymorphic 1960 // On entry: RDI := unused, RSI := receiver 1961 SETUP_SAVE_REFS_AND_ARGS_FRAME // save callee saves 1962 movq %rsi, %rdi // RDI := receiver 1963 movq %gs:THREAD_SELF_OFFSET, %rsi // RSI := Thread (self) 1964 movq %rsp, %rdx // RDX := pass SP 1965 call SYMBOL(artInvokePolymorphic) // invoke with (receiver, self, SP) 1966 // save the code pointer 1967 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1968 movq %rax, %xmm0 // Result is in RAX. Copy to FP result register. 1969 RETURN_OR_DELIVER_PENDING_EXCEPTION 1970END_FUNCTION art_quick_invoke_polymorphic 1971 1972DEFINE_FUNCTION art_quick_invoke_custom 1973 SETUP_SAVE_REFS_AND_ARGS_FRAME // save callee saves 1974 // RDI := call_site_index 1975 movq %gs:THREAD_SELF_OFFSET, %rsi // RSI := Thread::Current() 1976 movq %rsp, %rdx // RDX := SP 1977 call SYMBOL(artInvokeCustom) // artInvokeCustom(Thread*, SP) 1978 RESTORE_SAVE_REFS_AND_ARGS_FRAME 1979 movq %rax, %xmm0 // Result is in RAX. Copy to FP result register. 1980 RETURN_OR_DELIVER_PENDING_EXCEPTION 1981END_FUNCTION art_quick_invoke_custom 1982 1983// Wrap ExecuteSwitchImpl in assembly method which specifies DEX PC for unwinding. 1984// Argument 0: RDI: The context pointer for ExecuteSwitchImpl. 1985// Argument 1: RSI: Pointer to the templated ExecuteSwitchImpl to call. 1986// Argument 2: RDX: The value of DEX PC (memory address of the methods bytecode). 1987DEFINE_FUNCTION ExecuteSwitchImplAsm 1988 PUSH rbx // Spill RBX 1989 movq %rdx, %rbx // RBX = DEX PC (callee save register) 1990 CFI_DEFINE_DEX_PC_WITH_OFFSET(0 /* RAX */, 3 /* RBX */, 0) 1991 1992 call *%rsi // Call the wrapped function 1993 1994 POP rbx // Restore RBX 1995 ret 1996END_FUNCTION ExecuteSwitchImplAsm 1997 1998// On entry: edi is the class, r11 is the inline cache. r10 and rax are available. 1999DEFINE_FUNCTION art_quick_update_inline_cache 2000#if (INLINE_CACHE_SIZE != 5) 2001#error "INLINE_CACHE_SIZE not as expected." 2002#endif 2003 // Don't update the cache if we are marking. 2004 cmpl LITERAL(0), %gs:THREAD_IS_GC_MARKING_OFFSET 2005 jnz .Ldone 2006.Lentry1: 2007 movl INLINE_CACHE_CLASSES_OFFSET(%r11), %eax 2008 cmpl %edi, %eax 2009 je .Ldone 2010 cmpl LITERAL(0), %eax 2011 jne .Lentry2 2012 lock cmpxchg %edi, INLINE_CACHE_CLASSES_OFFSET(%r11) 2013 jz .Ldone 2014 jmp .Lentry1 2015.Lentry2: 2016 movl (INLINE_CACHE_CLASSES_OFFSET+4)(%r11), %eax 2017 cmpl %edi, %eax 2018 je .Ldone 2019 cmpl LITERAL(0), %eax 2020 jne .Lentry3 2021 lock cmpxchg %edi, (INLINE_CACHE_CLASSES_OFFSET+4)(%r11) 2022 jz .Ldone 2023 jmp .Lentry2 2024.Lentry3: 2025 movl (INLINE_CACHE_CLASSES_OFFSET+8)(%r11), %eax 2026 cmpl %edi, %eax 2027 je .Ldone 2028 cmpl LITERAL(0), %eax 2029 jne .Lentry4 2030 lock cmpxchg %edi, (INLINE_CACHE_CLASSES_OFFSET+8)(%r11) 2031 jz .Ldone 2032 jmp .Lentry3 2033.Lentry4: 2034 movl (INLINE_CACHE_CLASSES_OFFSET+12)(%r11), %eax 2035 cmpl %edi, %eax 2036 je .Ldone 2037 cmpl LITERAL(0), %eax 2038 jne .Lentry5 2039 lock cmpxchg %edi, (INLINE_CACHE_CLASSES_OFFSET+12)(%r11) 2040 jz .Ldone 2041 jmp .Lentry4 2042.Lentry5: 2043 // Unconditionally store, the cache is megamorphic. 2044 movl %edi, (INLINE_CACHE_CLASSES_OFFSET+16)(%r11) 2045.Ldone: 2046 ret 2047END_FUNCTION art_quick_update_inline_cache 2048 2049// On entry, method is at the bottom of the stack. 2050DEFINE_FUNCTION art_quick_compile_optimized 2051 SETUP_SAVE_EVERYTHING_FRAME 2052 movq FRAME_SIZE_SAVE_EVERYTHING(%rsp), %rdi // pass ArtMethod 2053 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 2054 call SYMBOL(artCompileOptimized) // (ArtMethod*, Thread*) 2055 RESTORE_SAVE_EVERYTHING_FRAME // restore frame up to return address 2056 ret 2057END_FUNCTION art_quick_compile_optimized 2058 2059// On entry, method is at the bottom of the stack. 2060DEFINE_FUNCTION art_quick_method_entry_hook 2061 SETUP_SAVE_EVERYTHING_FRAME 2062 2063 movq FRAME_SIZE_SAVE_EVERYTHING(%rsp), %rdi // pass ArtMethod 2064 movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() 2065 movq %rsp, %rdx // SP 2066 2067 call SYMBOL(artMethodEntryHook) // (ArtMethod*, Thread*, sp) 2068 2069 RESTORE_SAVE_EVERYTHING_FRAME 2070 ret 2071END_FUNCTION art_quick_method_entry_hook 2072 2073// On entry, method is at the bottom of the stack. 2074DEFINE_FUNCTION art_quick_method_exit_hook 2075 SETUP_SAVE_EVERYTHING_FRAME 2076 2077 // R8 passed from JITed code contains frame_size 2078 leaq 16(%rsp), %rcx // floating-point result pointer in kSaveEverything 2079 // frame 2080 leaq 144(%rsp), %rdx // integer result pointer in kSaveEverything frame 2081 leaq FRAME_SIZE_SAVE_EVERYTHING(%rsp), %rsi // ArtMethod** 2082 movq %gs:THREAD_SELF_OFFSET, %rdi // Thread::Current 2083 call SYMBOL(artMethodExitHook) // (Thread*, ArtMethod**, gpr_res*, fpr_res*, 2084 // frame_size) 2085 2086 // Normal return. 2087 RESTORE_SAVE_EVERYTHING_FRAME 2088 ret 2089END_FUNCTION art_quick_method_exit_hook 2090