• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "asm_support_x86_64.S"
18#include "interpreter/cfi_asm_support.h"
19
20#include "arch/quick_alloc_entrypoints.S"
21
22MACRO0(ASSERT_USE_READ_BARRIER)
23#if !defined(USE_READ_BARRIER)
24    int3
25    int3
26#endif
27END_MACRO
28
29// For x86, the CFA is esp+4, the address above the pushed return address on the stack.
30
31
32    /*
33     * Macro that sets up the callee save frame to conform with
34     * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs)
35     */
36MACRO0(SETUP_SAVE_REFS_AND_ARGS_FRAME)
37#if defined(__APPLE__)
38    int3
39    int3
40#else
41    SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY
42    // R10 := Runtime::Current()
43    LOAD_RUNTIME_INSTANCE r10
44    // R10 := ArtMethod* for ref and args callee save frame method.
45    movq RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET(%r10), %r10
46    // Store ArtMethod* to bottom of stack.
47    movq %r10, 0(%rsp)
48    // Store rsp as the top quick frame.
49    movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET
50#endif  // __APPLE__
51END_MACRO
52
53MACRO0(SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_RDI)
54    SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY
55    // Store ArtMethod to bottom of stack.
56    movq %rdi, 0(%rsp)
57    // Store rsp as the stop quick frame.
58    movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET
59END_MACRO
60
61    /*
62     * Macro that sets up the callee save frame to conform with
63     * Runtime::CreateCalleeSaveMethod(kSaveEverything)
64     * when R14 and R15 are already saved.
65     */
66MACRO1(SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
67#if defined(__APPLE__)
68    int3
69    int3
70#else
71    // Save core registers from highest to lowest to agree with core spills bitmap.
72    // R14 and R15, or at least placeholders for them, are already on the stack.
73    PUSH r13
74    PUSH r12
75    PUSH r11
76    PUSH r10
77    PUSH r9
78    PUSH r8
79    PUSH rdi
80    PUSH rsi
81    PUSH rbp
82    PUSH rbx
83    PUSH rdx
84    PUSH rcx
85    PUSH rax
86    // Create space for FPRs and stack alignment padding.
87    subq MACRO_LITERAL(8 + 16 * 8), %rsp
88    CFI_ADJUST_CFA_OFFSET(8 + 16 * 8)
89    // R10 := Runtime::Current()
90    LOAD_RUNTIME_INSTANCE r10
91    // Save FPRs.
92    movq %xmm0, 8(%rsp)
93    movq %xmm1, 16(%rsp)
94    movq %xmm2, 24(%rsp)
95    movq %xmm3, 32(%rsp)
96    movq %xmm4, 40(%rsp)
97    movq %xmm5, 48(%rsp)
98    movq %xmm6, 56(%rsp)
99    movq %xmm7, 64(%rsp)
100    movq %xmm8, 72(%rsp)
101    movq %xmm9, 80(%rsp)
102    movq %xmm10, 88(%rsp)
103    movq %xmm11, 96(%rsp)
104    movq %xmm12, 104(%rsp)
105    movq %xmm13, 112(%rsp)
106    movq %xmm14, 120(%rsp)
107    movq %xmm15, 128(%rsp)
108    // Push ArtMethod* for save everything frame method.
109    pushq \runtime_method_offset(%r10)
110    CFI_ADJUST_CFA_OFFSET(8)
111    // Store rsp as the top quick frame.
112    movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET
113
114    // Ugly compile-time check, but we only have the preprocessor.
115    // Last +8: implicit return address pushed on stack when caller made call.
116#if (FRAME_SIZE_SAVE_EVERYTHING != 15 * 8 + 16 * 8 + 16 + 8)
117#error "FRAME_SIZE_SAVE_EVERYTHING(X86_64) size not as expected."
118#endif
119#endif  // __APPLE__
120END_MACRO
121
122    /*
123     * Macro that sets up the callee save frame to conform with
124     * Runtime::CreateCalleeSaveMethod(kSaveEverything)
125     * when R15 is already saved.
126     */
127MACRO1(SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
128    PUSH r14
129    SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED \runtime_method_offset
130END_MACRO
131
132    /*
133     * Macro that sets up the callee save frame to conform with
134     * Runtime::CreateCalleeSaveMethod(kSaveEverything)
135     */
136MACRO1(SETUP_SAVE_EVERYTHING_FRAME, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
137    PUSH r15
138    SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED \runtime_method_offset
139END_MACRO
140
141MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_FRPS)
142    // Restore FPRs. Method and padding is still on the stack.
143    movq 16(%rsp), %xmm0
144    movq 24(%rsp), %xmm1
145    movq 32(%rsp), %xmm2
146    movq 40(%rsp), %xmm3
147    movq 48(%rsp), %xmm4
148    movq 56(%rsp), %xmm5
149    movq 64(%rsp), %xmm6
150    movq 72(%rsp), %xmm7
151    movq 80(%rsp), %xmm8
152    movq 88(%rsp), %xmm9
153    movq 96(%rsp), %xmm10
154    movq 104(%rsp), %xmm11
155    movq 112(%rsp), %xmm12
156    movq 120(%rsp), %xmm13
157    movq 128(%rsp), %xmm14
158    movq 136(%rsp), %xmm15
159END_MACRO
160
161MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_GPRS_EXCEPT_RAX)
162    // Restore callee and GPR args (except RAX), mixed together to agree with core spills bitmap.
163    POP rcx
164    POP rdx
165    POP rbx
166    POP rbp
167    POP rsi
168    POP rdi
169    POP r8
170    POP r9
171    POP r10
172    POP r11
173    POP r12
174    POP r13
175    POP r14
176    POP r15
177END_MACRO
178
179MACRO0(RESTORE_SAVE_EVERYTHING_FRAME)
180    RESTORE_SAVE_EVERYTHING_FRAME_FRPS
181
182    // Remove save everything callee save method, stack alignment padding and FPRs.
183    addq MACRO_LITERAL(16 + 16 * 8), %rsp
184    CFI_ADJUST_CFA_OFFSET(-(16 + 16 * 8))
185
186    POP rax
187    RESTORE_SAVE_EVERYTHING_FRAME_GPRS_EXCEPT_RAX
188END_MACRO
189
190MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_KEEP_RAX)
191    RESTORE_SAVE_EVERYTHING_FRAME_FRPS
192
193    // Remove save everything callee save method, stack alignment padding and FPRs, skip RAX.
194    addq MACRO_LITERAL(16 + 16 * 8 + 8), %rsp
195    CFI_ADJUST_CFA_OFFSET(-(16 + 16 * 8 + 8))
196
197    RESTORE_SAVE_EVERYTHING_FRAME_GPRS_EXCEPT_RAX
198END_MACRO
199
200MACRO2(NO_ARG_RUNTIME_EXCEPTION, c_name, cxx_name)
201    DEFINE_FUNCTION VAR(c_name)
202    SETUP_SAVE_ALL_CALLEE_SAVES_FRAME  // save all registers as basis for long jump context
203    // Outgoing argument set up
204    movq %gs:THREAD_SELF_OFFSET, %rdi  // pass Thread::Current()
205    call CALLVAR(cxx_name)             // cxx_name(Thread*)
206    UNREACHABLE
207    END_FUNCTION VAR(c_name)
208END_MACRO
209
210MACRO2(NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING, c_name, cxx_name)
211    DEFINE_FUNCTION VAR(c_name)
212    SETUP_SAVE_EVERYTHING_FRAME        // save all registers as basis for long jump context
213    // Outgoing argument set up
214    movq %gs:THREAD_SELF_OFFSET, %rdi  // pass Thread::Current()
215    call CALLVAR(cxx_name)             // cxx_name(Thread*)
216    UNREACHABLE
217    END_FUNCTION VAR(c_name)
218END_MACRO
219
220MACRO2(ONE_ARG_RUNTIME_EXCEPTION, c_name, cxx_name)
221    DEFINE_FUNCTION VAR(c_name)
222    SETUP_SAVE_ALL_CALLEE_SAVES_FRAME  // save all registers as basis for long jump context
223    // Outgoing argument set up
224    movq %gs:THREAD_SELF_OFFSET, %rsi  // pass Thread::Current()
225    call CALLVAR(cxx_name)             // cxx_name(arg1, Thread*)
226    UNREACHABLE
227    END_FUNCTION VAR(c_name)
228END_MACRO
229
230MACRO2(TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING, c_name, cxx_name)
231    DEFINE_FUNCTION VAR(c_name)
232    SETUP_SAVE_EVERYTHING_FRAME        // save all registers as basis for long jump context
233    // Outgoing argument set up
234    movq %gs:THREAD_SELF_OFFSET, %rdx  // pass Thread::Current()
235    call CALLVAR(cxx_name)             // cxx_name(Thread*)
236    UNREACHABLE
237    END_FUNCTION VAR(c_name)
238END_MACRO
239
240    /*
241     * Called by managed code to create and deliver a NullPointerException.
242     */
243NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode
244
245    /*
246     * Call installed by a signal handler to create and deliver a NullPointerException.
247     */
248DEFINE_FUNCTION_CUSTOM_CFA art_quick_throw_null_pointer_exception_from_signal, 2 * __SIZEOF_POINTER__
249    // Fault address and return address were saved by the fault handler.
250    // Save all registers as basis for long jump context; R15 will replace fault address later.
251    SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED
252    // Retrieve fault address and save R15.
253    movq (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__)(%rsp), %rdi
254    movq %r15, (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__)(%rsp)
255    CFI_REL_OFFSET(%r15, (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__))
256    // Outgoing argument set up; RDI already contains the fault address.
257    movq %gs:THREAD_SELF_OFFSET, %rsi  // pass Thread::Current()
258    call SYMBOL(artThrowNullPointerExceptionFromSignal)  // (addr, self)
259    UNREACHABLE
260END_FUNCTION art_quick_throw_null_pointer_exception_from_signal
261
262    /*
263     * Called by managed code to create and deliver an ArithmeticException.
264     */
265NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_div_zero, artThrowDivZeroFromCode
266
267    /*
268     * Called by managed code to create and deliver a StackOverflowError.
269     */
270NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode
271
272    /*
273     * Called by managed code, saves callee saves and then calls artThrowException
274     * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception.
275     */
276ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode
277
278    /*
279     * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds
280     * index, arg2 holds limit.
281     */
282TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_array_bounds, artThrowArrayBoundsFromCode
283
284    /*
285     * Called by managed code to create and deliver a StringIndexOutOfBoundsException
286     * as if thrown from a call to String.charAt(). Arg1 holds index, arg2 holds limit.
287     */
288TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_string_bounds, artThrowStringBoundsFromCode
289
290    /*
291     * All generated callsites for interface invokes and invocation slow paths will load arguments
292     * as usual - except instead of loading arg0/rdi with the target Method*, arg0/rdi will contain
293     * the method_idx.  This wrapper will save arg1-arg3, and call the appropriate C helper.
294     * NOTE: "this" is first visible argument of the target, and so can be found in arg1/rsi.
295     *
296     * The helper will attempt to locate the target and return a 128-bit result in rax/rdx consisting
297     * of the target Method* in rax and method->code_ in rdx.
298     *
299     * If unsuccessful, the helper will return null/????. There will be a pending exception in the
300     * thread and we branch to another stub to deliver it.
301     *
302     * On success this wrapper will restore arguments and *jump* to the target, leaving the return
303     * location on the stack.
304     *
305     * Adapted from x86 code.
306     */
307MACRO1(INVOKE_TRAMPOLINE_BODY, cxx_name)
308    SETUP_SAVE_REFS_AND_ARGS_FRAME  // save callee saves in case allocation triggers GC
309    // Helper signature is always
310    // (method_idx, *this_object, *caller_method, *self, sp)
311
312    movq %gs:THREAD_SELF_OFFSET, %rdx                      // pass Thread
313    movq %rsp, %rcx                                        // pass SP
314
315    call CALLVAR(cxx_name)                                 // cxx_name(arg1, arg2, Thread*, SP)
316                                                           // save the code pointer
317    movq %rax, %rdi
318    movq %rdx, %rax
319    RESTORE_SAVE_REFS_AND_ARGS_FRAME
320
321    testq %rdi, %rdi
322    jz 1f
323
324    // Tail call to intended method.
325    jmp *%rax
3261:
327    DELIVER_PENDING_EXCEPTION
328END_MACRO
329MACRO2(INVOKE_TRAMPOLINE, c_name, cxx_name)
330    DEFINE_FUNCTION VAR(c_name)
331    INVOKE_TRAMPOLINE_BODY RAW_VAR(cxx_name)
332    END_FUNCTION VAR(c_name)
333END_MACRO
334
335INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, artInvokeInterfaceTrampolineWithAccessCheck
336
337INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, artInvokeStaticTrampolineWithAccessCheck
338INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, artInvokeDirectTrampolineWithAccessCheck
339INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, artInvokeSuperTrampolineWithAccessCheck
340INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, artInvokeVirtualTrampolineWithAccessCheck
341
342
343    /*
344     * Helper for quick invocation stub to set up XMM registers. Assumes r10 == shorty,
345     * r11 == arg_array. Clobbers r10, r11 and al. Branches to xmm_setup_finished if it encounters
346     * the end of the shorty.
347     */
348MACRO2(LOOP_OVER_SHORTY_LOADING_XMMS, xmm_reg, finished)
3491: // LOOP
350    movb (%r10), %al              // al := *shorty
351    addq MACRO_LITERAL(1), %r10   // shorty++
352    cmpb MACRO_LITERAL(0), %al    // if (al == '\0') goto xmm_setup_finished
353    je VAR(finished)
354    cmpb MACRO_LITERAL(68), %al   // if (al == 'D') goto FOUND_DOUBLE
355    je 2f
356    cmpb MACRO_LITERAL(70), %al   // if (al == 'F') goto FOUND_FLOAT
357    je 3f
358    addq MACRO_LITERAL(4), %r11   // arg_array++
359    //  Handle extra space in arg array taken by a long.
360    cmpb MACRO_LITERAL(74), %al   // if (al != 'J') goto LOOP
361    jne 1b
362    addq MACRO_LITERAL(4), %r11   // arg_array++
363    jmp 1b                        // goto LOOP
3642:  // FOUND_DOUBLE
365    movsd (%r11), REG_VAR(xmm_reg)
366    addq MACRO_LITERAL(8), %r11   // arg_array+=2
367    jmp 4f
3683:  // FOUND_FLOAT
369    movss (%r11), REG_VAR(xmm_reg)
370    addq MACRO_LITERAL(4), %r11   // arg_array++
3714:
372END_MACRO
373
374    /*
375     * Helper for quick invocation stub to set up GPR registers. Assumes r10 == shorty,
376     * r11 == arg_array. Clobbers r10, r11 and al. Branches to gpr_setup_finished if it encounters
377     * the end of the shorty.
378     */
379MACRO3(LOOP_OVER_SHORTY_LOADING_GPRS, gpr_reg64, gpr_reg32, finished)
3801: // LOOP
381    movb (%r10), %al              // al := *shorty
382    addq MACRO_LITERAL(1), %r10   // shorty++
383    cmpb MACRO_LITERAL(0), %al    // if (al == '\0') goto gpr_setup_finished
384    je  VAR(finished)
385    cmpb MACRO_LITERAL(74), %al   // if (al == 'J') goto FOUND_LONG
386    je 2f
387    cmpb MACRO_LITERAL(70), %al   // if (al == 'F') goto SKIP_FLOAT
388    je 3f
389    cmpb MACRO_LITERAL(68), %al   // if (al == 'D') goto SKIP_DOUBLE
390    je 4f
391    movl (%r11), REG_VAR(gpr_reg32)
392    addq MACRO_LITERAL(4), %r11   // arg_array++
393    jmp 5f
3942:  // FOUND_LONG
395    movq (%r11), REG_VAR(gpr_reg64)
396    addq MACRO_LITERAL(8), %r11   // arg_array+=2
397    jmp 5f
3983:  // SKIP_FLOAT
399    addq MACRO_LITERAL(4), %r11   // arg_array++
400    jmp 1b
4014:  // SKIP_DOUBLE
402    addq MACRO_LITERAL(8), %r11   // arg_array+=2
403    jmp 1b
4045:
405END_MACRO
406
407    /*
408     * Quick invocation stub.
409     * On entry:
410     *   [sp] = return address
411     *   rdi = method pointer
412     *   rsi = argument array that must at least contain the this pointer.
413     *   rdx = size of argument array in bytes
414     *   rcx = (managed) thread pointer
415     *   r8 = JValue* result
416     *   r9 = char* shorty
417     */
418DEFINE_FUNCTION art_quick_invoke_stub
419#if defined(__APPLE__)
420    int3
421    int3
422#else
423    // Set up argument XMM registers.
424    leaq 1(%r9), %r10             // R10 := shorty + 1  ; ie skip return arg character.
425    leaq 4(%rsi), %r11            // R11 := arg_array + 4 ; ie skip this pointer.
426    LOOP_OVER_SHORTY_LOADING_XMMS xmm0, .Lxmm_setup_finished
427    LOOP_OVER_SHORTY_LOADING_XMMS xmm1, .Lxmm_setup_finished
428    LOOP_OVER_SHORTY_LOADING_XMMS xmm2, .Lxmm_setup_finished
429    LOOP_OVER_SHORTY_LOADING_XMMS xmm3, .Lxmm_setup_finished
430    LOOP_OVER_SHORTY_LOADING_XMMS xmm4, .Lxmm_setup_finished
431    LOOP_OVER_SHORTY_LOADING_XMMS xmm5, .Lxmm_setup_finished
432    LOOP_OVER_SHORTY_LOADING_XMMS xmm6, .Lxmm_setup_finished
433    LOOP_OVER_SHORTY_LOADING_XMMS xmm7, .Lxmm_setup_finished
434    .balign 16
435.Lxmm_setup_finished:
436    PUSH rbp                      // Save rbp.
437    PUSH r8                       // Save r8/result*.
438    PUSH r9                       // Save r9/shorty*.
439    PUSH rbx                      // Save native callee save rbx
440    PUSH r12                      // Save native callee save r12
441    PUSH r13                      // Save native callee save r13
442    PUSH r14                      // Save native callee save r14
443    PUSH r15                      // Save native callee save r15
444    movq %rsp, %rbp               // Copy value of stack pointer into base pointer.
445    CFI_DEF_CFA_REGISTER(rbp)
446
447    movl %edx, %r10d
448    addl LITERAL(100), %edx        // Reserve space for return addr, StackReference<method>, rbp,
449                                   // r8, r9, rbx, r12, r13, r14, and r15 in frame.
450    andl LITERAL(0xFFFFFFF0), %edx // Align frame size to 16 bytes.
451    subl LITERAL(72), %edx         // Remove space for return address, rbp, r8, r9, rbx, r12,
452                                   // r13, r14, and r15
453    subq %rdx, %rsp                // Reserve stack space for argument array.
454
455#if (STACK_REFERENCE_SIZE != 4)
456#error "STACK_REFERENCE_SIZE(X86_64) size not as expected."
457#endif
458    movq LITERAL(0), (%rsp)       // Store null for method*
459
460    movl %r10d, %ecx              // Place size of args in rcx.
461    movq %rdi, %rax               // rax := method to be called
462    movq %rsi, %r11               // r11 := arg_array
463    leaq 8(%rsp), %rdi            // rdi is pointing just above the ArtMethod* in the stack
464                                  // arguments.
465    // Copy arg array into stack.
466    rep movsb                     // while (rcx--) { *rdi++ = *rsi++ }
467    leaq 1(%r9), %r10             // r10 := shorty + 1  ; ie skip return arg character
468    movq %rax, %rdi               // rdi := method to be called
469    movl (%r11), %esi             // rsi := this pointer
470    addq LITERAL(4), %r11         // arg_array++
471    LOOP_OVER_SHORTY_LOADING_GPRS rdx, edx, .Lgpr_setup_finished
472    LOOP_OVER_SHORTY_LOADING_GPRS rcx, ecx, .Lgpr_setup_finished
473    LOOP_OVER_SHORTY_LOADING_GPRS r8, r8d, .Lgpr_setup_finished
474    LOOP_OVER_SHORTY_LOADING_GPRS r9, r9d, .Lgpr_setup_finished
475.Lgpr_setup_finished:
476    call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method.
477    movq %rbp, %rsp               // Restore stack pointer.
478    POP r15                       // Pop r15
479    POP r14                       // Pop r14
480    POP r13                       // Pop r13
481    POP r12                       // Pop r12
482    POP rbx                       // Pop rbx
483    POP r9                        // Pop r9 - shorty*
484    POP r8                        // Pop r8 - result*.
485    POP rbp                       // Pop rbp
486    cmpb LITERAL(68), (%r9)       // Test if result type char == 'D'.
487    je .Lreturn_double_quick
488    cmpb LITERAL(70), (%r9)       // Test if result type char == 'F'.
489    je .Lreturn_float_quick
490    movq %rax, (%r8)              // Store the result assuming its a long, int or Object*
491    ret
492.Lreturn_double_quick:
493    movsd %xmm0, (%r8)            // Store the double floating point result.
494    ret
495.Lreturn_float_quick:
496    movss %xmm0, (%r8)            // Store the floating point result.
497    ret
498#endif  // __APPLE__
499END_FUNCTION art_quick_invoke_stub
500
501    /*
502     * Quick invocation stub.
503     * On entry:
504     *   [sp] = return address
505     *   rdi = method pointer
506     *   rsi = argument array or null if no arguments.
507     *   rdx = size of argument array in bytes
508     *   rcx = (managed) thread pointer
509     *   r8 = JValue* result
510     *   r9 = char* shorty
511     */
512DEFINE_FUNCTION art_quick_invoke_static_stub
513#if defined(__APPLE__)
514    int3
515    int3
516#else
517    // Set up argument XMM registers.
518    leaq 1(%r9), %r10             // R10 := shorty + 1  ; ie skip return arg character
519    movq %rsi, %r11               // R11 := arg_array
520    LOOP_OVER_SHORTY_LOADING_XMMS xmm0, .Lxmm_setup_finished2
521    LOOP_OVER_SHORTY_LOADING_XMMS xmm1, .Lxmm_setup_finished2
522    LOOP_OVER_SHORTY_LOADING_XMMS xmm2, .Lxmm_setup_finished2
523    LOOP_OVER_SHORTY_LOADING_XMMS xmm3, .Lxmm_setup_finished2
524    LOOP_OVER_SHORTY_LOADING_XMMS xmm4, .Lxmm_setup_finished2
525    LOOP_OVER_SHORTY_LOADING_XMMS xmm5, .Lxmm_setup_finished2
526    LOOP_OVER_SHORTY_LOADING_XMMS xmm6, .Lxmm_setup_finished2
527    LOOP_OVER_SHORTY_LOADING_XMMS xmm7, .Lxmm_setup_finished2
528    .balign 16
529.Lxmm_setup_finished2:
530    PUSH rbp                      // Save rbp.
531    PUSH r8                       // Save r8/result*.
532    PUSH r9                       // Save r9/shorty*.
533    PUSH rbx                      // Save rbx
534    PUSH r12                      // Save r12
535    PUSH r13                      // Save r13
536    PUSH r14                      // Save r14
537    PUSH r15                      // Save r15
538    movq %rsp, %rbp               // Copy value of stack pointer into base pointer.
539    CFI_DEF_CFA_REGISTER(rbp)
540
541    movl %edx, %r10d
542    addl LITERAL(100), %edx        // Reserve space for return addr, StackReference<method>, rbp,
543                                   // r8, r9, r12, r13, r14, and r15 in frame.
544    andl LITERAL(0xFFFFFFF0), %edx // Align frame size to 16 bytes.
545    subl LITERAL(72), %edx         // Remove space for return address, rbp, r8, r9, rbx, r12,
546                                   // r13, r14, and r15.
547    subq %rdx, %rsp                // Reserve stack space for argument array.
548
549#if (STACK_REFERENCE_SIZE != 4)
550#error "STACK_REFERENCE_SIZE(X86_64) size not as expected."
551#endif
552    movq LITERAL(0), (%rsp)        // Store null for method*
553
554    movl %r10d, %ecx               // Place size of args in rcx.
555    movq %rdi, %rax                // rax := method to be called
556    movq %rsi, %r11                // r11 := arg_array
557    leaq 8(%rsp), %rdi             // rdi is pointing just above the ArtMethod* in the
558                                   // stack arguments.
559    // Copy arg array into stack.
560    rep movsb                      // while (rcx--) { *rdi++ = *rsi++ }
561    leaq 1(%r9), %r10              // r10 := shorty + 1  ; ie skip return arg character
562    movq %rax, %rdi                // rdi := method to be called
563    LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, .Lgpr_setup_finished2
564    LOOP_OVER_SHORTY_LOADING_GPRS rdx, edx, .Lgpr_setup_finished2
565    LOOP_OVER_SHORTY_LOADING_GPRS rcx, ecx, .Lgpr_setup_finished2
566    LOOP_OVER_SHORTY_LOADING_GPRS r8, r8d, .Lgpr_setup_finished2
567    LOOP_OVER_SHORTY_LOADING_GPRS r9, r9d, .Lgpr_setup_finished2
568.Lgpr_setup_finished2:
569    call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method.
570    movq %rbp, %rsp                // Restore stack pointer.
571    POP r15                        // Pop r15
572    POP r14                        // Pop r14
573    POP r13                        // Pop r13
574    POP r12                        // Pop r12
575    POP rbx                        // Pop rbx
576    POP r9                         // Pop r9 - shorty*.
577    POP r8                         // Pop r8 - result*.
578    POP rbp                        // Pop rbp
579    cmpb LITERAL(68), (%r9)        // Test if result type char == 'D'.
580    je .Lreturn_double_quick2
581    cmpb LITERAL(70), (%r9)        // Test if result type char == 'F'.
582    je .Lreturn_float_quick2
583    movq %rax, (%r8)               // Store the result assuming its a long, int or Object*
584    ret
585.Lreturn_double_quick2:
586    movsd %xmm0, (%r8)             // Store the double floating point result.
587    ret
588.Lreturn_float_quick2:
589    movss %xmm0, (%r8)             // Store the floating point result.
590    ret
591#endif  // __APPLE__
592END_FUNCTION art_quick_invoke_static_stub
593
594    /*
595     * Long jump stub.
596     * On entry:
597     *   rdi = gprs
598     *   rsi = fprs
599     */
600DEFINE_FUNCTION art_quick_do_long_jump
601#if defined(__APPLE__)
602    int3
603    int3
604#else
605    // Restore FPRs.
606    movq 0(%rsi), %xmm0
607    movq 8(%rsi), %xmm1
608    movq 16(%rsi), %xmm2
609    movq 24(%rsi), %xmm3
610    movq 32(%rsi), %xmm4
611    movq 40(%rsi), %xmm5
612    movq 48(%rsi), %xmm6
613    movq 56(%rsi), %xmm7
614    movq 64(%rsi), %xmm8
615    movq 72(%rsi), %xmm9
616    movq 80(%rsi), %xmm10
617    movq 88(%rsi), %xmm11
618    movq 96(%rsi), %xmm12
619    movq 104(%rsi), %xmm13
620    movq 112(%rsi), %xmm14
621    movq 120(%rsi), %xmm15
622    // Restore FPRs.
623    movq %rdi, %rsp   // RSP points to gprs.
624    // Load all registers except RSP and RIP with values in gprs.
625    popq %r15
626    popq %r14
627    popq %r13
628    popq %r12
629    popq %r11
630    popq %r10
631    popq %r9
632    popq %r8
633    popq %rdi
634    popq %rsi
635    popq %rbp
636    addq LITERAL(8), %rsp   // Skip rsp
637    popq %rbx
638    popq %rdx
639    popq %rcx
640    popq %rax
641    popq %rsp      // Load stack pointer.
642    ret            // From higher in the stack pop rip.
643#endif  // __APPLE__
644END_FUNCTION art_quick_do_long_jump
645
646MACRO3(ONE_ARG_DOWNCALL, c_name, cxx_name, return_macro)
647    DEFINE_FUNCTION VAR(c_name)
648    SETUP_SAVE_REFS_ONLY_FRAME           // save ref containing registers for GC
649    // Outgoing argument set up
650    movq %gs:THREAD_SELF_OFFSET, %rsi    // pass Thread::Current()
651    call CALLVAR(cxx_name)               // cxx_name(arg0, Thread*)
652    RESTORE_SAVE_REFS_ONLY_FRAME         // restore frame up to return address
653    CALL_MACRO(return_macro)             // return or deliver exception
654    END_FUNCTION VAR(c_name)
655END_MACRO
656
657MACRO3(TWO_ARG_DOWNCALL, c_name, cxx_name, return_macro)
658    DEFINE_FUNCTION VAR(c_name)
659    SETUP_SAVE_REFS_ONLY_FRAME           // save ref containing registers for GC
660    // Outgoing argument set up
661    movq %gs:THREAD_SELF_OFFSET, %rdx    // pass Thread::Current()
662    call CALLVAR(cxx_name)               // cxx_name(arg0, arg1, Thread*)
663    RESTORE_SAVE_REFS_ONLY_FRAME         // restore frame up to return address
664    CALL_MACRO(return_macro)             // return or deliver exception
665    END_FUNCTION VAR(c_name)
666END_MACRO
667
668MACRO3(THREE_ARG_DOWNCALL, c_name, cxx_name, return_macro)
669    DEFINE_FUNCTION VAR(c_name)
670    SETUP_SAVE_REFS_ONLY_FRAME          // save ref containing registers for GC
671    // Outgoing argument set up
672    movq %gs:THREAD_SELF_OFFSET, %rcx   // pass Thread::Current()
673    call CALLVAR(cxx_name)              // cxx_name(arg0, arg1, arg2, Thread*)
674    RESTORE_SAVE_REFS_ONLY_FRAME        // restore frame up to return address
675    CALL_MACRO(return_macro)            // return or deliver exception
676    END_FUNCTION VAR(c_name)
677END_MACRO
678
679MACRO3(FOUR_ARG_DOWNCALL, c_name, cxx_name, return_macro)
680    DEFINE_FUNCTION VAR(c_name)
681    SETUP_SAVE_REFS_ONLY_FRAME          // save ref containing registers for GC
682    // Outgoing argument set up
683    movq %gs:THREAD_SELF_OFFSET, %r8    // pass Thread::Current()
684    call CALLVAR(cxx_name)              // cxx_name(arg1, arg2, arg3, arg4, Thread*)
685    RESTORE_SAVE_REFS_ONLY_FRAME        // restore frame up to return address
686    CALL_MACRO(return_macro)            // return or deliver exception
687    END_FUNCTION VAR(c_name)
688END_MACRO
689
690MACRO3(ONE_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro)
691    DEFINE_FUNCTION VAR(c_name)
692    SETUP_SAVE_REFS_ONLY_FRAME
693                                        // arg0 is in rdi
694    movq %gs:THREAD_SELF_OFFSET, %rsi   // pass Thread::Current()
695    call CALLVAR(cxx_name)              // cxx_name(arg0, Thread*)
696    RESTORE_SAVE_REFS_ONLY_FRAME        // restore frame up to return address
697    CALL_MACRO(return_macro)
698    END_FUNCTION VAR(c_name)
699END_MACRO
700
701MACRO3(TWO_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro)
702    DEFINE_FUNCTION VAR(c_name)
703    SETUP_SAVE_REFS_ONLY_FRAME
704                                        // arg0 and arg1 are in rdi/rsi
705    movq %gs:THREAD_SELF_OFFSET, %rdx   // pass Thread::Current()
706    call CALLVAR(cxx_name)              // (arg0, arg1, Thread*)
707    RESTORE_SAVE_REFS_ONLY_FRAME        // restore frame up to return address
708    CALL_MACRO(return_macro)
709    END_FUNCTION VAR(c_name)
710END_MACRO
711
712MACRO3(THREE_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro)
713    DEFINE_FUNCTION VAR(c_name)
714    SETUP_SAVE_REFS_ONLY_FRAME
715                                        // arg0, arg1, and arg2 are in rdi/rsi/rdx
716    movq %gs:THREAD_SELF_OFFSET, %rcx   // pass Thread::Current()
717    call CALLVAR(cxx_name)              // cxx_name(arg0, arg1, arg2, Thread*)
718    RESTORE_SAVE_REFS_ONLY_FRAME        // restore frame up to return address
719    CALL_MACRO(return_macro)            // return or deliver exception
720    END_FUNCTION VAR(c_name)
721END_MACRO
722
723    /*
724     * Macro for resolution and initialization of indexed DEX file
725     * constants such as classes and strings.
726     */
727MACRO3(ONE_ARG_SAVE_EVERYTHING_DOWNCALL, c_name, cxx_name, runtime_method_offset = RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET)
728    DEFINE_FUNCTION VAR(c_name)
729    SETUP_SAVE_EVERYTHING_FRAME \runtime_method_offset  // save everything for GC
730    // Outgoing argument set up
731    movl %eax, %edi                               // pass the index of the constant as arg0
732    movq %gs:THREAD_SELF_OFFSET, %rsi             // pass Thread::Current()
733    call CALLVAR(cxx_name)                        // cxx_name(arg0, Thread*)
734    testl %eax, %eax                              // If result is null, deliver pending exception.
735    jz 1f
736    DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_RAX /*is_ref=*/1
7371:
738    DELIVER_PENDING_EXCEPTION_FRAME_READY
739    END_FUNCTION VAR(c_name)
740END_MACRO
741
742MACRO2(ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT, c_name, cxx_name)
743    ONE_ARG_SAVE_EVERYTHING_DOWNCALL \c_name, \cxx_name, RUNTIME_SAVE_EVERYTHING_FOR_CLINIT_METHOD_OFFSET
744END_MACRO
745
746MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER)
747    testq %rax, %rax               // rax == 0 ?
748    jz  1f                         // if rax == 0 goto 1
749    DEOPT_OR_RETURN /*is_ref=*/1   // Check if deopt is required
7501:                                 // deliver exception on current thread
751    DELIVER_PENDING_EXCEPTION
752END_MACRO
753
754
755MACRO0(RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION)
756    movq %gs:THREAD_EXCEPTION_OFFSET, %rcx // get exception field
757    testq %rcx, %rcx               // rcx == 0 ?
758    jnz 1f                         // if rcx != 0 goto 1
759    DEOPT_OR_RETURN                // Check if deopt is required
7601:                                 // deliver exception on current thread
761    DELIVER_PENDING_EXCEPTION
762END_MACRO
763
764MACRO1(DEOPT_OR_RETURN, is_ref = 0)
765  cmpl LITERAL(0), %gs:THREAD_DEOPT_CHECK_REQUIRED_OFFSET
766  jne 2f
767  ret
7682:
769  SETUP_SAVE_EVERYTHING_FRAME
770  movq LITERAL(\is_ref), %rdx          // pass if result is a reference
771  movq %rax, %rsi                      // pass the result
772  movq %gs:THREAD_SELF_OFFSET, %rdi    // pass Thread::Current
773  call SYMBOL(artDeoptimizeIfNeeded)
774  CFI_REMEMBER_STATE
775  RESTORE_SAVE_EVERYTHING_FRAME
776  ret
777  CFI_RESTORE_STATE_AND_DEF_CFA rsp, FRAME_SIZE_SAVE_EVERYTHING
778END_MACRO
779
780MACRO1(DEOPT_OR_RESTORE_SAVE_EVERYTHING_FRAME_AND_RETURN_RAX, is_ref = 0)
781  cmpl LITERAL(0), %gs:THREAD_DEOPT_CHECK_REQUIRED_OFFSET
782  jne 2f
783  CFI_REMEMBER_STATE
784  RESTORE_SAVE_EVERYTHING_FRAME_KEEP_RAX
785  ret
786  CFI_RESTORE_STATE_AND_DEF_CFA rsp, FRAME_SIZE_SAVE_EVERYTHING
7872:
788  movq %rax, SAVE_EVERYTHING_FRAME_RAX_OFFSET(%rsp) // update result in the frame
789  movq LITERAL(\is_ref), %rdx                       // pass if result is a reference
790  movq %rax, %rsi                                   // pass the result
791  movq %gs:THREAD_SELF_OFFSET, %rdi                 // pass Thread::Current
792  call SYMBOL(artDeoptimizeIfNeeded)
793  CFI_REMEMBER_STATE
794  RESTORE_SAVE_EVERYTHING_FRAME
795  ret
796  CFI_RESTORE_STATE_AND_DEF_CFA rsp, FRAME_SIZE_SAVE_EVERYTHING
797END_MACRO
798
799
800
801MACRO0(RETURN_IF_EAX_ZERO)
802    testl %eax, %eax               // eax == 0 ?
803    jnz  1f                        // if eax != 0 goto 1
804    DEOPT_OR_RETURN                // Check if we need a deopt
8051:                                 // deliver exception on current thread
806    DELIVER_PENDING_EXCEPTION
807END_MACRO
808
809// Generate the allocation entrypoints for each allocator.
810GENERATE_ALLOC_ENTRYPOINTS_FOR_NON_TLAB_ALLOCATORS
811
812// Comment out allocators that have x86_64 specific asm.
813// Region TLAB:
814// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB)
815// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB)
816GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB)
817GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_region_tlab, RegionTLAB)
818// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB)
819// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_region_tlab, RegionTLAB)
820// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_region_tlab, RegionTLAB)
821// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_region_tlab, RegionTLAB)
822// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_region_tlab, RegionTLAB)
823GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_region_tlab, RegionTLAB)
824GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_region_tlab, RegionTLAB)
825GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_region_tlab, RegionTLAB)
826// Normal TLAB:
827// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB)
828// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB)
829GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_tlab, TLAB)
830GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_OBJECT(_tlab, TLAB)
831// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_tlab, TLAB)
832// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED8(_tlab, TLAB)
833// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED16(_tlab, TLAB)
834// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED32(_tlab, TLAB)
835// GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED64(_tlab, TLAB)
836GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_tlab, TLAB)
837GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_tlab, TLAB)
838GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_tlab, TLAB)
839
840
841// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_rosalloc, RosAlloc).
842MACRO2(ART_QUICK_ALLOC_OBJECT_ROSALLOC, c_name, cxx_name)
843    DEFINE_FUNCTION VAR(c_name)
844    // Fast path rosalloc allocation.
845    // RDI: mirror::Class*, RAX: return value
846    // RSI, RDX, RCX, R8, R9: free.
847                                                           // Check if the thread local
848                                                           // allocation stack has room.
849    movq   %gs:THREAD_SELF_OFFSET, %r8                     // r8 = thread
850    movq   THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(%r8), %rcx  // rcx = alloc stack top.
851    cmpq   THREAD_LOCAL_ALLOC_STACK_END_OFFSET(%r8), %rcx
852    jae    .Lslow_path\c_name
853                                                           // Load the object size
854    movl   MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET(%rdi), %eax
855                                                           // Check if the size is for a thread
856                                                           // local allocation. Also does the
857                                                           // initialized and finalizable checks.
858    cmpl   LITERAL(ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE), %eax
859    ja     .Lslow_path\c_name
860                                                           // Compute the rosalloc bracket index
861                                                           // from the size.
862    shrq   LITERAL(ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT), %rax
863                                                           // Load the rosalloc run (r9)
864                                                           // Subtract __SIZEOF_POINTER__ to
865                                                           // subtract one from edi as there is no
866                                                           // 0 byte run and the size is already
867                                                           // aligned.
868    movq   (THREAD_ROSALLOC_RUNS_OFFSET - __SIZEOF_POINTER__)(%r8, %rax, __SIZEOF_POINTER__), %r9
869                                                           // Load the free list head (rax). This
870                                                           // will be the return val.
871    movq   (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)(%r9), %rax
872    testq  %rax, %rax
873    jz     .Lslow_path\c_name
874    // "Point of no slow path". Won't go to the slow path from here on. OK to clobber rdi and rsi.
875                                                           // Push the new object onto the thread
876                                                           // local allocation stack and
877                                                           // increment the thread local
878                                                           // allocation stack top.
879    movl   %eax, (%rcx)
880    addq   LITERAL(COMPRESSED_REFERENCE_SIZE), %rcx
881    movq   %rcx, THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(%r8)
882                                                           // Load the next pointer of the head
883                                                           // and update the list head with the
884                                                           // next pointer.
885    movq   ROSALLOC_SLOT_NEXT_OFFSET(%rax), %rcx
886    movq   %rcx, (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)(%r9)
887                                                           // Store the class pointer in the
888                                                           // header. This also overwrites the
889                                                           // next pointer. The offsets are
890                                                           // asserted to match.
891#if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET
892#error "Class pointer needs to overwrite next pointer."
893#endif
894    POISON_HEAP_REF edi
895    movl   %edi, MIRROR_OBJECT_CLASS_OFFSET(%rax)
896                                                           // Decrement the size of the free list
897    decl   (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)(%r9)
898                                                           // No fence necessary for x86.
899    ret
900.Lslow_path\c_name:
901    SETUP_SAVE_REFS_ONLY_FRAME                             // save ref containing registers for GC
902    // Outgoing argument set up
903    movq %gs:THREAD_SELF_OFFSET, %rsi                      // pass Thread::Current()
904    call CALLVAR(cxx_name)                                 // cxx_name(arg0, Thread*)
905    RESTORE_SAVE_REFS_ONLY_FRAME                           // restore frame up to return address
906    RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER       // return or deliver exception
907    END_FUNCTION VAR(c_name)
908END_MACRO
909
910ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_resolved_rosalloc, artAllocObjectFromCodeResolvedRosAlloc
911ART_QUICK_ALLOC_OBJECT_ROSALLOC art_quick_alloc_object_initialized_rosalloc, artAllocObjectFromCodeInitializedRosAlloc
912
913// The common fast path code for art_quick_alloc_object_resolved_region_tlab.
914// TODO: delete ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH since it is the same as
915// ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH.
916//
917// RDI: the class, RAX: return value.
918// RCX, RSI, RDX: scratch, r8: Thread::Current().
919MACRO1(ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH, slowPathLabel)
920    ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH(RAW_VAR(slowPathLabel))
921END_MACRO
922
923// The fast path code for art_quick_alloc_object_initialized_region_tlab.
924//
925// RDI: the class, RSI: ArtMethod*, RAX: return value.
926// RCX, RSI, RDX: scratch, r8: Thread::Current().
927MACRO1(ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH, slowPathLabel)
928    movq %gs:THREAD_SELF_OFFSET, %r8                           // r8 = thread
929    movl MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET(%rdi), %ecx // Load the object size.
930    movq THREAD_LOCAL_POS_OFFSET(%r8), %rax
931    addq %rax, %rcx                                            // Add size to pos, note that these
932                                                               // are both 32 bit ints, overflow
933                                                               // will cause the add to be past the
934                                                               // end of the thread local region.
935    cmpq THREAD_LOCAL_END_OFFSET(%r8), %rcx                    // Check if it fits.
936    ja   RAW_VAR(slowPathLabel)
937    movq %rcx, THREAD_LOCAL_POS_OFFSET(%r8)                    // Update thread_local_pos.
938    incq THREAD_LOCAL_OBJECTS_OFFSET(%r8)                      // Increase thread_local_objects.
939                                                               // Store the class pointer in the
940                                                               // header.
941                                                               // No fence needed for x86.
942    POISON_HEAP_REF edi
943    movl %edi, MIRROR_OBJECT_CLASS_OFFSET(%rax)
944    ret                                                        // Fast path succeeded.
945END_MACRO
946
947// The fast path code for art_quick_alloc_array_region_tlab.
948// Inputs: RDI: the class, RSI: int32_t component_count, R9: total_size
949// Free temps: RCX, RDX, R8
950// Output: RAX: return value.
951MACRO1(ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE, slowPathLabel)
952    movq %gs:THREAD_SELF_OFFSET, %rcx                          // rcx = thread
953    // Mask out the unaligned part to make sure we are 8 byte aligned.
954    andq LITERAL(OBJECT_ALIGNMENT_MASK_TOGGLED64), %r9
955    movq THREAD_LOCAL_POS_OFFSET(%rcx), %rax
956    addq %rax, %r9
957    cmpq THREAD_LOCAL_END_OFFSET(%rcx), %r9                    // Check if it fits.
958    ja   RAW_VAR(slowPathLabel)
959    movq %r9, THREAD_LOCAL_POS_OFFSET(%rcx)                    // Update thread_local_pos.
960    addq LITERAL(1), THREAD_LOCAL_OBJECTS_OFFSET(%rcx)         // Increase thread_local_objects.
961                                                               // Store the class pointer in the
962                                                               // header.
963                                                               // No fence needed for x86.
964    POISON_HEAP_REF edi
965    movl %edi, MIRROR_OBJECT_CLASS_OFFSET(%rax)
966    movl %esi, MIRROR_ARRAY_LENGTH_OFFSET(%rax)
967    ret                                                        // Fast path succeeded.
968END_MACRO
969
970// The common slow path code for art_quick_alloc_object_{resolved, initialized}_tlab
971// and art_quick_alloc_object_{resolved, initialized}_region_tlab.
972MACRO1(ALLOC_OBJECT_TLAB_SLOW_PATH, cxx_name)
973    SETUP_SAVE_REFS_ONLY_FRAME                             // save ref containing registers for GC
974    // Outgoing argument set up
975    movq %gs:THREAD_SELF_OFFSET, %rsi                      // pass Thread::Current()
976    call CALLVAR(cxx_name)                                 // cxx_name(arg0, Thread*)
977    RESTORE_SAVE_REFS_ONLY_FRAME                           // restore frame up to return address
978    RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER       // return or deliver exception
979END_MACRO
980
981// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB). May be
982// called with CC if the GC is not active.
983DEFINE_FUNCTION art_quick_alloc_object_resolved_tlab
984    // RDI: mirror::Class* klass
985    // RDX, RSI, RCX, R8, R9: free. RAX: return val.
986    ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH .Lart_quick_alloc_object_resolved_tlab_slow_path
987.Lart_quick_alloc_object_resolved_tlab_slow_path:
988    ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeResolvedTLAB
989END_FUNCTION art_quick_alloc_object_resolved_tlab
990
991// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB).
992// May be called with CC if the GC is not active.
993DEFINE_FUNCTION art_quick_alloc_object_initialized_tlab
994    // RDI: mirror::Class* klass
995    // RDX, RSI, RCX, R8, R9: free. RAX: return val.
996    ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH .Lart_quick_alloc_object_initialized_tlab_slow_path
997.Lart_quick_alloc_object_initialized_tlab_slow_path:
998    ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeInitializedTLAB
999END_FUNCTION art_quick_alloc_object_initialized_tlab
1000
1001MACRO0(COMPUTE_ARRAY_SIZE_UNKNOWN)
1002    movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rdi), %ecx        // Load component type.
1003    UNPOISON_HEAP_REF ecx
1004    movl MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET(%rcx), %ecx // Load primitive type.
1005    shrq MACRO_LITERAL(PRIMITIVE_TYPE_SIZE_SHIFT_SHIFT), %rcx        // Get component size shift.
1006    movq %rsi, %r9
1007    salq %cl, %r9                                              // Calculate array count shifted.
1008    // Add array header + alignment rounding.
1009    addq MACRO_LITERAL(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9
1010    // Add 4 extra bytes if we are doing a long array.
1011    addq MACRO_LITERAL(1), %rcx
1012    andq MACRO_LITERAL(4), %rcx
1013#if MIRROR_LONG_ARRAY_DATA_OFFSET != MIRROR_INT_ARRAY_DATA_OFFSET + 4
1014#error Long array data offset must be 4 greater than int array data offset.
1015#endif
1016    addq %rcx, %r9
1017END_MACRO
1018
1019MACRO0(COMPUTE_ARRAY_SIZE_8)
1020    // RDI: mirror::Class* klass, RSI: int32_t component_count
1021    // RDX, RCX, R8, R9: free. RAX: return val.
1022    movq %rsi, %r9
1023    // Add array header + alignment rounding.
1024    addq MACRO_LITERAL(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9
1025END_MACRO
1026
1027MACRO0(COMPUTE_ARRAY_SIZE_16)
1028    // RDI: mirror::Class* klass, RSI: int32_t component_count
1029    // RDX, RCX, R8, R9: free. RAX: return val.
1030    movq %rsi, %r9
1031    salq MACRO_LITERAL(1), %r9
1032    // Add array header + alignment rounding.
1033    addq MACRO_LITERAL(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9
1034END_MACRO
1035
1036MACRO0(COMPUTE_ARRAY_SIZE_32)
1037    // RDI: mirror::Class* klass, RSI: int32_t component_count
1038    // RDX, RCX, R8, R9: free. RAX: return val.
1039    movq %rsi, %r9
1040    salq MACRO_LITERAL(2), %r9
1041    // Add array header + alignment rounding.
1042    addq MACRO_LITERAL(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9
1043END_MACRO
1044
1045MACRO0(COMPUTE_ARRAY_SIZE_64)
1046    // RDI: mirror::Class* klass, RSI: int32_t component_count
1047    // RDX, RCX, R8, R9: free. RAX: return val.
1048    movq %rsi, %r9
1049    salq MACRO_LITERAL(3), %r9
1050    // Add array header + alignment rounding.
1051    addq MACRO_LITERAL(MIRROR_WIDE_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9
1052END_MACRO
1053
1054MACRO3(GENERATE_ALLOC_ARRAY_TLAB, c_entrypoint, cxx_name, size_setup)
1055    DEFINE_FUNCTION VAR(c_entrypoint)
1056    // RDI: mirror::Class* klass, RSI: int32_t component_count
1057    // RDX, RCX, R8, R9: free. RAX: return val.
1058    CALL_MACRO(size_setup)
1059    ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED_WITH_SIZE .Lslow_path\c_entrypoint
1060.Lslow_path\c_entrypoint:
1061    SETUP_SAVE_REFS_ONLY_FRAME                                 // save ref containing registers for GC
1062    // Outgoing argument set up
1063    movq %gs:THREAD_SELF_OFFSET, %rdx                          // pass Thread::Current()
1064    call CALLVAR(cxx_name)                                     // cxx_name(arg0, arg1, Thread*)
1065    RESTORE_SAVE_REFS_ONLY_FRAME                               // restore frame up to return address
1066    RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER           // return or deliver exception
1067    END_FUNCTION VAR(c_entrypoint)
1068END_MACRO
1069
1070
1071GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN
1072GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_8
1073GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_16
1074GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_32
1075GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_region_tlab, artAllocArrayFromCodeResolvedRegionTLAB, COMPUTE_ARRAY_SIZE_64
1076
1077GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_UNKNOWN
1078GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved8_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_8
1079GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved16_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_16
1080GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved32_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_32
1081GENERATE_ALLOC_ARRAY_TLAB art_quick_alloc_array_resolved64_tlab, artAllocArrayFromCodeResolvedTLAB, COMPUTE_ARRAY_SIZE_64
1082
1083// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB).
1084DEFINE_FUNCTION art_quick_alloc_object_resolved_region_tlab
1085    // Fast path region tlab allocation.
1086    // RDI: mirror::Class* klass
1087    // RDX, RSI, RCX, R8, R9: free. RAX: return val.
1088    ASSERT_USE_READ_BARRIER
1089    ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH .Lart_quick_alloc_object_resolved_region_tlab_slow_path
1090.Lart_quick_alloc_object_resolved_region_tlab_slow_path:
1091    ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeResolvedRegionTLAB
1092END_FUNCTION art_quick_alloc_object_resolved_region_tlab
1093
1094// A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB).
1095DEFINE_FUNCTION art_quick_alloc_object_initialized_region_tlab
1096    // Fast path region tlab allocation.
1097    // RDI: mirror::Class* klass
1098    // RDX, RSI, RCX, R8, R9: free. RAX: return val.
1099    ASSERT_USE_READ_BARRIER
1100    // No read barrier since the caller is responsible for that.
1101    ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH .Lart_quick_alloc_object_initialized_region_tlab_slow_path
1102.Lart_quick_alloc_object_initialized_region_tlab_slow_path:
1103    ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeInitializedRegionTLAB
1104END_FUNCTION art_quick_alloc_object_initialized_region_tlab
1105
1106ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_initialize_static_storage, artInitializeStaticStorageFromCode
1107ONE_ARG_SAVE_EVERYTHING_DOWNCALL_FOR_CLINIT art_quick_resolve_type, artResolveTypeFromCode
1108ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_type_and_verify_access, artResolveTypeAndVerifyAccessFromCode
1109ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_handle, artResolveMethodHandleFromCode
1110ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_method_type, artResolveMethodTypeFromCode
1111ONE_ARG_SAVE_EVERYTHING_DOWNCALL art_quick_resolve_string, artResolveStringFromCode
1112
1113TWO_ARG_REF_DOWNCALL art_quick_handle_fill_data, artHandleFillArrayDataFromCode, RETURN_IF_EAX_ZERO
1114
1115    /*
1116     * Entry from managed code that tries to lock the object in a fast path and
1117     * calls `artLockObjectFromCode()` for the difficult cases, may block for GC.
1118     * RDI holds the possibly null object to lock.
1119     */
1120DEFINE_FUNCTION art_quick_lock_object
1121    testq %rdi, %rdi                      // Null check object.
1122    jz   art_quick_lock_object_no_inline
1123    LOCK_OBJECT_FAST_PATH rdi, ecx, art_quick_lock_object_no_inline
1124END_FUNCTION art_quick_lock_object
1125
1126    /*
1127     * Entry from managed code that calls `artLockObjectFromCode()`, may block for GC.
1128     * RDI holds the possibly null object to lock.
1129     */
1130DEFINE_FUNCTION art_quick_lock_object_no_inline
1131    // This is also the slow path for art_quick_lock_object.
1132    SETUP_SAVE_REFS_ONLY_FRAME
1133    movq %gs:THREAD_SELF_OFFSET, %rsi     // pass Thread::Current()
1134    call SYMBOL(artLockObjectFromCode)    // artLockObjectFromCode(object, Thread*)
1135    RESTORE_SAVE_REFS_ONLY_FRAME          // restore frame up to return address
1136    RETURN_IF_EAX_ZERO
1137END_FUNCTION art_quick_lock_object_no_inline
1138
1139    /*
1140     * Entry from managed code that tries to unlock the object in a fast path and calls
1141     * `artUnlockObjectFromCode()` for the difficult cases and delivers exception on failure.
1142     * RDI holds the possibly null object to unlock.
1143     */
1144DEFINE_FUNCTION art_quick_unlock_object
1145    testq %rdi, %rdi                      // Null check object.
1146    jz   art_quick_lock_object_no_inline
1147    UNLOCK_OBJECT_FAST_PATH rdi, ecx, /*saved_rax*/ none, art_quick_unlock_object_no_inline
1148END_FUNCTION art_quick_unlock_object
1149
1150    /*
1151     * Entry from managed code that calls `artUnlockObjectFromCode()`
1152     * and delivers exception on failure.
1153     * RDI holds the possibly null object to unlock.
1154     */
1155DEFINE_FUNCTION art_quick_unlock_object_no_inline
1156    // This is also the slow path for art_quick_unlock_object.
1157    SETUP_SAVE_REFS_ONLY_FRAME
1158    movq %gs:THREAD_SELF_OFFSET, %rsi     // pass Thread::Current()
1159    call SYMBOL(artUnlockObjectFromCode)  // artUnlockObjectFromCode(object, Thread*)
1160    RESTORE_SAVE_REFS_ONLY_FRAME          // restore frame up to return address
1161    RETURN_IF_EAX_ZERO
1162END_FUNCTION art_quick_unlock_object_no_inline
1163
1164DEFINE_FUNCTION art_quick_check_instance_of
1165    // Type check using the bit string passes null as the target class. In that case just throw.
1166    testl %esi, %esi
1167    jz .Lthrow_class_cast_exception_for_bitstring_check
1168
1169    // We could check the super classes here but that is usually already checked in the caller.
1170    PUSH rdi                          // Save args for exc
1171    PUSH rsi
1172    subq LITERAL(8), %rsp             // Alignment padding.
1173    CFI_ADJUST_CFA_OFFSET(8)
1174    SETUP_FP_CALLEE_SAVE_FRAME
1175    call SYMBOL(artInstanceOfFromCode)  // (Object* obj, Class* ref_klass)
1176    testq %rax, %rax
1177    jz .Lthrow_class_cast_exception   // jump forward if not assignable
1178    CFI_REMEMBER_STATE
1179    RESTORE_FP_CALLEE_SAVE_FRAME
1180    addq LITERAL(24), %rsp            // pop arguments
1181    CFI_ADJUST_CFA_OFFSET(-24)
1182    ret
1183    CFI_RESTORE_STATE_AND_DEF_CFA rsp, 64  // Reset unwind info so following code unwinds.
1184
1185.Lthrow_class_cast_exception:
1186    RESTORE_FP_CALLEE_SAVE_FRAME
1187    addq LITERAL(8), %rsp             // pop padding
1188    CFI_ADJUST_CFA_OFFSET(-8)
1189    POP rsi                           // Pop arguments
1190    POP rdi
1191
1192.Lthrow_class_cast_exception_for_bitstring_check:
1193    SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context
1194    mov %gs:THREAD_SELF_OFFSET, %rdx  // pass Thread::Current()
1195    call SYMBOL(artThrowClassCastExceptionForObject)  // (Object* src, Class* dest, Thread*)
1196    UNREACHABLE
1197END_FUNCTION art_quick_check_instance_of
1198
1199
1200// Restore reg's value if reg is not the same as exclude_reg, otherwise just adjust stack.
1201MACRO2(POP_REG_NE, reg, exclude_reg)
1202    .ifc RAW_VAR(reg), RAW_VAR(exclude_reg)
1203      addq MACRO_LITERAL(8), %rsp
1204      CFI_ADJUST_CFA_OFFSET(-8)
1205    .else
1206      POP RAW_VAR(reg)
1207    .endif
1208END_MACRO
1209
1210DEFINE_FUNCTION art_quick_aput_obj
1211    test %edx, %edx              // store of null
1212    jz .Laput_obj_null
1213    movl MIRROR_OBJECT_CLASS_OFFSET(%rdi), %ecx
1214    UNPOISON_HEAP_REF ecx
1215#ifdef USE_READ_BARRIER
1216    cmpl LITERAL(0), %gs:THREAD_IS_GC_MARKING_OFFSET
1217    jnz .Laput_obj_gc_marking
1218#endif  // USE_READ_BARRIER
1219    movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rcx), %ecx
1220    cmpl MIRROR_OBJECT_CLASS_OFFSET(%rdx), %ecx  // Both poisoned if heap poisoning is enabled.
1221    jne .Laput_obj_check_assignability
1222.Laput_obj_store:
1223    POISON_HEAP_REF edx
1224    movl %edx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rdi, %rsi, 4)
1225    movq %gs:THREAD_CARD_TABLE_OFFSET, %rdx
1226    shrl LITERAL(CARD_TABLE_CARD_SHIFT), %edi
1227    movb %dl, (%rdx, %rdi)
1228    ret
1229
1230.Laput_obj_null:
1231    movl %edx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rdi, %rsi, 4)
1232    ret
1233
1234.Laput_obj_check_assignability:
1235    UNPOISON_HEAP_REF ecx         // Unpoison array component type if poisoning is enabled.
1236    PUSH_ARG rdi                  // Save arguments.
1237    PUSH_ARG rsi
1238    PUSH_ARG rdx
1239    movl MIRROR_OBJECT_CLASS_OFFSET(%rdx), %esi  // Pass arg2 = value's class.
1240    UNPOISON_HEAP_REF esi
1241.Laput_obj_check_assignability_call:
1242    movl %ecx, %edi               // Pass arg1 = array's component type.
1243    SETUP_FP_CALLEE_SAVE_FRAME
1244    call SYMBOL(artIsAssignableFromCode)  // (Class* a, Class* b)
1245    RESTORE_FP_CALLEE_SAVE_FRAME  // Resore FP registers.
1246    POP_ARG rdx                   // Restore arguments.
1247    POP_ARG rsi
1248    POP_ARG rdi
1249    testq %rax, %rax              // Check for exception.
1250    jz   .Laput_obj_throw_array_store_exception
1251    POISON_HEAP_REF edx
1252    movl %edx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rdi, %rsi, 4)
1253    movq %gs:THREAD_CARD_TABLE_OFFSET, %rdx
1254    shrl LITERAL(CARD_TABLE_CARD_SHIFT), %edi
1255    movb %dl, (%rdx, %rdi)
1256    ret
1257
1258.Laput_obj_throw_array_store_exception:
1259#ifdef USE_READ_BARRIER
1260    CFI_REMEMBER_STATE
1261#endif  // USE_READ_BARRIER
1262    SETUP_SAVE_ALL_CALLEE_SAVES_FRAME  // Save all registers as basis for long jump context.
1263    // Outgoing argument set up.
1264    movq %rdx, %rsi                         // Pass arg 2 = value.
1265    movq %gs:THREAD_SELF_OFFSET, %rdx       // Pass arg 3 = Thread::Current().
1266                                            // Pass arg 1 = array.
1267    call SYMBOL(artThrowArrayStoreException) // (array, value, Thread*)
1268    UNREACHABLE
1269
1270#ifdef USE_READ_BARRIER
1271    CFI_RESTORE_STATE_AND_DEF_CFA rsp, 4
1272.Laput_obj_gc_marking:
1273    // We need to align stack for `art_quick_read_barrier_mark_regNN`.
1274    INCREASE_FRAME 8                        // Stack alignment.
1275    call SYMBOL(art_quick_read_barrier_mark_reg01)  // Mark ECX
1276    movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rcx), %ecx
1277    UNPOISON_HEAP_REF ecx
1278    call SYMBOL(art_quick_read_barrier_mark_reg01)  // Mark ECX
1279    movl MIRROR_OBJECT_CLASS_OFFSET(%rdx), %eax
1280    UNPOISON_HEAP_REF eax
1281    call SYMBOL(art_quick_read_barrier_mark_reg00)  // Mark EAX
1282    DECREASE_FRAME 8                        // Remove stack alignment.
1283    cmpl %eax, %ecx
1284    je .Laput_obj_store
1285    // Prepare arguments in line with `.Laput_obj_check_assignability_call` and jump there.
1286    PUSH_ARG rdi                  // Save arguments.
1287    PUSH_ARG rsi
1288    PUSH_ARG rdx
1289    movl %eax, %esi               // Pass arg2 - type of the value to be stored.
1290    // The arg1 shall be moved at `.Ldo_assignability_check_call`.
1291    jmp .Laput_obj_check_assignability_call
1292#endif  // USE_READ_BARRIER
1293END_FUNCTION art_quick_aput_obj
1294
1295// TODO: This is quite silly on X86_64 now.
1296DEFINE_FUNCTION art_quick_memcpy
1297    call PLT_SYMBOL(memcpy)       // (void*, const void*, size_t)
1298    ret
1299END_FUNCTION art_quick_memcpy
1300
1301DEFINE_FUNCTION art_quick_test_suspend
1302    SETUP_SAVE_EVERYTHING_FRAME RUNTIME_SAVE_EVERYTHING_FOR_SUSPEND_CHECK_METHOD_OFFSET  // save everything for GC
1303    // Outgoing argument set up
1304    movq %gs:THREAD_SELF_OFFSET, %rdi           // pass Thread::Current()
1305    call SYMBOL(artTestSuspendFromCode)         // (Thread*)
1306    RESTORE_SAVE_EVERYTHING_FRAME               // restore frame up to return address
1307    ret
1308END_FUNCTION art_quick_test_suspend
1309
1310UNIMPLEMENTED art_quick_ldiv
1311UNIMPLEMENTED art_quick_lmod
1312UNIMPLEMENTED art_quick_lmul
1313UNIMPLEMENTED art_quick_lshl
1314UNIMPLEMENTED art_quick_lshr
1315UNIMPLEMENTED art_quick_lushr
1316
1317// Note: Functions `art{Get,Set}<Kind>{Static,Instance}FromCompiledCode` are
1318// defined with a macro in runtime/entrypoints/quick/quick_field_entrypoints.cc.
1319
1320THREE_ARG_REF_DOWNCALL art_quick_set8_instance, artSet8InstanceFromCompiledCode, RETURN_IF_EAX_ZERO
1321THREE_ARG_REF_DOWNCALL art_quick_set16_instance, artSet16InstanceFromCompiledCode, RETURN_IF_EAX_ZERO
1322THREE_ARG_REF_DOWNCALL art_quick_set32_instance, artSet32InstanceFromCompiledCode, RETURN_IF_EAX_ZERO
1323THREE_ARG_REF_DOWNCALL art_quick_set64_instance, artSet64InstanceFromCompiledCode, RETURN_IF_EAX_ZERO
1324THREE_ARG_REF_DOWNCALL art_quick_set_obj_instance, artSetObjInstanceFromCompiledCode, RETURN_IF_EAX_ZERO
1325
1326TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1327TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1328TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1329TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1330TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1331TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1332TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1333
1334TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCompiledCode, RETURN_IF_EAX_ZERO
1335TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCompiledCode, RETURN_IF_EAX_ZERO
1336TWO_ARG_REF_DOWNCALL art_quick_set32_static, artSet32StaticFromCompiledCode, RETURN_IF_EAX_ZERO
1337TWO_ARG_REF_DOWNCALL art_quick_set64_static, artSet64StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1338TWO_ARG_REF_DOWNCALL art_quick_set_obj_static, artSetObjStaticFromCompiledCode, RETURN_IF_EAX_ZERO
1339
1340ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1341ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1342ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1343ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1344ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1345ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1346ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCompiledCode, RETURN_OR_DEOPT_OR_DELIVER_PENDING_EXCEPTION
1347
1348DEFINE_FUNCTION art_quick_proxy_invoke_handler
1349    SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_RDI
1350
1351    movq %gs:THREAD_SELF_OFFSET, %rdx       // Pass Thread::Current().
1352    movq %rsp, %rcx                         // Pass SP.
1353    call SYMBOL(artQuickProxyInvokeHandler) // (proxy method, receiver, Thread*, SP)
1354    RESTORE_SAVE_REFS_AND_ARGS_FRAME
1355    movq %rax, %xmm0                        // Copy return value in case of float returns.
1356    RETURN_OR_DELIVER_PENDING_EXCEPTION
1357END_FUNCTION art_quick_proxy_invoke_handler
1358
1359    /*
1360     * Called to resolve an imt conflict.
1361     * rdi is the conflict ArtMethod.
1362     * rax is a hidden argument that holds the target interface method.
1363     *
1364     * Note that this stub writes to rdi.
1365     */
1366DEFINE_FUNCTION art_quick_imt_conflict_trampoline
1367#if defined(__APPLE__)
1368    int3
1369    int3
1370#else
1371    movq ART_METHOD_JNI_OFFSET_64(%rdi), %rdi  // Load ImtConflictTable
1372.Limt_table_iterate:
1373    cmpq %rax, 0(%rdi)
1374    jne .Limt_table_next_entry
1375    // We successfully hit an entry in the table. Load the target method
1376    // and jump to it.
1377    movq __SIZEOF_POINTER__(%rdi), %rdi
1378    jmp *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi)
1379.Limt_table_next_entry:
1380    // If the entry is null, the interface method is not in the ImtConflictTable.
1381    cmpq LITERAL(0), 0(%rdi)
1382    jz .Lconflict_trampoline
1383    // Iterate over the entries of the ImtConflictTable.
1384    addq LITERAL(2 * __SIZEOF_POINTER__), %rdi
1385    jmp .Limt_table_iterate
1386.Lconflict_trampoline:
1387    // Call the runtime stub to populate the ImtConflictTable and jump to the
1388    // resolved method.
1389    movq %rax, %rdi  // Load interface method
1390    INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline
1391#endif  // __APPLE__
1392END_FUNCTION art_quick_imt_conflict_trampoline
1393
1394DEFINE_FUNCTION art_quick_resolution_trampoline
1395    SETUP_SAVE_REFS_AND_ARGS_FRAME
1396    movq %gs:THREAD_SELF_OFFSET, %rdx
1397    movq %rsp, %rcx
1398    call SYMBOL(artQuickResolutionTrampoline) // (called, receiver, Thread*, SP)
1399    movq %rax, %r10               // Remember returned code pointer in R10.
1400    movq (%rsp), %rdi             // Load called method into RDI.
1401    RESTORE_SAVE_REFS_AND_ARGS_FRAME
1402    testq %r10, %r10              // If code pointer is null goto deliver pending exception.
1403    jz 1f
1404    jmp *%r10                     // Tail call into method.
14051:
1406    DELIVER_PENDING_EXCEPTION
1407END_FUNCTION art_quick_resolution_trampoline
1408
1409/* Generic JNI frame layout:
1410 *
1411 * #-------------------#
1412 * |                   |
1413 * | caller method...  |
1414 * #-------------------#    <--- SP on entry
1415 *
1416 *          |
1417 *          V
1418 *
1419 * #-------------------#
1420 * | caller method...  |
1421 * #-------------------#
1422 * | Return            |
1423 * | R15               |    callee save
1424 * | R14               |    callee save
1425 * | R13               |    callee save
1426 * | R12               |    callee save
1427 * | R9                |    arg5
1428 * | R8                |    arg4
1429 * | RSI/R6            |    arg1
1430 * | RBP/R5            |    callee save
1431 * | RBX/R3            |    callee save
1432 * | RDX/R2            |    arg2
1433 * | RCX/R1            |    arg3
1434 * | XMM7              |    float arg 8
1435 * | XMM6              |    float arg 7
1436 * | XMM5              |    float arg 6
1437 * | XMM4              |    float arg 5
1438 * | XMM3              |    float arg 4
1439 * | XMM2              |    float arg 3
1440 * | XMM1              |    float arg 2
1441 * | XMM0              |    float arg 1
1442 * | RDI/Method*       |  <- sp
1443 * #-------------------#
1444 * | Scratch Alloca    |    5K scratch space
1445 * #---------#---------#
1446 * |         | sp*     |
1447 * | Tramp.  #---------#
1448 * | args    | thread  |
1449 * | Tramp.  #---------#
1450 * |         | method  |
1451 * #-------------------#    <--- SP on artQuickGenericJniTrampoline
1452 *
1453 *           |
1454 *           v              artQuickGenericJniTrampoline
1455 *
1456 * #-------------------#
1457 * | caller method...  |
1458 * #-------------------#
1459 * | Return PC         |
1460 * | Callee-Saves      |
1461 * | padding           | // 8B
1462 * | Method*           |    <--- (1)
1463 * #-------------------#
1464 * | local ref cookie  | // 4B
1465 * | padding           | // 0B or 4B to align handle scope on 8B address
1466 * | handle scope      | // Size depends on number of references; multiple of 4B.
1467 * #-------------------#
1468 * | JNI Stack Args    | // Empty if all args fit into registers.
1469 * #-------------------#    <--- SP on native call (1)
1470 * | Free scratch      |
1471 * #-------------------#
1472 * | SP for JNI call   | // Pointer to (1).
1473 * #-------------------#
1474 * | Hidden arg        | // For @CriticalNative
1475 * #-------------------#
1476 * |                   |
1477 * | Stack for Regs    |    The trampoline assembly will pop these values
1478 * |                   |    into registers for native call
1479 * #-------------------#
1480 */
1481    /*
1482     * Called to do a generic JNI down-call
1483     */
1484DEFINE_FUNCTION art_quick_generic_jni_trampoline
1485    SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_RDI
1486
1487    movq %rsp, %rbp                 // save SP at (old) callee-save frame
1488    CFI_DEF_CFA_REGISTER(rbp)
1489
1490    subq LITERAL(GENERIC_JNI_TRAMPOLINE_RESERVED_AREA), %rsp
1491    // prepare for artQuickGenericJniTrampoline call
1492    // (Thread*, managed_sp, reserved_area)
1493    //    rdi       rsi           rdx   <= C calling convention
1494    //  gs:...      rbp           rsp   <= where they are
1495    movq %gs:THREAD_SELF_OFFSET, %rdi  // Pass Thread::Current().
1496    movq %rbp, %rsi                    // Pass managed frame SP.
1497    movq %rsp, %rdx                    // Pass reserved area.
1498    call SYMBOL(artQuickGenericJniTrampoline)  // (Thread*, sp)
1499
1500    // The C call will have registered the complete save-frame on success.
1501    // The result of the call is:
1502    //     %rax: pointer to native code, 0 on error.
1503    //     The bottom of the reserved area contains values for arg registers,
1504    //     hidden arg register and SP for out args for the call.
1505
1506    // Check for error (class init check or locking for synchronized native method can throw).
1507    test %rax, %rax
1508    jz .Lexception_in_native
1509
1510    // pop from the register-passing alloca region
1511    // what's the right layout?
1512    popq %rdi
1513    popq %rsi
1514    popq %rdx
1515    popq %rcx
1516    popq %r8
1517    popq %r9
1518    // TODO: skip floating point if unused, some flag.
1519    movq 0(%rsp), %xmm0
1520    movq 8(%rsp), %xmm1
1521    movq 16(%rsp), %xmm2
1522    movq 24(%rsp), %xmm3
1523    movq 32(%rsp), %xmm4
1524    movq 40(%rsp), %xmm5
1525    movq 48(%rsp), %xmm6
1526    movq 56(%rsp), %xmm7
1527
1528    // Save call target in scratch register.
1529    movq %rax, %r11
1530
1531    // Load hidden arg (rax) for @CriticalNative.
1532    movq 64(%rsp), %rax
1533    // Load SP for out args, releasing unneeded reserved area.
1534    movq 72(%rsp), %rsp
1535
1536    // native call
1537    call *%r11
1538
1539    // result sign extension is handled in C code
1540    // prepare for artQuickGenericJniEndTrampoline call
1541    // (Thread*,  result, result_f)
1542    //   rdi      rsi   rdx       <= C calling convention
1543    //  gs:...    rax   xmm0      <= where they are
1544    movq %gs:THREAD_SELF_OFFSET, %rdi
1545    movq %rax, %rsi
1546    movq %xmm0, %rdx
1547    call SYMBOL(artQuickGenericJniEndTrampoline)
1548
1549    // Pending exceptions possible.
1550    // TODO: use cmpq, needs direct encoding because of gas bug
1551    movq %gs:THREAD_EXCEPTION_OFFSET, %rcx
1552    test %rcx, %rcx
1553    jnz .Lexception_in_native
1554
1555    // Tear down the alloca.
1556    movq %rbp, %rsp
1557
1558    // store into fpr, for when it's a fpr return...
1559    movq %rax, %xmm0
1560
1561    LOAD_RUNTIME_INSTANCE rcx
1562    cmpb MACRO_LITERAL(0), RUN_EXIT_HOOKS_OFFSET_FROM_RUNTIME_INSTANCE(%rcx)
1563    jne .Lcall_method_exit_hook
1564.Lcall_method_exit_hook_done:
1565
1566    // Tear down the callee-save frame.
1567    CFI_REMEMBER_STATE
1568    CFI_DEF_CFA_REGISTER(rsp)
1569    // Load callee-save FPRs. Skip FP args.
1570    movq 80(%rsp), %xmm12
1571    movq 88(%rsp), %xmm13
1572    movq 96(%rsp), %xmm14
1573    movq 104(%rsp), %xmm15
1574    // Pop method, padding, FP args and two GRP args (rcx, rdx).
1575    DECREASE_FRAME 16 + 12*8 + 2*8
1576    // Load callee-save GPRs and skip args, mixed together to agree with core spills bitmap.
1577    POP rbx  // Callee save.
1578    POP rbp  // Callee save.
1579    DECREASE_FRAME 3*8  // Skip three args (RSI, R8, R9).
1580    POP r12  // Callee save.
1581    POP r13  // Callee save.
1582    POP r14  // Callee save.
1583    POP r15  // Callee save.
1584    ret
1585
1586    // Undo the unwinding information from above since it doesn't apply below.
1587    CFI_RESTORE_STATE_AND_DEF_CFA rbp, 208
1588
1589.Lcall_method_exit_hook:
1590   movq LITERAL(FRAME_SIZE_SAVE_REFS_AND_ARGS), %r8
1591   call art_quick_method_exit_hook
1592   jmp .Lcall_method_exit_hook_done
1593
1594.Lexception_in_native:
1595    pushq %gs:THREAD_TOP_QUICK_FRAME_OFFSET
1596    addq LITERAL(-1), (%rsp)  // Remove the GenericJNI tag.
1597    movq (%rsp), %rsp
1598    call art_deliver_pending_exception
1599END_FUNCTION art_quick_generic_jni_trampoline
1600
1601DEFINE_FUNCTION art_deliver_pending_exception
1602    // This will create a new save-all frame, required by the runtime.
1603    DELIVER_PENDING_EXCEPTION
1604END_FUNCTION art_deliver_pending_exception
1605
1606    /*
1607     * Called to bridge from the quick to interpreter ABI. On entry the arguments match those
1608     * of a quick call:
1609     * RDI = method being called / to bridge to.
1610     * RSI, RDX, RCX, R8, R9 are arguments to that method.
1611     */
1612DEFINE_FUNCTION art_quick_to_interpreter_bridge
1613    SETUP_SAVE_REFS_AND_ARGS_FRAME     // Set up frame and save arguments.
1614    movq %gs:THREAD_SELF_OFFSET, %rsi  // RSI := Thread::Current()
1615    movq %rsp, %rdx                    // RDX := sp
1616    call SYMBOL(artQuickToInterpreterBridge)  // (method, Thread*, SP)
1617    RESTORE_SAVE_REFS_AND_ARGS_FRAME   // TODO: no need to restore arguments in this case.
1618    movq %rax, %xmm0                   // Place return value also into floating point return value.
1619    RETURN_OR_DELIVER_PENDING_EXCEPTION    // return or deliver exception
1620END_FUNCTION art_quick_to_interpreter_bridge
1621
1622    /*
1623     * Called to catch an attempt to invoke an obsolete method.
1624     * RDI = method being called.
1625     */
1626ONE_ARG_RUNTIME_EXCEPTION art_invoke_obsolete_method_stub, artInvokeObsoleteMethod
1627
1628    /*
1629     * Compiled code has requested that we deoptimize into the interpreter. The deoptimization
1630     * will long jump to the interpreter bridge.
1631     */
1632DEFINE_FUNCTION art_quick_deoptimize_from_compiled_code
1633    SETUP_SAVE_EVERYTHING_FRAME
1634                                                // Stack should be aligned now.
1635    movq %gs:THREAD_SELF_OFFSET, %rsi           // Pass Thread.
1636    call SYMBOL(artDeoptimizeFromCompiledCode)  // (DeoptimizationKind, Thread*)
1637    UNREACHABLE
1638END_FUNCTION art_quick_deoptimize_from_compiled_code
1639
1640    /*
1641     * String's compareTo.
1642     *
1643     * On entry:
1644     *    rdi:   this string object (known non-null)
1645     *    rsi:   comp string object (known non-null)
1646     */
1647DEFINE_FUNCTION art_quick_string_compareto
1648    movl MIRROR_STRING_COUNT_OFFSET(%edi), %r8d
1649    movl MIRROR_STRING_COUNT_OFFSET(%esi), %r9d
1650    /* Build pointers to the start of string data */
1651    leal MIRROR_STRING_VALUE_OFFSET(%edi), %edi
1652    leal MIRROR_STRING_VALUE_OFFSET(%esi), %esi
1653#if (STRING_COMPRESSION_FEATURE)
1654    /* Differ cases */
1655    shrl    LITERAL(1), %r8d
1656    jnc     .Lstring_compareto_this_is_compressed
1657    shrl    LITERAL(1), %r9d
1658    jnc     .Lstring_compareto_that_is_compressed
1659    jmp     .Lstring_compareto_both_not_compressed
1660.Lstring_compareto_this_is_compressed:
1661    shrl    LITERAL(1), %r9d
1662    jnc     .Lstring_compareto_both_compressed
1663    /* Comparison this (8-bit) and that (16-bit) */
1664    mov     %r8d, %eax
1665    subl    %r9d, %eax
1666    mov     %r8d, %ecx
1667    cmovg   %r9d, %ecx
1668    /* Going into loop to compare each character */
1669    jecxz   .Lstring_compareto_keep_length1     // check loop counter (if 0 then stop)
1670.Lstring_compareto_loop_comparison_this_compressed:
1671    movzbl  (%edi), %r8d                        // move *(this_cur_char) byte to long
1672    movzwl  (%esi), %r9d                        // move *(that_cur_char) word to long
1673    addl    LITERAL(1), %edi                    // ++this_cur_char (8-bit)
1674    addl    LITERAL(2), %esi                    // ++that_cur_char (16-bit)
1675    subl    %r9d, %r8d
1676    loope   .Lstring_compareto_loop_comparison_this_compressed
1677    cmovne  %r8d, %eax                          // return eax = *(this_cur_char) - *(that_cur_char)
1678.Lstring_compareto_keep_length1:
1679    ret
1680.Lstring_compareto_that_is_compressed:
1681    movl    %r8d, %eax
1682    subl    %r9d, %eax
1683    mov     %r8d, %ecx
1684    cmovg   %r9d, %ecx
1685    /* Comparison this (8-bit) and that (16-bit) */
1686    jecxz   .Lstring_compareto_keep_length2     // check loop counter (if 0, don't compare)
1687.Lstring_compareto_loop_comparison_that_compressed:
1688    movzwl  (%edi), %r8d                        // move *(this_cur_char) word to long
1689    movzbl  (%esi), %r9d                        // move *(that_cur_chat) byte to long
1690    addl    LITERAL(2), %edi                    // ++this_cur_char (16-bit)
1691    addl    LITERAL(1), %esi                    // ++that_cur_char (8-bit)
1692    subl    %r9d, %r8d
1693    loope   .Lstring_compareto_loop_comparison_that_compressed
1694    cmovne  %r8d, %eax                          // return eax = *(this_cur_char) - *(that_cur_char)
1695.Lstring_compareto_keep_length2:
1696    ret
1697.Lstring_compareto_both_compressed:
1698    /* Calculate min length and count diff */
1699    movl    %r8d, %ecx
1700    movl    %r8d, %eax
1701    subl    %r9d, %eax
1702    cmovg   %r9d, %ecx
1703    jecxz   .Lstring_compareto_keep_length3
1704    repe    cmpsb
1705    je      .Lstring_compareto_keep_length3
1706    movzbl  -1(%edi), %eax        // get last compared char from this string (8-bit)
1707    movzbl  -1(%esi), %ecx        // get last compared char from comp string (8-bit)
1708    jmp     .Lstring_compareto_count_difference
1709#endif // STRING_COMPRESSION_FEATURE
1710.Lstring_compareto_both_not_compressed:
1711    /* Calculate min length and count diff */
1712    movl    %r8d, %ecx
1713    movl    %r8d, %eax
1714    subl    %r9d, %eax
1715    cmovg   %r9d, %ecx
1716    /*
1717     * At this point we have:
1718     *   eax: value to return if first part of strings are equal
1719     *   ecx: minimum among the lengths of the two strings
1720     *   esi: pointer to comp string data
1721     *   edi: pointer to this string data
1722     */
1723    jecxz .Lstring_compareto_keep_length3
1724    repe  cmpsw                   // find nonmatching chars in [%esi] and [%edi], up to length %ecx
1725    je    .Lstring_compareto_keep_length3
1726    movzwl  -2(%edi), %eax        // get last compared char from this string (16-bit)
1727    movzwl  -2(%esi), %ecx        // get last compared char from comp string (16-bit)
1728.Lstring_compareto_count_difference:
1729    subl  %ecx, %eax              // return the difference
1730.Lstring_compareto_keep_length3:
1731    ret
1732END_FUNCTION art_quick_string_compareto
1733
1734UNIMPLEMENTED art_quick_memcmp16
1735
1736DEFINE_FUNCTION art_quick_instance_of
1737    SETUP_FP_CALLEE_SAVE_FRAME
1738    subq LITERAL(8), %rsp                      // Alignment padding.
1739    CFI_ADJUST_CFA_OFFSET(8)
1740    call SYMBOL(artInstanceOfFromCode)         // (mirror::Object*, mirror::Class*)
1741    addq LITERAL(8), %rsp
1742    CFI_ADJUST_CFA_OFFSET(-8)
1743    RESTORE_FP_CALLEE_SAVE_FRAME
1744    ret
1745END_FUNCTION art_quick_instance_of
1746
1747DEFINE_FUNCTION art_quick_string_builder_append
1748    SETUP_SAVE_REFS_ONLY_FRAME                // save ref containing registers for GC
1749    // Outgoing argument set up
1750    leaq FRAME_SIZE_SAVE_REFS_ONLY + __SIZEOF_POINTER__(%rsp), %rsi  // pass args
1751    movq %gs:THREAD_SELF_OFFSET, %rdx         // pass Thread::Current()
1752    call artStringBuilderAppend               // (uint32_t, const unit32_t*, Thread*)
1753    RESTORE_SAVE_REFS_ONLY_FRAME              // restore frame up to return address
1754    RETURN_IF_RESULT_IS_NON_ZERO_OR_DEOPT_OR_DELIVER  // return or deopt or deliver exception
1755END_FUNCTION art_quick_string_builder_append
1756
1757// Create a function `name` calling the ReadBarrier::Mark routine,
1758// getting its argument and returning its result through register
1759// `reg`, saving and restoring all caller-save registers.
1760//
1761// The generated function follows a non-standard runtime calling
1762// convention:
1763// - register `reg` (which may be different from RDI) is used to pass the (sole) argument,
1764// - register `reg` (which may be different from RAX) is used to return the result,
1765// - all other registers are callee-save (the values they hold are preserved).
1766MACRO2(READ_BARRIER_MARK_REG, name, reg)
1767    DEFINE_FUNCTION VAR(name)
1768    // Null check so that we can load the lock word.
1769    testq REG_VAR(reg), REG_VAR(reg)
1770    jz .Lret_rb_\name
1771.Lnot_null_\name:
1772    // Check the mark bit, if it is 1 return.
1773    testl LITERAL(LOCK_WORD_MARK_BIT_MASK_SHIFTED), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(reg))
1774    jz .Lslow_rb_\name
1775    ret
1776.Lslow_rb_\name:
1777    PUSH rax
1778    movl MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(reg)), %eax
1779    addl LITERAL(LOCK_WORD_STATE_FORWARDING_ADDRESS_OVERFLOW), %eax
1780    // Jump if the addl caused eax to unsigned overflow. The only case where it overflows is the
1781    // forwarding address one.
1782    // Taken ~25% of the time.
1783    jnae .Lret_forwarding_address\name
1784
1785    // Save all potentially live caller-save core registers.
1786    movq 0(%rsp), %rax
1787    PUSH rcx
1788    PUSH rdx
1789    PUSH rsi
1790    PUSH rdi
1791    PUSH r8
1792    PUSH r9
1793    PUSH r10
1794    PUSH r11
1795    // Create space for caller-save floating-point registers.
1796    subq MACRO_LITERAL(12 * 8), %rsp
1797    CFI_ADJUST_CFA_OFFSET(12 * 8)
1798    // Save all potentially live caller-save floating-point registers.
1799    movq %xmm0, 0(%rsp)
1800    movq %xmm1, 8(%rsp)
1801    movq %xmm2, 16(%rsp)
1802    movq %xmm3, 24(%rsp)
1803    movq %xmm4, 32(%rsp)
1804    movq %xmm5, 40(%rsp)
1805    movq %xmm6, 48(%rsp)
1806    movq %xmm7, 56(%rsp)
1807    movq %xmm8, 64(%rsp)
1808    movq %xmm9, 72(%rsp)
1809    movq %xmm10, 80(%rsp)
1810    movq %xmm11, 88(%rsp)
1811    SETUP_FP_CALLEE_SAVE_FRAME
1812
1813    .ifnc RAW_VAR(reg), rdi
1814      movq REG_VAR(reg), %rdi       // Pass arg1 - obj from `reg`.
1815    .endif
1816    call SYMBOL(artReadBarrierMark) // artReadBarrierMark(obj)
1817    .ifnc RAW_VAR(reg), rax
1818      movq %rax, REG_VAR(reg)       // Return result into `reg`.
1819    .endif
1820
1821    RESTORE_FP_CALLEE_SAVE_FRAME
1822    // Restore floating-point registers.
1823    movq 0(%rsp), %xmm0
1824    movq 8(%rsp), %xmm1
1825    movq 16(%rsp), %xmm2
1826    movq 24(%rsp), %xmm3
1827    movq 32(%rsp), %xmm4
1828    movq 40(%rsp), %xmm5
1829    movq 48(%rsp), %xmm6
1830    movq 56(%rsp), %xmm7
1831    movq 64(%rsp), %xmm8
1832    movq 72(%rsp), %xmm9
1833    movq 80(%rsp), %xmm10
1834    movq 88(%rsp), %xmm11
1835    // Remove floating-point registers.
1836    addq MACRO_LITERAL(12 * 8), %rsp
1837    CFI_ADJUST_CFA_OFFSET(-(12 * 8))
1838    // Restore core regs, except `reg`, as it is used to return the
1839    // result of this function (simply remove it from the stack instead).
1840    POP_REG_NE r11, RAW_VAR(reg)
1841    POP_REG_NE r10, RAW_VAR(reg)
1842    POP_REG_NE r9, RAW_VAR(reg)
1843    POP_REG_NE r8, RAW_VAR(reg)
1844    POP_REG_NE rdi, RAW_VAR(reg)
1845    POP_REG_NE rsi, RAW_VAR(reg)
1846    POP_REG_NE rdx, RAW_VAR(reg)
1847    POP_REG_NE rcx, RAW_VAR(reg)
1848    POP_REG_NE rax, RAW_VAR(reg)
1849.Lret_rb_\name:
1850    ret
1851.Lret_forwarding_address\name:
1852    // The overflow cleared the top bits.
1853    sall LITERAL(LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT), %eax
1854    movq %rax, REG_VAR(reg)
1855    POP_REG_NE rax, RAW_VAR(reg)
1856    ret
1857    END_FUNCTION VAR(name)
1858END_MACRO
1859
1860READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg00, rax
1861READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, rcx
1862READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, rdx
1863READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, rbx
1864// Note: There is no art_quick_read_barrier_mark_reg04, as register 4 (RSP)
1865// cannot be used to pass arguments.
1866READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, rbp
1867READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, rsi
1868READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, rdi
1869READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, r8
1870READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, r9
1871READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, r10
1872READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, r11
1873READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, r12
1874READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, r13
1875READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, r14
1876READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg15, r15
1877
1878DEFINE_FUNCTION art_quick_read_barrier_slow
1879    SETUP_FP_CALLEE_SAVE_FRAME
1880    subq LITERAL(8), %rsp           // Alignment padding.
1881    CFI_ADJUST_CFA_OFFSET(8)
1882    call SYMBOL(artReadBarrierSlow) // artReadBarrierSlow(ref, obj, offset)
1883    addq LITERAL(8), %rsp
1884    CFI_ADJUST_CFA_OFFSET(-8)
1885    RESTORE_FP_CALLEE_SAVE_FRAME
1886    ret
1887END_FUNCTION art_quick_read_barrier_slow
1888
1889DEFINE_FUNCTION art_quick_read_barrier_for_root_slow
1890    SETUP_FP_CALLEE_SAVE_FRAME
1891    subq LITERAL(8), %rsp                  // Alignment padding.
1892    CFI_ADJUST_CFA_OFFSET(8)
1893    call SYMBOL(artReadBarrierForRootSlow) // artReadBarrierForRootSlow(root)
1894    addq LITERAL(8), %rsp
1895    CFI_ADJUST_CFA_OFFSET(-8)
1896    RESTORE_FP_CALLEE_SAVE_FRAME
1897    ret
1898END_FUNCTION art_quick_read_barrier_for_root_slow
1899
1900    /*
1901     * On stack replacement stub.
1902     * On entry:
1903     *   [sp] = return address
1904     *   rdi = stack to copy
1905     *   rsi = size of stack
1906     *   rdx = pc to call
1907     *   rcx = JValue* result
1908     *   r8 = shorty
1909     *   r9 = thread
1910     *
1911     * Note that the native C ABI already aligned the stack to 16-byte.
1912     */
1913DEFINE_FUNCTION art_quick_osr_stub
1914    // Save the non-volatiles.
1915    PUSH rbp                      // Save rbp.
1916    PUSH rcx                      // Save rcx/result*.
1917    PUSH r8                       // Save r8/shorty*.
1918
1919    // Save callee saves.
1920    PUSH rbx
1921    PUSH r12
1922    PUSH r13
1923    PUSH r14
1924    PUSH r15
1925
1926    pushq LITERAL(0)              // Push null for ArtMethod*.
1927    CFI_ADJUST_CFA_OFFSET(8)
1928    movl %esi, %ecx               // rcx := size of stack
1929    movq %rdi, %rsi               // rsi := stack to copy
1930    movq %rsp, %rbp               // Save stack pointer to RBP for CFI use in .Losr_entry.
1931    call .Losr_entry
1932    CFI_REMEMBER_STATE
1933
1934    // Restore stack and callee-saves.
1935    addq LITERAL(8), %rsp
1936    CFI_ADJUST_CFA_OFFSET(-8)
1937    POP r15
1938    POP r14
1939    POP r13
1940    POP r12
1941    POP rbx
1942    POP r8
1943    POP rcx
1944    POP rbp
1945    movq %rax, (%rcx)              // Store the result.
1946    ret
1947.Losr_entry:
1948    CFI_RESTORE_STATE_AND_DEF_CFA rsp, 80
1949    // Since the call has pushed the return address we need to switch the CFA register to RBP.
1950    CFI_DEF_CFA_REGISTER(rbp)
1951
1952    subl LITERAL(8), %ecx         // Given stack size contains pushed frame pointer, substract it.
1953    subq %rcx, %rsp
1954    movq %rsp, %rdi               // rdi := beginning of stack
1955    rep movsb                     // while (rcx--) { *rdi++ = *rsi++ }
1956    jmp *%rdx
1957END_FUNCTION art_quick_osr_stub
1958
1959DEFINE_FUNCTION art_quick_invoke_polymorphic
1960                                                   // On entry: RDI := unused, RSI := receiver
1961    SETUP_SAVE_REFS_AND_ARGS_FRAME                 // save callee saves
1962    movq %rsi, %rdi                                // RDI := receiver
1963    movq %gs:THREAD_SELF_OFFSET, %rsi              // RSI := Thread (self)
1964    movq %rsp, %rdx                                // RDX := pass SP
1965    call SYMBOL(artInvokePolymorphic)              // invoke with (receiver, self, SP)
1966                                                   // save the code pointer
1967    RESTORE_SAVE_REFS_AND_ARGS_FRAME
1968    movq %rax, %xmm0                               // Result is in RAX. Copy to FP result register.
1969    RETURN_OR_DELIVER_PENDING_EXCEPTION
1970END_FUNCTION art_quick_invoke_polymorphic
1971
1972DEFINE_FUNCTION art_quick_invoke_custom
1973    SETUP_SAVE_REFS_AND_ARGS_FRAME                 // save callee saves
1974                                                   // RDI := call_site_index
1975    movq %gs:THREAD_SELF_OFFSET, %rsi              // RSI := Thread::Current()
1976    movq %rsp, %rdx                                // RDX := SP
1977    call SYMBOL(artInvokeCustom)                   // artInvokeCustom(Thread*, SP)
1978    RESTORE_SAVE_REFS_AND_ARGS_FRAME
1979    movq %rax, %xmm0                               // Result is in RAX. Copy to FP result register.
1980    RETURN_OR_DELIVER_PENDING_EXCEPTION
1981END_FUNCTION art_quick_invoke_custom
1982
1983// Wrap ExecuteSwitchImpl in assembly method which specifies DEX PC for unwinding.
1984//  Argument 0: RDI: The context pointer for ExecuteSwitchImpl.
1985//  Argument 1: RSI: Pointer to the templated ExecuteSwitchImpl to call.
1986//  Argument 2: RDX: The value of DEX PC (memory address of the methods bytecode).
1987DEFINE_FUNCTION ExecuteSwitchImplAsm
1988    PUSH rbx                 // Spill RBX
1989    movq %rdx, %rbx          // RBX = DEX PC (callee save register)
1990    CFI_DEFINE_DEX_PC_WITH_OFFSET(0 /* RAX */, 3 /* RBX */, 0)
1991
1992    call *%rsi               // Call the wrapped function
1993
1994    POP rbx                  // Restore RBX
1995    ret
1996END_FUNCTION ExecuteSwitchImplAsm
1997
1998// On entry: edi is the class, r11 is the inline cache. r10 and rax are available.
1999DEFINE_FUNCTION art_quick_update_inline_cache
2000#if (INLINE_CACHE_SIZE != 5)
2001#error "INLINE_CACHE_SIZE not as expected."
2002#endif
2003    // Don't update the cache if we are marking.
2004    cmpl LITERAL(0), %gs:THREAD_IS_GC_MARKING_OFFSET
2005    jnz .Ldone
2006.Lentry1:
2007    movl INLINE_CACHE_CLASSES_OFFSET(%r11), %eax
2008    cmpl %edi, %eax
2009    je .Ldone
2010    cmpl LITERAL(0), %eax
2011    jne .Lentry2
2012    lock cmpxchg %edi, INLINE_CACHE_CLASSES_OFFSET(%r11)
2013    jz .Ldone
2014    jmp .Lentry1
2015.Lentry2:
2016    movl (INLINE_CACHE_CLASSES_OFFSET+4)(%r11), %eax
2017    cmpl %edi, %eax
2018    je .Ldone
2019    cmpl LITERAL(0), %eax
2020    jne .Lentry3
2021    lock cmpxchg %edi, (INLINE_CACHE_CLASSES_OFFSET+4)(%r11)
2022    jz .Ldone
2023    jmp .Lentry2
2024.Lentry3:
2025    movl (INLINE_CACHE_CLASSES_OFFSET+8)(%r11), %eax
2026    cmpl %edi, %eax
2027    je .Ldone
2028    cmpl LITERAL(0), %eax
2029    jne .Lentry4
2030    lock cmpxchg %edi, (INLINE_CACHE_CLASSES_OFFSET+8)(%r11)
2031    jz .Ldone
2032    jmp .Lentry3
2033.Lentry4:
2034    movl (INLINE_CACHE_CLASSES_OFFSET+12)(%r11), %eax
2035    cmpl %edi, %eax
2036    je .Ldone
2037    cmpl LITERAL(0), %eax
2038    jne .Lentry5
2039    lock cmpxchg %edi, (INLINE_CACHE_CLASSES_OFFSET+12)(%r11)
2040    jz .Ldone
2041    jmp .Lentry4
2042.Lentry5:
2043    // Unconditionally store, the cache is megamorphic.
2044    movl %edi, (INLINE_CACHE_CLASSES_OFFSET+16)(%r11)
2045.Ldone:
2046    ret
2047END_FUNCTION art_quick_update_inline_cache
2048
2049// On entry, method is at the bottom of the stack.
2050DEFINE_FUNCTION art_quick_compile_optimized
2051    SETUP_SAVE_EVERYTHING_FRAME
2052    movq FRAME_SIZE_SAVE_EVERYTHING(%rsp), %rdi // pass ArtMethod
2053    movq %gs:THREAD_SELF_OFFSET, %rsi           // pass Thread::Current()
2054    call SYMBOL(artCompileOptimized)            // (ArtMethod*, Thread*)
2055    RESTORE_SAVE_EVERYTHING_FRAME               // restore frame up to return address
2056    ret
2057END_FUNCTION art_quick_compile_optimized
2058
2059// On entry, method is at the bottom of the stack.
2060DEFINE_FUNCTION art_quick_method_entry_hook
2061    SETUP_SAVE_EVERYTHING_FRAME
2062
2063    movq FRAME_SIZE_SAVE_EVERYTHING(%rsp), %rdi // pass ArtMethod
2064    movq %gs:THREAD_SELF_OFFSET, %rsi           // pass Thread::Current()
2065    movq %rsp, %rdx                             // SP
2066
2067    call SYMBOL(artMethodEntryHook)              // (ArtMethod*, Thread*, sp)
2068
2069    RESTORE_SAVE_EVERYTHING_FRAME
2070    ret
2071END_FUNCTION art_quick_method_entry_hook
2072
2073// On entry, method is at the bottom of the stack.
2074DEFINE_FUNCTION art_quick_method_exit_hook
2075    SETUP_SAVE_EVERYTHING_FRAME
2076
2077    // R8 passed from JITed code contains frame_size
2078    leaq 16(%rsp), %rcx                         // floating-point result pointer in kSaveEverything
2079                                                // frame
2080    leaq 144(%rsp), %rdx                        // integer result pointer in kSaveEverything frame
2081    leaq FRAME_SIZE_SAVE_EVERYTHING(%rsp), %rsi // ArtMethod**
2082    movq %gs:THREAD_SELF_OFFSET, %rdi           // Thread::Current
2083    call SYMBOL(artMethodExitHook)              // (Thread*, ArtMethod**, gpr_res*, fpr_res*,
2084                                                //  frame_size)
2085
2086    // Normal return.
2087    RESTORE_SAVE_EVERYTHING_FRAME
2088    ret
2089END_FUNCTION art_quick_method_exit_hook
2090