1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#ifndef ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_S_ 18#define ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_S_ 19 20#include "asm_support_x86_64.h" 21#include "interpreter/cfi_asm_support.h" 22 23// Regular gas(1) & current clang/llvm assembler support named macro parameters. 24#define MACRO0(macro_name) .macro macro_name 25#define MACRO1(macro_name, macro_arg1) .macro macro_name macro_arg1 26#define MACRO2(macro_name, macro_arg1, macro_arg2) .macro macro_name macro_arg1, macro_arg2 27#define MACRO3(macro_name, macro_arg1, macro_arg2, macro_arg3) .macro macro_name macro_arg1, macro_arg2, macro_arg3 28#define MACRO4(macro_name, macro_arg1, macro_arg2, macro_arg3, macro_arg4) .macro macro_name macro_arg1, macro_arg2, macro_arg3, macro_arg4 29#define END_MACRO .endm 30 31#if defined(__clang__) 32 // Clang/llvm does not support .altmacro. However, the clang/llvm preprocessor doesn't 33 // separate the backslash and parameter by a space. Everything just works. 34 #define RAW_VAR(name) \name 35 #define VAR(name) \name 36 #define CALLVAR(name) SYMBOL(\name) 37 #define PLT_VAR(name) \name@PLT 38 #define REG_VAR(name) %\name 39 #define CALL_MACRO(name) \name 40#else 41 // Regular gas(1) uses \argument_name for macro arguments. 42 // We need to turn on alternate macro syntax so we can use & instead or the preprocessor 43 // will screw us by inserting a space between the \ and the name. Even in this mode there's 44 // no special meaning to $, so literals are still just $x. The use of altmacro means % is a 45 // special character meaning care needs to be taken when passing registers as macro 46 // arguments. 47 .altmacro 48 #define RAW_VAR(name) name& 49 #define VAR(name) name& 50 #define CALLVAR(name) SYMBOL(name&) 51 #define PLT_VAR(name) name&@PLT 52 #define REG_VAR(name) %name 53 #define CALL_MACRO(name) name& 54#endif 55 56#define LITERAL(value) $value 57#if defined(__APPLE__) 58 #define MACRO_LITERAL(value) $(value) 59#else 60 #define MACRO_LITERAL(value) $value 61#endif 62 63#if defined(__APPLE__) 64 #define FUNCTION_TYPE(name) 65 #define SIZE(name) 66#else 67 #define FUNCTION_TYPE(name) .type name, @function 68 #define SIZE(name) .size name, .-name 69#endif 70 71 // CFI support. 72#if !defined(__APPLE__) 73 #define CFI_STARTPROC .cfi_startproc 74 #define CFI_ENDPROC .cfi_endproc 75 #define CFI_ADJUST_CFA_OFFSET(size) .cfi_adjust_cfa_offset size 76 #define CFI_DEF_CFA(reg,size) .cfi_def_cfa reg,size 77 #define CFI_DEF_CFA_REGISTER(reg) .cfi_def_cfa_register reg 78 #define CFI_RESTORE(reg) .cfi_restore reg 79 #define CFI_REL_OFFSET(reg,size) .cfi_rel_offset reg,size 80 #define CFI_REGISTER(orig_reg, current_reg) .cfi_register orig_reg, current_reg 81 #define CFI_REMEMBER_STATE .cfi_remember_state 82 // The spec is not clear whether the CFA is part of the saved state and tools 83 // differ in the behaviour, so explicitly set the CFA to avoid any ambiguity. 84 // The restored CFA state should match the CFA state during CFI_REMEMBER_STATE. 85 // `objdump -Wf libart.so | egrep "_cfa|_state"` is useful to audit the opcodes. 86 MACRO2(CFI_RESTORE_STATE_AND_DEF_CFA, reg, off) 87 .cfi_restore_state 88 .cfi_def_cfa \reg,\off 89 END_MACRO 90 #define CFI_RESTORE_STATE .cfi_restore_state 91#else 92 // Mac OS' doesn't like cfi_* directives. 93 #define CFI_STARTPROC 94 #define CFI_ENDPROC 95 #define CFI_ADJUST_CFA_OFFSET(size) 96 #define CFI_DEF_CFA(reg,size) 97 #define CFI_DEF_CFA_REGISTER(reg) 98 #define CFI_RESTORE(reg) 99 #define CFI_REL_OFFSET(reg,size) 100 #define CFI_REGISTER(orig_reg, current_reg) 101 #define CFI_REMEMBER_STATE 102 #define CFI_RESTORE_STATE_AND_DEF_CFA(off) 103 MACRO2(CFI_RESTORE_STATE_AND_DEF_CFA, reg, off) 104 END_MACRO 105 #define CFI_RESTORE_STATE 106#endif 107 108// The register numbers are a bit mixed up for x86-64. 109#define CFI_REG_rax 0 110#define CFI_REG_rcx 2 111#define CFI_REG_rdx 1 112#define CFI_REG_rbx 3 113#define CFI_REG_rsp 7 114#define CFI_REG_rbp 6 115#define CFI_REG_rsi 4 116#define CFI_REG_rdi 5 117#define CFI_REG_r8 8 118#define CFI_REG_r9 9 119#define CFI_REG_r10 10 120#define CFI_REG_r11 11 121#define CFI_REG_r12 12 122#define CFI_REG_r13 13 123#define CFI_REG_r14 14 124#define CFI_REG_r15 15 125#define CFI_REG_rip 16 126 127#define CFI_REG(reg) CFI_REG_##reg 128 129MACRO3(CFI_EXPRESSION_BREG, n, b, offset) 130 .if (-0x40 <= (\offset)) && ((\offset) < 0x40) 131 CFI_EXPRESSION_BREG_1(\n, \b, \offset) 132 .elseif (-0x2000 <= (\offset)) && ((\offset) < 0x2000) 133 CFI_EXPRESSION_BREG_2(\n, \b, \offset) 134 .else 135 .error "Unsupported offset" 136 .endif 137END_MACRO 138 139MACRO3(CFI_DEF_CFA_BREG_PLUS_UCONST, reg, offset, size) 140 .if ((\size) < 0) 141 .error "Size should be positive" 142 .endif 143 .if (((\offset) < -0x40) || ((\offset) >= 0x40)) 144 .error "Unsupported offset" 145 .endif 146 .if ((\size) < 0x80) 147 CFI_DEF_CFA_BREG_PLUS_UCONST_1_1(\reg, \offset, \size) 148 .elseif ((\size) < 0x4000) 149 CFI_DEF_CFA_BREG_PLUS_UCONST_1_2(\reg, \offset, \size) 150 .else 151 .error "Unsupported size" 152 .endif 153END_MACRO 154 155 // Symbols. 156#if !defined(__APPLE__) 157 #define SYMBOL(name) name 158 #define PLT_SYMBOL(name) name ## @PLT 159#else 160 #define SYMBOL(name) _ ## name 161 #define PLT_SYMBOL(name) _ ## name 162#endif 163 164// Directive to hide a function symbol. 165#if defined(__APPLE__) 166 #define ASM_HIDDEN .private_extern 167#else 168 #define ASM_HIDDEN .hidden 169#endif 170 171 /* Cache alignment for function entry */ 172MACRO0(ALIGN_FUNCTION_ENTRY) 173 .balign 16 174END_MACRO 175 176// TODO: we might need to use SYMBOL() here to add the underscore prefix 177// for mac builds. 178MACRO2(DEFINE_FUNCTION_CUSTOM_CFA, c_name, cfa_offset) 179 FUNCTION_TYPE(SYMBOL(\c_name)) 180 ASM_HIDDEN CALLVAR(c_name) 181 .globl CALLVAR(c_name) 182 ALIGN_FUNCTION_ENTRY 183CALLVAR(c_name): 184 CFI_STARTPROC 185 // Ensure we get an appropriate starting CFA. 186 CFI_DEF_CFA(rsp, RAW_VAR(cfa_offset)) 187END_MACRO 188 189MACRO1(DEFINE_FUNCTION, c_name) 190 DEFINE_FUNCTION_CUSTOM_CFA RAW_VAR(c_name), __SIZEOF_POINTER__ 191END_MACRO 192 193MACRO1(END_FUNCTION, c_name) 194 CFI_ENDPROC 195 SIZE(SYMBOL(\c_name)) 196END_MACRO 197 198MACRO1(PUSH, reg) 199 pushq REG_VAR(reg) 200 CFI_ADJUST_CFA_OFFSET(8) 201 CFI_REL_OFFSET(REG_VAR(reg), 0) 202END_MACRO 203 204MACRO1(POP, reg) 205 popq REG_VAR(reg) 206 CFI_ADJUST_CFA_OFFSET(-8) 207 CFI_RESTORE(REG_VAR(reg)) 208END_MACRO 209 210// Arguments do not need .cfi_rel_offset as they are caller-saved and 211// therefore cannot hold caller's variables or unwinding data. 212MACRO1(PUSH_ARG, reg) 213 pushq REG_VAR(reg) 214 CFI_ADJUST_CFA_OFFSET(8) 215END_MACRO 216 217MACRO1(POP_ARG, reg) 218 popq REG_VAR(reg) 219 CFI_ADJUST_CFA_OFFSET(-8) 220END_MACRO 221 222MACRO3(SAVE_REG_BASE, base, reg, offset) 223 movq REG_VAR(reg), RAW_VAR(offset)(REG_VAR(base)) 224 CFI_REL_OFFSET(REG_VAR(reg), RAW_VAR(offset)) 225END_MACRO 226 227MACRO3(RESTORE_REG_BASE, base, reg, offset) 228 movq RAW_VAR(offset)(REG_VAR(base)), REG_VAR(reg) 229 CFI_RESTORE(REG_VAR(reg)) 230END_MACRO 231 232MACRO1(INCREASE_FRAME, frame_adjustment) 233 subq MACRO_LITERAL(RAW_VAR(frame_adjustment)), %rsp 234 CFI_ADJUST_CFA_OFFSET((RAW_VAR(frame_adjustment))) 235END_MACRO 236 237MACRO1(DECREASE_FRAME, frame_adjustment) 238 addq MACRO_LITERAL(RAW_VAR(frame_adjustment)), %rsp 239 CFI_ADJUST_CFA_OFFSET(-(RAW_VAR(frame_adjustment))) 240END_MACRO 241 242MACRO1(UNIMPLEMENTED,name) 243 FUNCTION_TYPE(SYMBOL(\name)) 244 ASM_HIDDEN VAR(name) 245 .globl VAR(name) 246 ALIGN_FUNCTION_ENTRY 247VAR(name): 248 CFI_STARTPROC 249 int3 250 int3 251 CFI_ENDPROC 252 SIZE(SYMBOL(\name)) 253END_MACRO 254 255MACRO0(UNREACHABLE) 256 int3 257END_MACRO 258 259MACRO0(UNTESTED) 260 int3 261END_MACRO 262 263MACRO1(LOAD_RUNTIME_INSTANCE, reg) 264 movq _ZN3art7Runtime9instance_E@GOTPCREL(%rip), REG_VAR(reg) 265 movq (REG_VAR(reg)), REG_VAR(reg) 266END_MACRO 267 268// Macros to poison (negate) the reference for heap poisoning. 269MACRO1(POISON_HEAP_REF, rRef) 270#ifdef USE_HEAP_POISONING 271 negl REG_VAR(rRef) 272#endif // USE_HEAP_POISONING 273END_MACRO 274 275// Macros to unpoison (negate) the reference for heap poisoning. 276MACRO1(UNPOISON_HEAP_REF, rRef) 277#ifdef USE_HEAP_POISONING 278 negl REG_VAR(rRef) 279#endif // USE_HEAP_POISONING 280END_MACRO 281 282 /* 283 * Macro that sets up the callee save frame to conform with 284 * Runtime::CreateCalleeSaveMethod(kSaveRefsOnly) 285 */ 286MACRO0(SETUP_SAVE_REFS_ONLY_FRAME) 287#if defined(__APPLE__) 288 int3 289 int3 290#else 291 // R10 := Runtime::Current() 292 LOAD_RUNTIME_INSTANCE r10 293 // Save callee and GPR args, mixed together to agree with core spills bitmap. 294 PUSH r15 // Callee save. 295 PUSH r14 // Callee save. 296 PUSH r13 // Callee save. 297 PUSH r12 // Callee save. 298 PUSH rbp // Callee save. 299 PUSH rbx // Callee save. 300 // Create space for FPR args, plus space for ArtMethod*. 301 INCREASE_FRAME 8 + 4 * 8 302 // Save FPRs. 303 movq %xmm12, 8(%rsp) 304 movq %xmm13, 16(%rsp) 305 movq %xmm14, 24(%rsp) 306 movq %xmm15, 32(%rsp) 307 // R10 := ArtMethod* for refs only callee save frame method. 308 movq RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET(%r10), %r10 309 // Store ArtMethod* to bottom of stack. 310 movq %r10, 0(%rsp) 311 // Store rsp as the stop quick frame. 312 movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET 313 314 // Ugly compile-time check, but we only have the preprocessor. 315 // Last +8: implicit return address pushed on stack when caller made call. 316#if (FRAME_SIZE_SAVE_REFS_ONLY != 6 * 8 + 4 * 8 + 8 + 8) 317#error "FRAME_SIZE_SAVE_REFS_ONLY(X86_64) size not as expected." 318#endif 319#endif // __APPLE__ 320END_MACRO 321 322MACRO0(RESTORE_SAVE_REFS_ONLY_FRAME) 323 movq 8(%rsp), %xmm12 324 movq 16(%rsp), %xmm13 325 movq 24(%rsp), %xmm14 326 movq 32(%rsp), %xmm15 327 DECREASE_FRAME 8 + 4*8 328 // TODO: optimize by not restoring callee-saves restored by the ABI 329 POP rbx 330 POP rbp 331 POP r12 332 POP r13 333 POP r14 334 POP r15 335END_MACRO 336 337 /* 338 * Macro that sets up the callee save frame to conform with 339 * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs), except for storing the method. 340 */ 341MACRO0(SETUP_SAVE_REFS_AND_ARGS_FRAME_REGISTERS_ONLY) 342 // Save callee and GPR args, mixed together to agree with core spills bitmap. 343 PUSH r15 // Callee save. 344 PUSH r14 // Callee save. 345 PUSH r13 // Callee save. 346 PUSH r12 // Callee save. 347 PUSH_ARG r9 // Quick arg 5. 348 PUSH_ARG r8 // Quick arg 4. 349 PUSH_ARG rsi // Quick arg 1. 350 PUSH rbp // Callee save. 351 PUSH rbx // Callee save. 352 PUSH_ARG rdx // Quick arg 2. 353 PUSH_ARG rcx // Quick arg 3. 354 // Create space for FPR args and create 2 slots for ArtMethod* and padding. 355 INCREASE_FRAME 16 + 12 * 8 356 // Save FPRs. 357 movq %xmm0, 16(%rsp) 358 movq %xmm1, 24(%rsp) 359 movq %xmm2, 32(%rsp) 360 movq %xmm3, 40(%rsp) 361 movq %xmm4, 48(%rsp) 362 movq %xmm5, 56(%rsp) 363 movq %xmm6, 64(%rsp) 364 movq %xmm7, 72(%rsp) 365 movq %xmm12, 80(%rsp) 366 movq %xmm13, 88(%rsp) 367 movq %xmm14, 96(%rsp) 368 movq %xmm15, 104(%rsp) 369 370 // Ugly compile-time check, but we only have the preprocessor. 371 // Last +8: implicit return address pushed on stack when caller made call. 372#if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 11 * 8 + 12 * 8 + 16 + 8) 373#error "FRAME_SIZE_SAVE_REFS_AND_ARGS(X86_64) size not as expected." 374#endif 375END_MACRO 376 377MACRO0(RESTORE_SAVE_REFS_AND_ARGS_FRAME) 378 // Restore FPRs. 379 movq 16(%rsp), %xmm0 380 movq 24(%rsp), %xmm1 381 movq 32(%rsp), %xmm2 382 movq 40(%rsp), %xmm3 383 movq 48(%rsp), %xmm4 384 movq 56(%rsp), %xmm5 385 movq 64(%rsp), %xmm6 386 movq 72(%rsp), %xmm7 387 movq 80(%rsp), %xmm12 388 movq 88(%rsp), %xmm13 389 movq 96(%rsp), %xmm14 390 movq 104(%rsp), %xmm15 391 DECREASE_FRAME 80 + 4 * 8 392 // Restore callee and GPR args, mixed together to agree with core spills bitmap. 393 POP_ARG rcx 394 POP_ARG rdx 395 POP rbx 396 POP rbp 397 POP_ARG rsi 398 POP_ARG r8 399 POP_ARG r9 400 POP r12 401 POP r13 402 POP r14 403 POP r15 404END_MACRO 405 406 /* 407 * Macro that sets up the callee save frame to conform with 408 * Runtime::CreateCalleeSaveMethod(kSaveAllCalleeSaves) 409 */ 410MACRO0(SETUP_SAVE_ALL_CALLEE_SAVES_FRAME) 411#if defined(__APPLE__) 412 int3 413 int3 414#else 415 // R10 := Runtime::Current() 416 LOAD_RUNTIME_INSTANCE r10 417 // Save callee save registers to agree with core spills bitmap. 418 PUSH r15 // Callee save. 419 PUSH r14 // Callee save. 420 PUSH r13 // Callee save. 421 PUSH r12 // Callee save. 422 PUSH rbp // Callee save. 423 PUSH rbx // Callee save. 424 // Create space for FPR args, plus space for ArtMethod*. 425 INCREASE_FRAME 4 * 8 + 8 426 // Save FPRs. 427 movq %xmm12, 8(%rsp) 428 movq %xmm13, 16(%rsp) 429 movq %xmm14, 24(%rsp) 430 movq %xmm15, 32(%rsp) 431 // R10 := ArtMethod* for save all callee save frame method. 432 movq RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET(%r10), %r10 433 // Store ArtMethod* to bottom of stack. 434 movq %r10, 0(%rsp) 435 // Store rsp as the top quick frame. 436 movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET 437 438 // Ugly compile-time check, but we only have the preprocessor. 439 // Last +8: implicit return address pushed on stack when caller made call. 440#if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVES != 6 * 8 + 4 * 8 + 8 + 8) 441#error "FRAME_SIZE_SAVE_ALL_CALLEE_SAVES(X86_64) size not as expected." 442#endif 443#endif // __APPLE__ 444END_MACRO 445 446MACRO0(SETUP_FP_CALLEE_SAVE_FRAME) 447 // Create space for ART FP callee-saved registers 448 INCREASE_FRAME 4 * 8 449 movq %xmm12, 0(%rsp) 450 movq %xmm13, 8(%rsp) 451 movq %xmm14, 16(%rsp) 452 movq %xmm15, 24(%rsp) 453END_MACRO 454 455MACRO0(RESTORE_FP_CALLEE_SAVE_FRAME) 456 // Restore ART FP callee-saved registers 457 movq 0(%rsp), %xmm12 458 movq 8(%rsp), %xmm13 459 movq 16(%rsp), %xmm14 460 movq 24(%rsp), %xmm15 461 DECREASE_FRAME 4 * 8 462END_MACRO 463 464 /* 465 * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending 466 * exception is Thread::Current()->exception_ when the runtime method frame is ready. 467 */ 468MACRO0(DELIVER_PENDING_EXCEPTION_FRAME_READY) 469 // (Thread*) setup 470 movq %gs:THREAD_SELF_OFFSET, %rdi 471 call SYMBOL(artDeliverPendingExceptionFromCode) // artDeliverPendingExceptionFromCode(Thread*) 472 UNREACHABLE 473END_MACRO 474 /* 475 * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending 476 * exception is Thread::Current()->exception_. 477 */ 478MACRO0(DELIVER_PENDING_EXCEPTION) 479 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save callee saves for throw 480 DELIVER_PENDING_EXCEPTION_FRAME_READY 481END_MACRO 482 483MACRO0(RETURN_OR_DELIVER_PENDING_EXCEPTION) 484 cmpq MACRO_LITERAL(0), %gs:THREAD_EXCEPTION_OFFSET // compare exception field with 0 485 jne 1f // if exception != 0 goto 1 486 ret // return 4871: // deliver exception on current thread 488 DELIVER_PENDING_EXCEPTION 489END_MACRO 490 491// Locking is needed for both managed code and JNI stubs. 492MACRO3(LOCK_OBJECT_FAST_PATH, obj, tmp, slow_lock) 4931: 494 movl MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)), %eax // EAX := lock word 495 movl %gs:THREAD_ID_OFFSET, REG_VAR(tmp) // tmp: thread id. 496 xorl %eax, REG_VAR(tmp) // tmp: thread id with count 0 + read barrier bits. 497 testl LITERAL(LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED), %eax // Test the non-gc bits. 498 jnz 2f // Check if unlocked. 499 // Unlocked case - store tmp: original lock word plus thread id, preserved read barrier bits. 500 lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)) 501 jnz 1b // cmpxchg failed retry 502 ret 5032: // EAX: original lock word, tmp: thread id ^ EAX 504 // Check lock word state and thread id together, 505 testl LITERAL(LOCK_WORD_STATE_MASK_SHIFTED | LOCK_WORD_THIN_LOCK_OWNER_MASK_SHIFTED), \ 506 REG_VAR(tmp) 507 jne \slow_lock // Slow path if either of the two high bits are set. 508 // Increment the recursive lock count. 509 leal LOCK_WORD_THIN_LOCK_COUNT_ONE(%eax), REG_VAR(tmp) 510 testl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_MASK_SHIFTED), REG_VAR(tmp) 511 je \slow_lock // If count overflowed, go to slow lock. 512 // Update lockword for recursive lock, cmpxchg necessary for read barrier bits. 513 // EAX: old val, tmp: new val. 514 lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)) 515 jnz 1b // cmpxchg failed retry 516 ret 517END_MACRO 518 519// Unlocking is needed for both managed code and JNI stubs. 520MACRO4(UNLOCK_OBJECT_FAST_PATH, obj, tmp, saved_rax, slow_unlock) 5211: 522 movl MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)), %eax // EAX := lock word 523 movl %gs:THREAD_ID_OFFSET, REG_VAR(tmp) // tmp := thread id 524 xorl %eax, REG_VAR(tmp) // tmp := thread id ^ lock word 525 test LITERAL(LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED), REG_VAR(tmp) 526 jnz 2f // Check if simply locked. 527 // Transition to unlocked. 528#ifndef USE_READ_BARRIER 529 movl REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)) 530#else 531 lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)) 532 jnz 1b // cmpxchg failed retry 533#endif 534 .ifnc \saved_rax, none 535 movq REG_VAR(saved_rax), %rax // Restore RAX. 536 .endif 537 ret 5382: // EAX: original lock word, tmp: lock_word ^ thread id 539 // Check lock word state and thread id together. 540 testl LITERAL(LOCK_WORD_STATE_MASK_SHIFTED | LOCK_WORD_THIN_LOCK_OWNER_MASK_SHIFTED), \ 541 REG_VAR(tmp) 542 jnz \slow_unlock 543 // Update lockword for recursive unlock, cmpxchg necessary for read barrier bits. 544 // tmp: new lock word with decremented count. 545 leal -LOCK_WORD_THIN_LOCK_COUNT_ONE(%eax), REG_VAR(tmp) 546#ifndef USE_READ_BARRIER 547 // EAX: new lock word with decremented count. 548 movl REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)) 549#else 550 lock cmpxchg REG_VAR(tmp), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(obj)) 551 jnz 1b // cmpxchg failed retry 552#endif 553 .ifnc \saved_rax, none 554 movq REG_VAR(saved_rax), %rax // Restore RAX. 555 .endif 556 ret 557END_MACRO 558 559#endif // ART_RUNTIME_ARCH_X86_64_ASM_SUPPORT_X86_64_S_ 560