1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#ifndef ART_RUNTIME_ARCH_ARM64_ASM_SUPPORT_ARM64_S_ 18#define ART_RUNTIME_ARCH_ARM64_ASM_SUPPORT_ARM64_S_ 19 20#include "asm_support_arm64.h" 21#include "interpreter/cfi_asm_support.h" 22 23// Define special registers. 24 25// Register holding Thread::Current(). 26#define xSELF x19 27// Frame Pointer 28#define xFP x29 29// Link Register 30#define xLR x30 31// Define the intraprocedural linkage temporary registers. 32#define xIP0 x16 33#define wIP0 w16 34#define xIP1 x17 35#define wIP1 w17 36 37#ifdef RESERVE_MARKING_REGISTER 38// Marking Register, holding Thread::Current()->GetIsGcMarking(). 39#define wMR w20 40#endif 41 42// Implicit suspend check register. 43#define xSUSPEND x21 44 45.macro LOAD_PC_REL_ADDRESS reg, symbol 46 adr \reg, \symbol 47.endm 48 49.macro CALL_SYMBOL symbol 50 bl \symbol 51.endm 52 53.macro BRANCH_SYMBOL symbol 54 b \symbol 55.endm 56 57.macro BRANCH_SYMBOL_CBZ reg, symbol 58 cbz \reg, \symbol 59.endm 60 61.macro BRANCH_SYMBOL_NE symbol 62 b.ne \symbol 63.endm 64 65.macro BRANCH_SYMBOL_EQ symbol 66 b.eq \symbol 67.endm 68 69.macro CFI_EXPRESSION_BREG n, b, offset 70 .if (-0x40 <= (\offset)) && ((\offset) < 0x40) 71 CFI_EXPRESSION_BREG_1(\n, \b, \offset) 72 .elseif (-0x2000 <= (\offset)) && ((\offset) < 0x2000) 73 CFI_EXPRESSION_BREG_2(\n, \b, \offset) 74 .else 75 .error "Unsupported offset" 76 .endif 77.endm 78 79.macro CFI_DEF_CFA_BREG_PLUS_UCONST reg, offset, size 80 .if ((\size) < 0) 81 .error "Size should be positive" 82 .endif 83 .if (((\offset) < -0x40) || ((\offset) >= 0x40)) 84 .error "Unsupported offset" 85 .endif 86 .if ((\size) < 0x80) 87 CFI_DEF_CFA_BREG_PLUS_UCONST_1_1(\reg, \offset, \size) 88 .elseif ((\size) < 0x4000) 89 CFI_DEF_CFA_BREG_PLUS_UCONST_1_2(\reg, \offset, \size) 90 .else 91 .error "Unsupported size" 92 .endif 93.endm 94 95.macro CFI_REMEMBER_STATE 96 .cfi_remember_state 97.endm 98 99// The spec is not clear whether the CFA is part of the saved state and tools 100// differ in the behaviour, so explicitly set the CFA to avoid any ambiguity. 101// The restored CFA state should match the CFA state during CFI_REMEMBER_STATE. 102.macro CFI_RESTORE_STATE_AND_DEF_CFA reg, offset 103 .cfi_restore_state 104 .cfi_def_cfa \reg, \offset 105.endm 106 107.macro ENTRY_ALIGNED name, alignment 108 .type \name, #function 109 .hidden \name // Hide this as a global symbol, so we do not incur plt calls. 110 .global \name 111 .balign \alignment 112\name: 113 .cfi_startproc 114.endm 115 116.macro ENTRY name 117 ENTRY_ALIGNED \name, 16 118.endm 119 120.macro END name 121 .cfi_endproc 122 .size \name, .-\name 123.endm 124 125.macro UNIMPLEMENTED name 126 ENTRY \name 127 brk 0 128 END \name 129.endm 130 131// Macro to poison (negate) the reference for heap poisoning. 132.macro POISON_HEAP_REF rRef 133#ifdef USE_HEAP_POISONING 134 neg \rRef, \rRef 135#endif // USE_HEAP_POISONING 136.endm 137 138// Macro to unpoison (negate) the reference for heap poisoning. 139.macro UNPOISON_HEAP_REF rRef 140#ifdef USE_HEAP_POISONING 141 neg \rRef, \rRef 142#endif // USE_HEAP_POISONING 143.endm 144 145.macro INCREASE_FRAME frame_adjustment 146 sub sp, sp, #(\frame_adjustment) 147 .cfi_adjust_cfa_offset (\frame_adjustment) 148.endm 149 150.macro DECREASE_FRAME frame_adjustment 151 add sp, sp, #(\frame_adjustment) 152 .cfi_adjust_cfa_offset -(\frame_adjustment) 153.endm 154 155.macro SAVE_REG reg, offset 156 str \reg, [sp, #(\offset)] 157 .cfi_rel_offset \reg, (\offset) 158.endm 159 160.macro RESTORE_REG_BASE base, reg, offset 161 ldr \reg, [\base, #(\offset)] 162 .cfi_restore \reg 163.endm 164 165.macro RESTORE_REG reg, offset 166 RESTORE_REG_BASE sp, \reg, \offset 167.endm 168 169.macro SAVE_TWO_REGS_BASE base, reg1, reg2, offset 170 stp \reg1, \reg2, [\base, #(\offset)] 171 .cfi_rel_offset \reg1, (\offset) 172 .cfi_rel_offset \reg2, (\offset) + 8 173.endm 174 175.macro SAVE_TWO_REGS reg1, reg2, offset 176 SAVE_TWO_REGS_BASE sp, \reg1, \reg2, \offset 177.endm 178 179.macro SAVE_TWO_REGS_INCREASE_FRAME reg1, reg2, frame_adjustment 180 stp \reg1, \reg2, [sp, #-(\frame_adjustment)]! 181 .cfi_adjust_cfa_offset (\frame_adjustment) 182 .cfi_rel_offset \reg1, 0 183 .cfi_rel_offset \reg2, 8 184.endm 185 186.macro RESTORE_TWO_REGS_BASE base, reg1, reg2, offset 187 ldp \reg1, \reg2, [\base, #(\offset)] 188 .cfi_restore \reg1 189 .cfi_restore \reg2 190.endm 191 192.macro RESTORE_TWO_REGS reg1, reg2, offset 193 RESTORE_TWO_REGS_BASE sp, \reg1, \reg2, \offset 194.endm 195 196.macro RESTORE_TWO_REGS_DECREASE_FRAME reg1, reg2, frame_adjustment 197 ldp \reg1, \reg2, [sp], #(\frame_adjustment) 198 .cfi_restore \reg1 199 .cfi_restore \reg2 200 .cfi_adjust_cfa_offset -(\frame_adjustment) 201.endm 202 203#define ALL_ARGS_SIZE (/*x0-x7*/ 8 * 8 + /*d0-d7*/ 8 * 8) 204 205.macro SAVE_ALL_ARGS_INCREASE_FRAME extra_space 206 // Save register args x0-x7, d0-d7 and return address. 207 stp x0, x1, [sp, #-(ALL_ARGS_SIZE + \extra_space)]! 208 .cfi_adjust_cfa_offset (ALL_ARGS_SIZE + \extra_space) 209 stp x2, x3, [sp, #16] 210 stp x4, x5, [sp, #32] 211 stp x6, x7, [sp, #48] 212 stp d0, d1, [sp, #64] 213 stp d2, d3, [sp, #80] 214 stp d4, d5, [sp, #96] 215 stp d6, d7, [sp, #112] 216.endm 217 218.macro RESTORE_ALL_ARGS_DECREASE_FRAME extra_space 219 ldp x2, x3, [sp, #16] 220 ldp x4, x5, [sp, #32] 221 ldp x6, x7, [sp, #48] 222 ldp d0, d1, [sp, #64] 223 ldp d2, d3, [sp, #80] 224 ldp d4, d5, [sp, #96] 225 ldp d6, d7, [sp, #112] 226 ldp x0, x1, [sp], #(ALL_ARGS_SIZE + \extra_space) 227 .cfi_adjust_cfa_offset -(ALL_ARGS_SIZE + \extra_space) 228.endm 229 230.macro LOAD_RUNTIME_INSTANCE reg 231#if __has_feature(hwaddress_sanitizer) 232 adrp \reg, :pg_hi21_nc:_ZN3art7Runtime9instance_E 233#else 234 adrp \reg, _ZN3art7Runtime9instance_E 235#endif 236 ldr \reg, [\reg, #:lo12:_ZN3art7Runtime9instance_E] 237.endm 238 239// Macro to refresh the Marking Register (W20). 240// 241// This macro must be called at the end of functions implementing 242// entrypoints that possibly (directly or indirectly) perform a 243// suspend check (before they return). 244.macro REFRESH_MARKING_REGISTER 245#ifdef RESERVE_MARKING_REGISTER 246 ldr wMR, [xSELF, #THREAD_IS_GC_MARKING_OFFSET] 247#endif 248.endm 249 250// Macro to refresh the suspend check register. 251// 252// We do not refresh `xSUSPEND` after every transition to Runnable, so there is 253// a chance that an implicit suspend check loads null to xSUSPEND but before 254// causing a SIGSEGV at the next implicit suspend check we make a runtime call 255// that performs the suspend check explicitly. This can cause a spurious fault 256// without a pending suspend check request but it should be rare and the fault 257// overhead was already expected when we triggered the suspend check, we just 258// pay the price later than expected. 259.macro REFRESH_SUSPEND_CHECK_REGISTER 260 ldr xSUSPEND, [xSELF, #THREAD_SUSPEND_TRIGGER_OFFSET] 261.endm 262 263 /* 264 * Macro that sets up the callee save frame to conform with 265 * Runtime::CreateCalleeSaveMethod(kSaveRefsOnly). 266 */ 267.macro SETUP_SAVE_REFS_ONLY_FRAME 268 // art::Runtime* xIP0 = art::Runtime::instance_; 269 // Our registers aren't intermixed - just spill in order. 270 LOAD_RUNTIME_INSTANCE xIP0 271 272 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveRefOnly]; 273 ldr xIP0, [xIP0, RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET] 274 275 INCREASE_FRAME 96 276 277 // Ugly compile-time check, but we only have the preprocessor. 278#if (FRAME_SIZE_SAVE_REFS_ONLY != 96) 279#error "FRAME_SIZE_SAVE_REFS_ONLY(ARM64) size not as expected." 280#endif 281 282 // GP callee-saves. 283 // x20 paired with ArtMethod* - see below. 284 SAVE_TWO_REGS x21, x22, 16 285 SAVE_TWO_REGS x23, x24, 32 286 SAVE_TWO_REGS x25, x26, 48 287 SAVE_TWO_REGS x27, x28, 64 288 SAVE_TWO_REGS x29, xLR, 80 289 290 // Store ArtMethod* Runtime::callee_save_methods_[kSaveRefsOnly]. 291 // Note: We could avoid saving X20 in the case of Baker read 292 // barriers, as it is overwritten by REFRESH_MARKING_REGISTER 293 // later; but it's not worth handling this special case. 294 stp xIP0, x20, [sp] 295 .cfi_rel_offset x20, 8 296 297 // Place sp in Thread::Current()->top_quick_frame. 298 mov xIP0, sp 299 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 300.endm 301 302// TODO: Probably no need to restore registers preserved by aapcs64. 303.macro RESTORE_SAVE_REFS_ONLY_FRAME 304 // Callee-saves. 305 // Note: Likewise, we could avoid restoring X20 in the case of Baker 306 // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER 307 // later; but it's not worth handling this special case. 308 RESTORE_REG x20, 8 309 RESTORE_TWO_REGS x21, x22, 16 310 RESTORE_TWO_REGS x23, x24, 32 311 RESTORE_TWO_REGS x25, x26, 48 312 RESTORE_TWO_REGS x27, x28, 64 313 RESTORE_TWO_REGS x29, xLR, 80 314 315 DECREASE_FRAME 96 316.endm 317 318.macro SETUP_SAVE_REFS_AND_ARGS_FRAME_INTERNAL base 319 // Ugly compile-time check, but we only have the preprocessor. 320#if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 224) 321#error "FRAME_SIZE_SAVE_REFS_AND_ARGS(ARM64) size not as expected." 322#endif 323 324 // Stack alignment filler [\base, #8]. 325 // FP args. 326 stp d0, d1, [\base, #16] 327 stp d2, d3, [\base, #32] 328 stp d4, d5, [\base, #48] 329 stp d6, d7, [\base, #64] 330 331 // Core args. 332 stp x1, x2, [\base, #80] 333 stp x3, x4, [\base, #96] 334 stp x5, x6, [\base, #112] 335 336 // x7, Callee-saves. 337 // Note: We could avoid saving X20 in the case of Baker read 338 // barriers, as it is overwritten by REFRESH_MARKING_REGISTER 339 // later; but it's not worth handling this special case. 340 stp x7, x20, [\base, #128] 341 .cfi_rel_offset x20, 136 342 SAVE_TWO_REGS_BASE \base, x21, x22, 144 343 SAVE_TWO_REGS_BASE \base, x23, x24, 160 344 SAVE_TWO_REGS_BASE \base, x25, x26, 176 345 SAVE_TWO_REGS_BASE \base, x27, x28, 192 346 347 // x29(callee-save) and LR. 348 SAVE_TWO_REGS_BASE \base, x29, xLR, 208 349.endm 350 351// TODO: Probably no need to restore registers preserved by aapcs64. (That would require 352// auditing all users to make sure they restore aapcs64 callee-save registers they clobber.) 353.macro RESTORE_SAVE_REFS_AND_ARGS_FRAME_INTERNAL base 354 // FP args. 355 ldp d0, d1, [\base, #16] 356 ldp d2, d3, [\base, #32] 357 ldp d4, d5, [\base, #48] 358 ldp d6, d7, [\base, #64] 359 360 // Core args. 361 ldp x1, x2, [\base, #80] 362 ldp x3, x4, [\base, #96] 363 ldp x5, x6, [\base, #112] 364 365 // x7, callee-saves and LR. 366 // Note: Likewise, we could avoid restoring X20 in the case of Baker 367 // read barriers, as it is overwritten by REFRESH_MARKING_REGISTER 368 // later; but it's not worth handling this special case. 369 ldp x7, x20, [\base, #128] 370 .cfi_restore x20 371 RESTORE_TWO_REGS_BASE \base, x21, x22, 144 372 RESTORE_TWO_REGS_BASE \base, x23, x24, 160 373 RESTORE_TWO_REGS_BASE \base, x25, x26, 176 374 RESTORE_TWO_REGS_BASE \base, x27, x28, 192 375 RESTORE_TWO_REGS_BASE \base, x29, xLR, 208 376.endm 377 378.macro RESTORE_SAVE_REFS_AND_ARGS_FRAME 379 RESTORE_SAVE_REFS_AND_ARGS_FRAME_INTERNAL sp 380 DECREASE_FRAME FRAME_SIZE_SAVE_REFS_AND_ARGS 381.endm 382 383.macro SAVE_ALL_CALLEE_SAVES offset 384 // FP callee-saves. 385 stp d8, d9, [sp, #(0 + \offset)] 386 stp d10, d11, [sp, #(16 + \offset)] 387 stp d12, d13, [sp, #(32 + \offset)] 388 stp d14, d15, [sp, #(48 + \offset)] 389 390 // GP callee-saves 391 SAVE_TWO_REGS x19, x20, (64 + \offset) 392 SAVE_TWO_REGS x21, x22, (80 + \offset) 393 SAVE_TWO_REGS x23, x24, (96 + \offset) 394 SAVE_TWO_REGS x25, x26, (112 + \offset) 395 SAVE_TWO_REGS x27, x28, (128 + \offset) 396 SAVE_TWO_REGS x29, xLR, (144 + \offset) 397.endm 398 399 /* 400 * Macro that sets up the callee save frame to conform with 401 * Runtime::CreateCalleeSaveMethod(kSaveAllCalleeSaves) 402 */ 403.macro SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 404 // art::Runtime* xIP0 = art::Runtime::instance_; 405 // Our registers aren't intermixed - just spill in order. 406 LOAD_RUNTIME_INSTANCE xIP0 407 408 // ArtMethod* xIP0 = Runtime::instance_->callee_save_methods_[kSaveAllCalleeSaves]; 409 ldr xIP0, [xIP0, RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET] 410 411 INCREASE_FRAME 176 412 413 // Ugly compile-time check, but we only have the preprocessor. 414#if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVES != 176) 415#error "FRAME_SIZE_SAVE_ALL_CALLEE_SAVES(ARM64) size not as expected." 416#endif 417 418 // Stack alignment filler [sp, #8]. 419 SAVE_ALL_CALLEE_SAVES 16 420 421 // Store ArtMethod* Runtime::callee_save_methods_[kSaveAllCalleeSaves]. 422 str xIP0, [sp] 423 // Place sp in Thread::Current()->top_quick_frame. 424 mov xIP0, sp 425 str xIP0, [xSELF, # THREAD_TOP_QUICK_FRAME_OFFSET] 426.endm 427 428 /* 429 * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending 430 * exception is Thread::Current()->exception_ when the runtime method frame is ready. 431 */ 432.macro DELIVER_PENDING_EXCEPTION_FRAME_READY 433 mov x0, xSELF 434 435 // Point of no return. 436 CALL_SYMBOL artDeliverPendingExceptionFromCode // artDeliverPendingExceptionFromCode(Thread*) 437 CALL_SYMBOL art_quick_do_long_jump // (Context*) 438 brk 0 // Unreached 439.endm 440 441 /* 442 * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending 443 * exception is Thread::Current()->exception_. 444 */ 445.macro DELIVER_PENDING_EXCEPTION 446 SETUP_SAVE_ALL_CALLEE_SAVES_FRAME 447 DELIVER_PENDING_EXCEPTION_FRAME_READY 448.endm 449 450.macro RETURN_OR_DELIVER_PENDING_EXCEPTION_REG reg 451 ldr \reg, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 452 cbnz \reg, 1f 453 ret 4541: 455 DELIVER_PENDING_EXCEPTION 456.endm 457 458.macro RETURN_OR_DELIVER_PENDING_EXCEPTION 459 RETURN_OR_DELIVER_PENDING_EXCEPTION_REG xIP0 460.endm 461 462// Locking is needed for both managed code and JNI stubs. 463.macro LOCK_OBJECT_FAST_PATH obj, slow_lock, can_be_null 464 // Use scratch registers x8-x11 as temporaries. 465 ldr w9, [xSELF, #THREAD_ID_OFFSET] 466 .if \can_be_null 467 BRANCH_SYMBOL_CBZ \obj, \slow_lock 468 .endif 469 // Exclusive load/store has no immediate anymore. 470 add x8, \obj, #MIRROR_OBJECT_LOCK_WORD_OFFSET 4711: 472 ldaxr w10, [x8] // Acquire needed only in most common case. 473 eor w11, w10, w9 // Prepare the value to store if unlocked 474 // (thread id, count of 0 and preserved read barrier bits), 475 // or prepare to compare thread id for recursive lock check 476 // (lock_word.ThreadId() ^ self->ThreadId()). 477 tst w10, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // Test the non-gc bits. 478 b.ne 2f // Check if unlocked. 479 // Unlocked case - store w11: original lock word plus thread id, preserved read barrier bits. 480 stxr w10, w11, [x8] 481 cbnz w10, 1b // If the store failed, retry. 482 ret 4832: // w10: original lock word, w9: thread id, w11: w10 ^ w9 484 // Check lock word state and thread id together, 485 tst w11, #(LOCK_WORD_STATE_MASK_SHIFTED | LOCK_WORD_THIN_LOCK_OWNER_MASK_SHIFTED) 486 BRANCH_SYMBOL_NE \slow_lock 487 add w11, w10, #LOCK_WORD_THIN_LOCK_COUNT_ONE // Increment the recursive lock count. 488 tst w11, #LOCK_WORD_THIN_LOCK_COUNT_MASK_SHIFTED // Test the new thin lock count. 489 BRANCH_SYMBOL_EQ \slow_lock // Zero as the new count indicates overflow, go 490 // slow path. 491 stxr w10, w11, [x8] 492 cbnz w10, 1b // If the store failed, retry. 493 ret 494.endm 495 496// Unlocking is needed for both managed code and JNI stubs. 497.macro UNLOCK_OBJECT_FAST_PATH obj, slow_unlock, can_be_null 498 // Use scratch registers x8-x11 as temporaries. 499 ldr w9, [xSELF, #THREAD_ID_OFFSET] 500 .if \can_be_null 501 BRANCH_SYMBOL_CBZ \obj, \slow_unlock 502 .endif 503 // Exclusive load/store has no immediate anymore. 504 add x8, \obj, #MIRROR_OBJECT_LOCK_WORD_OFFSET 5051: 506#ifndef USE_READ_BARRIER 507 ldr w10, [x8] 508#else 509 ldxr w10, [x8] // Need to use atomic instructions for read barrier. 510#endif 511 eor w11, w10, w9 // Prepare the value to store if simply locked 512 // (mostly 0s, and preserved read barrier bits), 513 // or prepare to compare thread id for recursive lock check 514 // (lock_word.ThreadId() ^ self->ThreadId()). 515 tst w11, #LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED // Test the non-gc bits. 516 b.ne 2f // Locked recursively or by other thread? 517 // Transition to unlocked. 518#ifndef USE_READ_BARRIER 519 stlr w11, [x8] 520#else 521 stlxr w10, w11, [x8] // Need to use atomic instructions for read barrier. 522 cbnz w10, 1b // If the store failed, retry. 523#endif 524 ret 5252: 526 // Check lock word state and thread id together. 527 tst w11, #(LOCK_WORD_STATE_MASK_SHIFTED | LOCK_WORD_THIN_LOCK_OWNER_MASK_SHIFTED) 528 BRANCH_SYMBOL_NE \slow_unlock 529 sub w11, w10, #LOCK_WORD_THIN_LOCK_COUNT_ONE // decrement count 530#ifndef USE_READ_BARRIER 531 str w11, [x8] 532#else 533 stxr w10, w11, [x8] // Need to use atomic instructions for read barrier. 534 cbnz w10, 1b // If the store failed, retry. 535#endif 536 ret 537.endm 538 539#endif // ART_RUNTIME_ARCH_ARM64_ASM_SUPPORT_ARM64_S_ 540