1%def header(): 2/* 3 * Copyright (C) 2020 The Android Open Source Project 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18/* 19 * This is a #include, not a %include, because we want the C pre-processor 20 * to expand the macros into assembler assignment statements. 21 */ 22#include "asm_support.h" 23#include "arch/arm64/asm_support_arm64.S" 24 25/** 26 * ARM64 Runtime register usage conventions. 27 * 28 * r0 : w0 is 32-bit return register and x0 is 64-bit. 29 * r0-r7 : Argument registers. 30 * r8-r15 : Caller save registers (used as temporary registers). 31 * r16-r17: Also known as ip0-ip1, respectively. Used as scratch registers by 32 * the linker, by the trampolines and other stubs (the compiler uses 33 * these as temporary registers). 34 * r18 : Reserved for platform (SCS, shadow call stack) 35 * r19 : Pointer to thread-local storage. 36 * r20-r29: Callee save registers. 37 * r30 : (lr) is reserved (the link register). 38 * rsp : (sp) is reserved (the stack pointer). 39 * rzr : (zr) is reserved (the zero register). 40 * 41 * Floating-point registers 42 * v0-v31 43 * 44 * v0 : s0 is return register for singles (32-bit) and d0 for doubles (64-bit). 45 * This is analogous to the C/C++ (hard-float) calling convention. 46 * v0-v7 : Floating-point argument registers in both Dalvik and C/C++ conventions. 47 * Also used as temporary and codegen scratch registers. 48 * 49 * v0-v7 and v16-v31 : Caller save registers (used as temporary registers). 50 * v8-v15 : bottom 64-bits preserved across C calls (d8-d15 are preserved). 51 * 52 * v16-v31: Used as codegen temp/scratch. 53 * v8-v15 : Can be used for promotion. 54 * 55 * Must maintain 16-byte stack alignment. 56 * 57 * Nterp notes: 58 * 59 * The following registers have fixed assignments: 60 * 61 * reg nick purpose 62 * x19 xSELF self (Thread) pointer 63 * x20 wMR marking register 64 * x21 xSUSPEND suspend check register 65 * x29 xFP interpreted frame pointer, used for accessing locals and args 66 * x22 xPC interpreted program counter, used for fetching instructions 67 * x23 xINST first 16-bit code unit of current instruction 68 * x24 xIBASE interpreted instruction base pointer, used for computed goto 69 * x25 xREFS base of object references of dex registers. 70 * x16 ip scratch reg 71 * x17 ip2 scratch reg (used by macros) 72 * 73 * Macros are provided for common operations. They MUST NOT alter unspecified registers or 74 * condition codes. 75*/ 76 77/* single-purpose registers, given names for clarity */ 78#define CFI_DEX 22 // DWARF register number of the register holding dex-pc (xPC). 79#define CFI_TMP 0 // DWARF register number of the first argument register (r0). 80#define xPC x22 81#define xINST x23 82#define wINST w23 83#define xIBASE x24 84#define xREFS x25 85#define CFI_REFS 25 86#define ip x16 87#define ip2 x17 88#define wip w16 89#define wip2 w17 90 91// To avoid putting ifdefs arond the use of wMR, make sure it's defined. 92// IsNterpSupported returns false for configurations that don't have wMR (typically CMS). 93#ifndef wMR 94#define wMR w20 95#endif 96 97// Temporary registers while setting up a frame. 98#define xNEW_FP x26 99#define xNEW_REFS x27 100#define CFI_NEW_REFS 27 101 102// +8 for the ArtMethod of the caller. 103#define OFFSET_TO_FIRST_ARGUMENT_IN_STACK (CALLEE_SAVES_SIZE + 8) 104 105/* 106 * Fetch the next instruction from xPC into wINST. Does not advance xPC. 107 */ 108.macro FETCH_INST 109 ldrh wINST, [xPC] 110.endm 111 112/* 113 * Fetch the next instruction from the specified offset. Advances xPC 114 * to point to the next instruction. "count" is in 16-bit code units. 115 * 116 * Because of the limited size of immediate constants on ARM, this is only 117 * suitable for small forward movements (i.e. don't try to implement "goto" 118 * with this). 119 * 120 * This must come AFTER anything that can throw an exception, or the 121 * exception catch may miss. (This also implies that it must come after 122 * EXPORT_PC.) 123 */ 124.macro FETCH_ADVANCE_INST count 125 ldrh wINST, [xPC, #((\count)*2)]! 126.endm 127 128/* 129 * Similar to FETCH_ADVANCE_INST, but does not update xPC. Used to load 130 * xINST ahead of possible exception point. Be sure to manually advance xPC 131 * later. 132 */ 133.macro PREFETCH_INST count 134 ldrh wINST, [xPC, #((\count)*2)] 135.endm 136 137/* Advance xPC by some number of code units. */ 138.macro ADVANCE count 139 add xPC, xPC, #((\count)*2) 140.endm 141 142/* 143 * Fetch a half-word code unit from an offset past the current PC. The 144 * "count" value is in 16-bit code units. Does not advance xPC. 145 * 146 * The "_S" variant works the same but treats the value as signed. 147 */ 148.macro FETCH reg, count 149 ldrh \reg, [xPC, #((\count)*2)] 150.endm 151 152.macro FETCH_S reg, count 153 ldrsh \reg, [xPC, #((\count)*2)] 154.endm 155 156/* 157 * Fetch one byte from an offset past the current PC. Pass in the same 158 * "count" as you would for FETCH, and an additional 0/1 indicating which 159 * byte of the halfword you want (lo/hi). 160 */ 161.macro FETCH_B reg, count, byte 162 ldrb \reg, [xPC, #((\count)*2+(\byte))] 163.endm 164 165/* 166 * Put the instruction's opcode field into the specified register. 167 */ 168.macro GET_INST_OPCODE reg 169 and \reg, xINST, #255 170.endm 171 172/* 173 * Begin executing the opcode in _reg. Clobbers reg 174 */ 175 176.macro GOTO_OPCODE reg 177 add \reg, xIBASE, \reg, lsl #${handler_size_bits} 178 br \reg 179.endm 180 181/* 182 * Get/set the 32-bit value from a Dalvik register. 183 */ 184.macro GET_VREG reg, vreg 185 ldr \reg, [xFP, \vreg, uxtw #2] 186.endm 187.macro GET_VREG_OBJECT reg, vreg 188 ldr \reg, [xREFS, \vreg, uxtw #2] 189.endm 190.macro SET_VREG reg, vreg 191 str \reg, [xFP, \vreg, uxtw #2] 192 str wzr, [xREFS, \vreg, uxtw #2] 193.endm 194.macro SET_VREG_OBJECT reg, vreg 195 str \reg, [xFP, \vreg, uxtw #2] 196 str \reg, [xREFS, \vreg, uxtw #2] 197.endm 198.macro SET_VREG_FLOAT reg, vreg 199 str \reg, [xFP, \vreg, uxtw #2] 200 str wzr, [xREFS, \vreg, uxtw #2] 201.endm 202 203/* 204 * Get/set the 64-bit value from a Dalvik register. 205 */ 206.macro GET_VREG_WIDE reg, vreg 207 add ip2, xFP, \vreg, uxtw #2 208 ldr \reg, [ip2] 209.endm 210.macro SET_VREG_WIDE reg, vreg 211 add ip2, xFP, \vreg, uxtw #2 212 str \reg, [ip2] 213 add ip2, xREFS, \vreg, uxtw #2 214 str xzr, [ip2] 215.endm 216.macro GET_VREG_DOUBLE reg, vreg 217 add ip2, xFP, \vreg, uxtw #2 218 ldr \reg, [ip2] 219.endm 220.macro SET_VREG_DOUBLE reg, vreg 221 add ip2, xFP, \vreg, uxtw #2 222 str \reg, [ip2] 223 add ip2, xREFS, \vreg, uxtw #2 224 str xzr, [ip2] 225.endm 226 227/* 228 * Get the 32-bit value from a Dalvik register and sign-extend to 64-bit. 229 * Used to avoid an extra instruction in int-to-long. 230 */ 231.macro GET_VREG_S reg, vreg 232 ldrsw \reg, [xFP, \vreg, uxtw #2] 233.endm 234 235// An assembly entry that has a OatQuickMethodHeader prefix. 236.macro OAT_ENTRY name, end 237 .type \name, #function 238 .hidden \name 239 .global \name 240 .balign 16 241 // Padding of 3 * 4 bytes to get 16 bytes alignment of code entry. 242 .long 0 243 .long 0 244 .long 0 245 // OatQuickMethodHeader. Note that the top two bits must be clear. 246 .long (\end - \name) 247\name: 248.endm 249 250.macro SIZE name 251 .size \name, .-\name 252.endm 253 254.macro NAME_START name 255 .type \name, #function 256 .hidden \name // Hide this as a global symbol, so we do not incur plt calls. 257 .global \name 258 /* Cache alignment for function entry */ 259 .balign 16 260\name: 261.endm 262 263.macro NAME_END name 264 SIZE \name 265.endm 266 267// Macro for defining entrypoints into runtime. We don't need to save registers 268// (we're not holding references there), but there is no 269// kDontSave runtime method. So just use the kSaveRefsOnly runtime method. 270.macro NTERP_TRAMPOLINE name, helper 271ENTRY \name 272 SETUP_SAVE_REFS_ONLY_FRAME 273 bl \helper 274 RESTORE_SAVE_REFS_ONLY_FRAME 275 REFRESH_MARKING_REGISTER 276 ldr xIP0, [xSELF, # THREAD_EXCEPTION_OFFSET] // Get exception field. 277 cbnz xIP0, nterp_deliver_pending_exception 278 ret 279END \name 280.endm 281 282.macro CLEAR_STATIC_VOLATILE_MARKER reg 283 and \reg, \reg, #-2 284.endm 285 286.macro CLEAR_INSTANCE_VOLATILE_MARKER reg 287 neg \reg, \reg 288.endm 289 290.macro EXPORT_PC 291 str xPC, [xREFS, #-16] 292.endm 293 294.macro BRANCH 295 add xPC, xPC, wINST, sxtw #1 // update xPC 296 // Update method counter and do a suspend check if the branch is negative or zero. 297 cmp wINST, #0 298 b.le 2f 2991: 300 FETCH wINST, 0 // load wINST 301 GET_INST_OPCODE ip // extract opcode from wINST 302 GOTO_OPCODE ip // jump to next instruction 3032: 304 ldr x0, [sp] 305 ldrh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] 306#if (NTERP_HOTNESS_VALUE != 0) 307#error Expected 0 for hotness value 308#endif 309 // If the counter is at zero, handle this in the runtime. 310 cbz w2, NterpHandleHotnessOverflow 311 add x2, x2, #-1 312 strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] 313 DO_SUSPEND_CHECK continue_label=1b 314 b 1b 315.endm 316 317// Uses x12, x13, and x14 as temporaries. 318.macro FETCH_CODE_ITEM_INFO code_item, registers, outs, ins, load_ins 319 tbz \code_item, #0, 4f 320 and \code_item, \code_item, #-2 // Remove the extra bit that marks it's a compact dex file 321 ldrh w13, [\code_item, #COMPACT_CODE_ITEM_FIELDS_OFFSET] 322 ubfx \registers, w13, #COMPACT_CODE_ITEM_REGISTERS_SIZE_SHIFT, #4 323 ubfx \outs, w13, #COMPACT_CODE_ITEM_OUTS_SIZE_SHIFT, #4 324 .if \load_ins 325 ubfx \ins, w13, #COMPACT_CODE_ITEM_INS_SIZE_SHIFT, #4 326 .else 327 ubfx w14, w13, #COMPACT_CODE_ITEM_INS_SIZE_SHIFT, #4 328 add \registers, \registers, w14 329 .endif 330 ldrh w13, [\code_item, #COMPACT_CODE_ITEM_FLAGS_OFFSET] 331 tst w13, #COMPACT_CODE_ITEM_REGISTERS_INS_OUTS_FLAGS 332 b.eq 3f 333 sub x14, \code_item, #4 334 tst w13, #COMPACT_CODE_ITEM_INSNS_FLAG 335 csel x14, x14, \code_item, ne 336 337 tbz w13, #COMPACT_CODE_ITEM_REGISTERS_BIT, 1f 338 ldrh w12, [x14, #-2]! 339 add \registers, \registers, w12 3401: 341 tbz w13, #COMPACT_CODE_ITEM_INS_BIT, 2f 342 ldrh w12, [x14, #-2]! 343 .if \load_ins 344 add \ins, \ins, w12 345 .else 346 add \registers, \registers, w12 347 .endif 3482: 349 tbz w13, #COMPACT_CODE_ITEM_OUTS_BIT, 3f 350 ldrh w12, [x14, #-2]! 351 add \outs, \outs, w12 3523: 353 .if \load_ins 354 add \registers, \registers, \ins 355 .endif 356 add \code_item, \code_item, #COMPACT_CODE_ITEM_INSNS_OFFSET 357 b 5f 3584: 359 // Fetch dex register size. 360 ldrh \registers, [\code_item, #CODE_ITEM_REGISTERS_SIZE_OFFSET] 361 // Fetch outs size. 362 ldrh \outs, [\code_item, #CODE_ITEM_OUTS_SIZE_OFFSET] 363 .if \load_ins 364 ldrh \ins, [\code_item, #CODE_ITEM_INS_SIZE_OFFSET] 365 .endif 366 add \code_item, \code_item, #CODE_ITEM_INSNS_OFFSET 3675: 368.endm 369 370.macro TEST_IF_MARKING label 371 cbnz wMR, \label 372.endm 373 374// Setup the stack to start executing the method. Expects: 375// - x0 to contain the ArtMethod 376// 377// Outputs 378// - ip contains the dex registers size 379// - x28 contains the old stack pointer. 380// - \code_item is replaced with a pointer to the instructions 381// - if load_ins is 1, w15 contains the ins 382// 383// Uses ip, ip2, x12, x13, x14 as temporaries. 384.macro SETUP_STACK_FRAME code_item, refs, fp, cfi_refs, load_ins 385 FETCH_CODE_ITEM_INFO \code_item, wip, wip2, w15, \load_ins 386 387 // Compute required frame size: ((2 * ip) + ip2) * 4 + 24 388 // 24 is for saving the previous frame, pc, and method being executed. 389 add x14, ip, ip 390 add x14, x14, ip2 391 lsl x14, x14, #2 392 add x14, x14, #24 393 394 // Compute new stack pointer in x14 395 sub x14, sp, x14 396 // Alignment 397 and x14, x14, #-16 398 399 // Set reference and dex registers, align to pointer size for previous frame and dex pc. 400 add \refs, x14, ip2, lsl #2 401 add \refs, \refs, 28 402 and \refs, \refs, #(-__SIZEOF_POINTER__) 403 add \fp, \refs, ip, lsl #2 404 405 // Now setup the stack pointer. 406 mov x28, sp 407 .cfi_def_cfa_register x28 408 mov sp, x14 409 str x28, [\refs, #-8] 410 CFI_DEF_CFA_BREG_PLUS_UCONST \cfi_refs, -8, CALLEE_SAVES_SIZE 411 412 // Put nulls in reference frame. 413 cbz ip, 2f 414 mov ip2, \refs 4151: 416 str xzr, [ip2], #8 // May clear vreg[0]. 417 cmp ip2, \fp 418 b.lo 1b 4192: 420 // Save the ArtMethod. 421 str x0, [sp] 422.endm 423 424// Increase method hotness and do suspend check before starting executing the method. 425.macro START_EXECUTING_INSTRUCTIONS 426 ldr x0, [sp] 427 ldrh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] 428#if (NTERP_HOTNESS_VALUE != 0) 429#error Expected 0 for hotness value 430#endif 431 // If the counter is at zero, handle this in the runtime. 432 cbz w2, 3f 433 add x2, x2, #-1 434 strh w2, [x0, #ART_METHOD_HOTNESS_COUNT_OFFSET] 4351: 436 DO_SUSPEND_CHECK continue_label=2f 4372: 438 FETCH_INST 439 GET_INST_OPCODE ip 440 GOTO_OPCODE ip 4413: 442 CHECK_AND_UPDATE_SHARED_MEMORY_METHOD if_hot=4f, if_not_hot=1b 4434: 444 mov x1, xzr 445 mov x2, xFP 446 bl nterp_hot_method 447 b 2b 448.endm 449 450.macro SPILL_ALL_CALLEE_SAVES 451 INCREASE_FRAME CALLEE_SAVES_SIZE 452 // Note: we technically don't need to save x19 and x20, 453 // but the runtime will expect those values to be there when unwinding 454 // (see Arm64Context::DoLongJump checking for the thread register). 455 SAVE_ALL_CALLEE_SAVES 0 456.endm 457 458.macro RESTORE_ALL_CALLEE_SAVES 459 // FP callee-saves 460 ldp d8, d9, [sp, #0] 461 ldp d10, d11, [sp, #16] 462 ldp d12, d13, [sp, #32] 463 ldp d14, d15, [sp, #48] 464 465 // GP callee-saves. 466 // No need to restore x19 (it's always the thread), and 467 // don't restore x20 (the marking register) as it may have been updated, 468 // don't restore x21 (the suspend check register) as it may have been updated. 469 RESTORE_REG x22, 88 470 RESTORE_TWO_REGS x23, x24, 96 471 RESTORE_TWO_REGS x25, x26, 112 472 RESTORE_TWO_REGS x27, x28, 128 473 RESTORE_TWO_REGS x29, lr, 144 474 475 DECREASE_FRAME CALLEE_SAVES_SIZE 476.endm 477 478.macro SPILL_ALL_ARGUMENTS 479 stp x0, x1, [sp, #-128]! 480 stp x2, x3, [sp, #16] 481 stp x4, x5, [sp, #32] 482 stp x6, x7, [sp, #48] 483 stp d0, d1, [sp, #64] 484 stp d2, d3, [sp, #80] 485 stp d4, d5, [sp, #96] 486 stp d6, d7, [sp, #112] 487.endm 488 489.macro RESTORE_ALL_ARGUMENTS 490 ldp x2, x3, [sp, #16] 491 ldp x4, x5, [sp, #32] 492 ldp x6, x7, [sp, #48] 493 ldp d0, d1, [sp, #64] 494 ldp d2, d3, [sp, #80] 495 ldp d4, d5, [sp, #96] 496 ldp d6, d7, [sp, #112] 497 ldp x0, x1, [sp], #128 498.endm 499 500// Helper to setup the stack after doing a nterp to nterp call. This will setup: 501// - xNEW_FP: the new pointer to dex registers 502// - xNEW_REFS: the new pointer to references 503// - xPC: the new PC pointer to execute 504// - x2: value in instruction to decode the number of arguments. 505// - x3: first dex register 506// - x4: top of dex register array 507// 508// The method expects: 509// - x0 to contain the ArtMethod 510// - x8 to contain the code item 511.macro SETUP_STACK_FOR_INVOKE 512 // We do the same stack overflow check as the compiler. See CanMethodUseNterp 513 // in how we limit the maximum nterp frame size. 514 sub x16, sp, #STACK_OVERFLOW_RESERVED_BYTES 515 ldr wzr, [x16] 516 517 // Spill all callee saves to have a consistent stack frame whether we 518 // are called by compiled code or nterp. 519 SPILL_ALL_CALLEE_SAVES 520 521 // Setup the frame. 522 SETUP_STACK_FRAME x8, xNEW_REFS, xNEW_FP, CFI_NEW_REFS, load_ins=0 523 // Make x4 point to the top of the dex register array. 524 add x4, xNEW_FP, ip, uxtx #2 525 526 // Fetch instruction information before replacing xPC. 527 // TODO: move this down to the method that uses it, fetching it directly from wINST. 528 FETCH_B w2, 0, 1 529 // TODO: we could avoid this as instance invokes already fetch it. 530 FETCH w3, 2 531 532 // Set the dex pc pointer. 533 mov xPC, x8 534 CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) 535.endm 536 537// Setup arguments based on a non-range nterp to nterp call, and start executing 538// the method. We expect: 539// - xNEW_FP: the new pointer to dex registers 540// - xNEW_REFS: the new pointer to references 541// - xPC: the new PC pointer to execute 542// - x2: number of arguments (bits 4-7), 5th argument if any (bits 0-3) 543// - x3: first dex register 544// - x4: top of dex register array 545// - x1: receiver if non-static. 546.macro SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 547 // /* op vA, vB, {vC...vG} */ 548 asr ip2, x2, #4 549 cbz ip2, 6f 550 mov ip, #-4 551 cmp ip2, #2 552 b.lt 1f 553 b.eq 2f 554 cmp ip2, #4 555 b.lt 3f 556 b.eq 4f 557 558 // We use a decrementing ip to store references relative 559 // to xNEW_FP and dex registers relative to x4 560 // 561 // TODO: We could set up ip as the number of registers (this can be an additional output from 562 // SETUP_STACK_FOR_INVOKE) and then just decrement it by one before copying each arg. 563 // Maybe even introduce macros NEW_VREG_ADDRESS/NEW_VREG_REF_ADDRESS. 5645: 565 and x2, x2, #15 566 GET_VREG_OBJECT w5, w2 567 str w5, [xNEW_FP, ip] 568 GET_VREG w5, w2 569 str w5, [x4, ip] 570 sub ip, ip, #4 5714: 572 asr x2, x3, #12 573 GET_VREG_OBJECT w5, w2 574 str w5, [xNEW_FP, ip] 575 GET_VREG w5, w2 576 str w5, [x4, ip] 577 sub ip, ip, #4 5783: 579 ubfx x2, x3, #8, #4 580 GET_VREG_OBJECT w5, w2 581 str w5, [xNEW_FP, ip] 582 GET_VREG w5, w2 583 str w5, [x4, ip] 584 sub ip, ip, #4 5852: 586 ubfx x2, x3, #4, #4 587 GET_VREG_OBJECT w5, w2 588 str w5, [xNEW_FP, ip] 589 GET_VREG w5, w2 590 str w5, [x4, ip] 591 .if !\is_string_init 592 sub ip, ip, #4 593 .endif 5941: 595 .if \is_string_init 596 // Ignore the first argument 597 .elseif \is_static 598 and x2, x3, #0xf 599 GET_VREG_OBJECT w5, w2 600 str w5, [xNEW_FP, ip] 601 GET_VREG w5, w2 602 str w5, [x4, ip] 603 .else 604 str w1, [xNEW_FP, ip] 605 str w1, [x4, ip] 606 .endif 607 6086: 609 // Start executing the method. 610 mov xFP, xNEW_FP 611 mov xREFS, xNEW_REFS 612 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, CALLEE_SAVES_SIZE 613 START_EXECUTING_INSTRUCTIONS 614.endm 615 616// Setup arguments based on a range nterp to nterp call, and start executing 617// the method. 618// - xNEW_FP: the new pointer to dex registers 619// - xNEW_REFS: the new pointer to references 620// - xPC: the new PC pointer to execute 621// - x2: number of arguments 622// - x3: first dex register 623// - x4: top of dex register array 624// - x1: receiver if non-static. 625// 626// Uses ip, ip2, x5, x6 as temporaries. 627.macro SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 628 mov ip, #-4 629 .if \is_string_init 630 // Ignore the first argument 631 sub x2, x2, #1 632 add x3, x3, #1 633 .elseif !\is_static 634 sub x2, x2, #1 635 add x3, x3, #1 636 .endif 637 638 cbz x2, 2f 639 add ip2, xREFS, x3, lsl #2 // pointer to first argument in reference array 640 add ip2, ip2, x2, lsl #2 // pointer to last argument in reference array 641 add x5, xFP, x3, lsl #2 // pointer to first argument in register array 642 add x6, x5, x2, lsl #2 // pointer to last argument in register array 6431: 644 ldr w7, [ip2, #-4]! 645 str w7, [xNEW_FP, ip] 646 sub x2, x2, 1 647 ldr w7, [x6, #-4]! 648 str w7, [x4, ip] 649 sub ip, ip, 4 650 cbnz x2, 1b 6512: 652 .if \is_string_init 653 // Ignore first argument 654 .elseif !\is_static 655 str w1, [xNEW_FP, ip] 656 str w1, [x4, ip] 657 .endif 658 mov xFP, xNEW_FP 659 mov xREFS, xNEW_REFS 660 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, CALLEE_SAVES_SIZE 661 START_EXECUTING_INSTRUCTIONS 662.endm 663 664.macro GET_SHORTY dest, is_interface, is_polymorphic, is_custom 665 stp x0, x1, [sp, #-16]! 666 .if \is_polymorphic 667 ldr x0, [sp, #16] 668 mov x1, xPC 669 bl NterpGetShortyFromInvokePolymorphic 670 .elseif \is_custom 671 ldr x0, [sp, #16] 672 mov x1, xPC 673 bl NterpGetShortyFromInvokeCustom 674 .elseif \is_interface 675 ldr x0, [sp, #16] 676 FETCH w1, 1 677 bl NterpGetShortyFromMethodId 678 .else 679 bl NterpGetShorty 680 .endif 681 mov \dest, x0 682 ldp x0, x1, [sp], #16 683.endm 684 685.macro GET_SHORTY_SLOW_PATH dest, is_interface 686 // Save all registers that can hold arguments in the fast path. 687 stp x0, x1, [sp, #-32]! 688 str w2, [sp, #16] 689 str s0, [sp, #20] 690 .if \is_interface 691 ldr x0, [sp, #32] 692 FETCH w1, 1 693 bl NterpGetShortyFromMethodId 694 .else 695 bl NterpGetShorty 696 .endif 697 mov \dest, x0 698 ldr w2, [sp, #16] 699 ldr s0, [sp, #20] 700 ldp x0, x1, [sp], #32 701.endm 702 703// Input: x0 contains the ArtMethod 704// Output: x8 contains the code item 705.macro GET_CODE_ITEM 706 ldr x8, [x0, #ART_METHOD_DATA_OFFSET_64] 707.endm 708 709.macro DO_ENTRY_POINT_CHECK call_compiled_code 710 // On entry, the method is x0, the instance is x1 711 adr x2, ExecuteNterpImpl 712 ldr x3, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 713 cmp x2, x3 714 b.ne \call_compiled_code 715.endm 716 717.macro UPDATE_REGISTERS_FOR_STRING_INIT old_value, new_value 718 mov wip, wzr 7191: 720 GET_VREG_OBJECT wip2, wip 721 cmp wip2, \old_value 722 b.ne 2f 723 SET_VREG_OBJECT \new_value, wip 7242: 725 add wip, wip, #1 726 add ip2, xREFS, wip, uxtw #2 727 cmp ip2, xFP 728 b.ne 1b 729.endm 730 731// Puts the next floating point argument into the expected register, 732// fetching values based on a non-range invoke. 733// Uses ip and ip2. 734.macro LOOP_OVER_SHORTY_LOADING_FPS dreg, sreg, inst, shorty, arg_index, finished 7351: // LOOP 736 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 737 cbz wip, \finished // if (wip == '\0') goto finished 738 cmp wip, #68 // if (wip == 'D') goto FOUND_DOUBLE 739 b.eq 2f 740 cmp wip, #70 // if (wip == 'F') goto FOUND_FLOAT 741 b.eq 3f 742 lsr \inst, \inst, #4 743 add \arg_index, \arg_index, #1 744 // Handle extra argument in arg array taken by a long. 745 cmp wip, #74 // if (wip != 'J') goto LOOP 746 b.ne 1b 747 lsr \inst, \inst, #4 748 add \arg_index, \arg_index, #1 749 b 1b // goto LOOP 7502: // FOUND_DOUBLE 751 and ip, \inst, #0xf 752 GET_VREG wip, wip 753 lsr \inst, \inst, #4 754 add \arg_index, \arg_index, #1 755 cmp \arg_index, #4 756 b.eq 5f 757 and ip2, \inst, #0xf 758 lsr \inst, \inst, #4 759 add \arg_index, \arg_index, #1 760 b 6f 7615: 762 // TODO: Extract from wINST here and below? (Requires using a different register 763 // in the COMMON_INVOKE_NON_RANGE.) 764 FETCH_B wip2, 0, 1 765 and wip2, wip2, #0xf 7666: 767 GET_VREG wip2, wip2 768 add ip, ip, ip2, lsl #32 769 fmov \dreg, ip 770 b 4f 7713: // FOUND_FLOAT 772 cmp \arg_index, #4 773 b.eq 7f 774 and ip, \inst, #0xf 775 lsr \inst, \inst, #4 776 add \arg_index, \arg_index, #1 777 b 8f 7787: 779 FETCH_B wip, 0, 1 780 and wip, wip, #0xf 7818: 782 GET_VREG \sreg, wip 7834: 784.endm 785 786// Puts the next int/long/object argument in the expected register, 787// fetching values based on a non-range invoke. 788// Uses ip and ip2. 789.macro LOOP_OVER_SHORTY_LOADING_GPRS gpr_reg64, gpr_reg32, inst, shorty, arg_index, finished 7901: // LOOP 791 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 792 cbz wip, \finished // if (wip == '\0') goto finished 793 cmp wip, #74 // if (wip == 'J') goto FOUND_LONG 794 b.eq 2f 795 cmp wip, #70 // if (wip == 'F') goto SKIP_FLOAT 796 b.eq 3f 797 cmp wip, #68 // if (wip == 'D') goto SKIP_DOUBLE 798 b.eq 4f 799 cmp \arg_index, #4 800 b.eq 7f 801 and ip, \inst, #0xf 802 lsr \inst, \inst, #4 803 add \arg_index, \arg_index, #1 804 b 8f 8057: 806 FETCH_B wip, 0, 1 807 and wip, wip, #0xf 8088: 809 GET_VREG \gpr_reg32, wip 810 b 5f 8112: // FOUND_LONG 812 and ip, \inst, #0xf 813 GET_VREG wip, wip 814 lsr \inst, \inst, #4 815 add \arg_index, \arg_index, #1 816 cmp \arg_index, #4 817 b.eq 9f 818 and ip2, \inst, #0xf 819 lsr \inst, \inst, #4 820 add \arg_index, \arg_index, #1 821 b 10f 8229: 823 FETCH_B wip2, 0, 1 824 and wip2, wip2, #0xf 82510: 826 GET_VREG wip2, wip2 827 add \gpr_reg64, ip, ip2, lsl #32 828 b 5f 8293: // SKIP_FLOAT 830 lsr \inst, \inst, #4 831 add \arg_index, \arg_index, #1 832 b 1b 8334: // SKIP_DOUBLE 834 lsr \inst, \inst, #4 835 add \arg_index, \arg_index, #1 836 cmp \arg_index, #4 837 b.eq 1b 838 lsr \inst, \inst, #4 839 add \arg_index, \arg_index, #1 840 b 1b 8415: 842.endm 843 844.macro SETUP_RETURN_VALUE shorty 845 ldrb wip, [\shorty] 846 cmp ip, #68 // Test if result type char == 'D'. 847 b.eq 1f 848 cmp ip, #70 // Test if result type char == 'F'. 849 b.ne 2f 850 fmov w0, s0 851 b 2f 8521: 853 fmov x0, d0 8542: 855.endm 856 857.macro COMMON_INVOKE_NON_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 858 .if \is_polymorphic 859 // We always go to compiled code for polymorphic calls. 860 .elseif \is_custom 861 // We always go to compiled code for custom calls. 862 .else 863 DO_ENTRY_POINT_CHECK .Lcall_compiled_code_\suffix 864 GET_CODE_ITEM 865 .if \is_string_init 866 bl nterp_to_nterp_string_init_non_range 867 .elseif \is_static 868 bl nterp_to_nterp_static_non_range 869 .else 870 bl nterp_to_nterp_instance_non_range 871 .endif 872 b .Ldone_return_\suffix 873 .endif 874 875.Lcall_compiled_code_\suffix: 876 .if \is_polymorphic 877 // No fast path for polymorphic calls. 878 .elseif \is_custom 879 // No fast path for custom calls. 880 .elseif \is_string_init 881 // No fast path for string.init. 882 .else 883 ldr wip, [x0, #ART_METHOD_ACCESS_FLAGS_OFFSET] 884 tbz wip, #ART_METHOD_NTERP_INVOKE_FAST_PATH_FLAG_BIT, .Lfast_path_with_few_args_\suffix 885 FETCH_B wip2, 0, 1 886 asr ip, ip2, #4 887 .if \is_static 888 cbz ip, .Linvoke_fast_path_\suffix 889 .else 890 cmp ip, #1 891 b.eq .Linvoke_fast_path_\suffix 892 .endif 893 FETCH w8, 2 894 cmp ip, #2 895 .if \is_static 896 b.lt .Lone_arg_fast_path_\suffix 897 .endif 898 b.eq .Ltwo_args_fast_path_\suffix 899 cmp ip, #4 900 b.lt .Lthree_args_fast_path_\suffix 901 b.eq .Lfour_args_fast_path_\suffix 902 903 and ip, ip2, #15 904 GET_VREG w5, wip 905.Lfour_args_fast_path_\suffix: 906 asr ip, x8, #12 907 GET_VREG w4, wip 908.Lthree_args_fast_path_\suffix: 909 ubfx ip, x8, #8, #4 910 GET_VREG w3, wip 911.Ltwo_args_fast_path_\suffix: 912 ubfx ip, x8, #4, #4 913 GET_VREG w2, wip 914.Lone_arg_fast_path_\suffix: 915 .if \is_static 916 and ip, x8, #0xf 917 GET_VREG w1, wip 918 .else 919 // First argument already in w1. 920 .endif 921.Linvoke_fast_path_\suffix: 922 .if \is_interface 923 // Setup hidden argument. 924 mov ip2, x26 925 .endif 926 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 927 blr lr 928 FETCH_ADVANCE_INST 3 929 GET_INST_OPCODE ip 930 GOTO_OPCODE ip 931 932.Lfast_path_with_few_args_\suffix: 933 // Fast path when we have zero or one argument (modulo 'this'). If there 934 // is one argument, we can put it in both floating point and core register. 935 FETCH_B w2, 0, 1 936 .if \is_static 937 cmp w2, #(2 << 4) 938 .else 939 cmp w2, #(3 << 4) 940 .endif 941 b.ge .Lget_shorty_\suffix 942 .if \is_static 943 tbz w2, #4, .Linvoke_with_few_args_\suffix 944 .else 945 tbnz w2, #4, .Linvoke_with_few_args_\suffix 946 .endif 947 FETCH w2, 2 948 .if \is_static 949 and w2, w2, #0xf // dex register of first argument 950 GET_VREG w1, w2 951 fmov s0, w1 952 .else 953 ubfx x2, x2, #4, #4 // dex register of second argument 954 GET_VREG w2, w2 955 fmov s0, w2 956 .endif 957.Linvoke_with_few_args_\suffix: 958 // Check if the next instruction is move-result or move-result-wide. 959 // If it is, we fetch the shorty and jump to the regular invocation. 960 FETCH w27, 3 961 and ip, x27, #0xfe 962 cmp ip, #0x0a 963 b.eq .Lget_shorty_and_invoke_\suffix 964 .if \is_interface 965 // Setup hidden argument. 966 mov ip2, x26 967 .endif 968 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 969 blr lr 970 # TODO: Use some other register for shorty and prefetch the instruction directly to wINST. 971 mov xINST, x27 972 ADVANCE 3 973 GET_INST_OPCODE ip 974 GOTO_OPCODE ip 975.Lget_shorty_and_invoke_\suffix: 976 GET_SHORTY_SLOW_PATH xINST, \is_interface 977 b .Lgpr_setup_finished_\suffix 978 .endif 979 980.Lget_shorty_\suffix: 981 GET_SHORTY xINST, \is_interface, \is_polymorphic, \is_custom 982 // From this point: 983 // - xINST contains shorty (in callee-save to switch over return value after call). 984 // - x0 contains method 985 // - x1 contains 'this' pointer for instance method. 986 // - for interface calls, x26 contains the interface method. 987 add x9, xINST, #1 // shorty + 1 ; ie skip return arg character 988 FETCH w11, 2 // arguments 989 .if \is_string_init 990 lsr x11, x11, #4 991 mov x10, #1 // ignore first argument 992 .elseif \is_static 993 mov x10, xzr // arg_index 994 .else 995 lsr x11, x11, #4 996 mov x10, #1 // ignore first argument 997 .endif 998 LOOP_OVER_SHORTY_LOADING_FPS d0, s0, x11, x9, x10, .Lxmm_setup_finished_\suffix 999 LOOP_OVER_SHORTY_LOADING_FPS d1, s1, x11, x9, x10, .Lxmm_setup_finished_\suffix 1000 LOOP_OVER_SHORTY_LOADING_FPS d2, s2, x11, x9, x10, .Lxmm_setup_finished_\suffix 1001 LOOP_OVER_SHORTY_LOADING_FPS d3, s3, x11, x9, x10, .Lxmm_setup_finished_\suffix 1002 LOOP_OVER_SHORTY_LOADING_FPS d4, s4, x11, x9, x10, .Lxmm_setup_finished_\suffix 1003.Lxmm_setup_finished_\suffix: 1004 add x9, xINST, #1 // shorty + 1 ; ie skip return arg character 1005 FETCH w11, 2 // arguments 1006 .if \is_string_init 1007 lsr x11, x11, #4 1008 mov x10, #1 // ignore first argument 1009 LOOP_OVER_SHORTY_LOADING_GPRS x1, w1, x11, x9, x10, .Lgpr_setup_finished_\suffix 1010 .elseif \is_static 1011 mov x10, xzr // arg_index 1012 LOOP_OVER_SHORTY_LOADING_GPRS x1, w1, x11, x9, x10, .Lgpr_setup_finished_\suffix 1013 .else 1014 lsr x11, x11, #4 1015 mov x10, #1 // ignore first argument 1016 .endif 1017 LOOP_OVER_SHORTY_LOADING_GPRS x2, w2, x11, x9, x10, .Lgpr_setup_finished_\suffix 1018 LOOP_OVER_SHORTY_LOADING_GPRS x3, w3, x11, x9, x10, .Lgpr_setup_finished_\suffix 1019 LOOP_OVER_SHORTY_LOADING_GPRS x4, w4, x11, x9, x10, .Lgpr_setup_finished_\suffix 1020 LOOP_OVER_SHORTY_LOADING_GPRS x5, w5, x11, x9, x10, .Lgpr_setup_finished_\suffix 1021.Lgpr_setup_finished_\suffix: 1022 .if \is_polymorphic 1023 bl art_quick_invoke_polymorphic 1024 .elseif \is_custom 1025 bl art_quick_invoke_custom 1026 .else 1027 .if \is_interface 1028 // Setup hidden argument. 1029 mov ip2, x26 1030 .endif 1031 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1032 blr lr 1033 .endif 1034 SETUP_RETURN_VALUE xINST 1035.Ldone_return_\suffix: 1036 /* resume execution of caller */ 1037 .if \is_string_init 1038 FETCH w11, 2 // arguments 1039 and x11, x11, #0xf 1040 GET_VREG w1, w11 1041 UPDATE_REGISTERS_FOR_STRING_INIT w1, w0 1042 .endif 1043 1044 .if \is_polymorphic 1045 FETCH_ADVANCE_INST 4 1046 .else 1047 FETCH_ADVANCE_INST 3 1048 .endif 1049 GET_INST_OPCODE ip 1050 GOTO_OPCODE ip 1051.endm 1052 1053// Puts the next floating point argument into the expected register, 1054// fetching values based on a range invoke. 1055// Uses ip as temporary. 1056.macro LOOP_RANGE_OVER_SHORTY_LOADING_FPS dreg, sreg, shorty, arg_index, stack_index, finished 10571: // LOOP 1058 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1059 cbz wip, \finished // if (wip == '\0') goto finished 1060 cmp wip, #68 // if (wip == 'D') goto FOUND_DOUBLE 1061 b.eq 2f 1062 cmp wip, #70 // if (wip == 'F') goto FOUND_FLOAT 1063 b.eq 3f 1064 add \arg_index, \arg_index, #1 1065 add \stack_index, \stack_index, #1 1066 // Handle extra argument in arg array taken by a long. 1067 cmp wip, #74 // if (wip != 'J') goto LOOP 1068 b.ne 1b 1069 add \arg_index, \arg_index, #1 1070 add \stack_index, \stack_index, #1 1071 b 1b // goto LOOP 10722: // FOUND_DOUBLE 1073 GET_VREG_DOUBLE \dreg, \arg_index 1074 add \arg_index, \arg_index, #2 1075 add \stack_index, \stack_index, #2 1076 b 4f 10773: // FOUND_FLOAT 1078 GET_VREG \sreg, \arg_index 1079 add \arg_index, \arg_index, #1 1080 add \stack_index, \stack_index, #1 10814: 1082.endm 1083 1084// Puts the next floating point argument into the expected stack slot, 1085// fetching values based on a range invoke. 1086// Uses ip as temporary. 1087// 1088// TODO: We could just copy all the vregs to the stack slots in a simple loop 1089// without looking at the shorty at all. (We could also drop 1090// the "stack_index" from the macros for loading registers.) We could also do 1091// that conditionally if argument word count > 6; otherwise we know that all 1092// args fit into registers. 1093.macro LOOP_RANGE_OVER_FPs shorty, arg_index, stack_index, finished 10941: // LOOP 1095 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1096 cbz wip, \finished // if (wip == '\0') goto finished 1097 cmp wip, #68 // if (wip == 'D') goto FOUND_DOUBLE 1098 b.eq 2f 1099 cmp wip, #70 // if (wip == 'F') goto FOUND_FLOAT 1100 b.eq 3f 1101 add \arg_index, \arg_index, #1 1102 add \stack_index, \stack_index, #1 1103 // Handle extra argument in arg array taken by a long. 1104 cmp wip, #74 // if (wip != 'J') goto LOOP 1105 b.ne 1b 1106 add \arg_index, \arg_index, #1 1107 add \stack_index, \stack_index, #1 1108 b 1b // goto LOOP 11092: // FOUND_DOUBLE 1110 GET_VREG_WIDE ip, \arg_index 1111 add ip2, sp, \stack_index, uxtw #2 1112 str ip, [ip2] 1113 add \arg_index, \arg_index, #2 1114 add \stack_index, \stack_index, #2 1115 b 1b 11163: // FOUND_FLOAT 1117 GET_VREG wip, \arg_index 1118 str wip, [sp, \stack_index, uxtw #2] 1119 add \arg_index, \arg_index, #1 1120 add \stack_index, \stack_index, #1 1121 b 1b 1122.endm 1123 1124// Puts the next int/long/object argument in the expected register, 1125// fetching values based on a range invoke. 1126// Uses ip as temporary. 1127.macro LOOP_RANGE_OVER_SHORTY_LOADING_GPRS reg64, reg32, shorty, arg_index, stack_index, finished 11281: // LOOP 1129 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1130 cbz wip, \finished // if (wip == '\0') goto finished 1131 cmp wip, #74 // if (wip == 'J') goto FOUND_LONG 1132 b.eq 2f 1133 cmp wip, #70 // if (wip == 'F') goto SKIP_FLOAT 1134 b.eq 3f 1135 cmp wip, #68 // if (wip == 'D') goto SKIP_DOUBLE 1136 b.eq 4f 1137 GET_VREG \reg32, \arg_index 1138 add \arg_index, \arg_index, #1 1139 add \stack_index, \stack_index, #1 1140 b 5f 11412: // FOUND_LONG 1142 GET_VREG_WIDE \reg64, \arg_index 1143 add \arg_index, \arg_index, #2 1144 add \stack_index, \stack_index, #2 1145 b 5f 11463: // SKIP_FLOAT 1147 add \arg_index, \arg_index, #1 1148 add \stack_index, \stack_index, #1 1149 b 1b 11504: // SKIP_DOUBLE 1151 add \arg_index, \arg_index, #2 1152 add \stack_index, \stack_index, #2 1153 b 1b 11545: 1155.endm 1156 1157// Puts the next int/long/object argument in the expected stack slot, 1158// fetching values based on a range invoke. 1159// Uses ip as temporary. 1160.macro LOOP_RANGE_OVER_INTs shorty, arg_index, stack_index, finished 11611: // LOOP 1162 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1163 cbz wip, \finished // if (wip == '\0') goto finished 1164 cmp wip, #74 // if (wip == 'J') goto FOUND_LONG 1165 b.eq 2f 1166 cmp wip, #70 // if (wip == 'F') goto SKIP_FLOAT 1167 b.eq 3f 1168 cmp wip, #68 // if (wip == 'D') goto SKIP_DOUBLE 1169 b.eq 4f 1170 GET_VREG wip, \arg_index 1171 str wip, [sp, \stack_index, uxtw #2] 1172 add \arg_index, \arg_index, #1 1173 add \stack_index, \stack_index, #1 1174 b 1b 11752: // FOUND_LONG 1176 GET_VREG_WIDE ip, \arg_index 1177 add ip2, sp, \stack_index, uxtw #2 1178 str ip, [ip2] 1179 add \arg_index, \arg_index, #2 1180 add \stack_index, \stack_index, #2 1181 b 1b 11823: // SKIP_FLOAT 1183 add \arg_index, \arg_index, #1 1184 add \stack_index, \stack_index, #1 1185 b 1b 11864: // SKIP_DOUBLE 1187 add \arg_index, \arg_index, #2 1188 add \stack_index, \stack_index, #2 1189 b 1b 1190.endm 1191 1192.macro COMMON_INVOKE_RANGE is_static=0, is_interface=0, suffix="", is_string_init=0, is_polymorphic=0, is_custom=0 1193 .if \is_polymorphic 1194 // We always go to compiled code for polymorphic calls. 1195 .elseif \is_custom 1196 // We always go to compiled code for custom calls. 1197 .else 1198 DO_ENTRY_POINT_CHECK .Lcall_compiled_code_range_\suffix 1199 GET_CODE_ITEM 1200 .if \is_string_init 1201 bl nterp_to_nterp_string_init_range 1202 .elseif \is_static 1203 bl nterp_to_nterp_static_range 1204 .else 1205 bl nterp_to_nterp_instance_range 1206 .endif 1207 b .Ldone_return_range_\suffix 1208 .endif 1209 1210.Lcall_compiled_code_range_\suffix: 1211 .if \is_polymorphic 1212 // No fast path for polymorphic calls. 1213 .elseif \is_custom 1214 // No fast path for custom calls. 1215 .elseif \is_string_init 1216 // No fast path for string.init. 1217 .else 1218 ldr wip, [x0, #ART_METHOD_ACCESS_FLAGS_OFFSET] 1219 tbz wip, #ART_METHOD_NTERP_INVOKE_FAST_PATH_FLAG_BIT, .Lfast_path_with_few_args_range_\suffix 1220 FETCH_B wip2, 0, 1 // Number of arguments 1221 .if \is_static 1222 cbz ip2, .Linvoke_fast_path_range_\suffix 1223 .else 1224 cmp ip2, #1 1225 b.eq .Linvoke_fast_path_range_\suffix 1226 .endif 1227 FETCH wip, 2 // dex register of first argument 1228 add x8, xFP, wip, uxtw #2 // location of first dex register value 1229 cmp ip2, #2 1230 .if \is_static 1231 b.lt .Lone_arg_fast_path_range_\suffix 1232 .endif 1233 b.eq .Ltwo_args_fast_path_range_\suffix 1234 cmp ip2, #4 1235 b.lt .Lthree_args_fast_path_range_\suffix 1236 b.eq .Lfour_args_fast_path_range_\suffix 1237 cmp ip2, #6 1238 b.lt .Lfive_args_fast_path_range_\suffix 1239 b.eq .Lsix_args_fast_path_range_\suffix 1240 cmp ip2, #7 1241 b.eq .Lseven_args_fast_path_range_\suffix 1242 // Setup x8 to point to the stack location of parameters we do not need 1243 // to put parameters in. 1244 add x9, sp, #8 // Add space for the ArtMethod 1245 1246.Lloop_over_fast_path_range_\suffix: 1247 sub ip2, ip2, #1 1248 ldr wip, [x8, ip2, lsl #2] 1249 str wip, [x9, ip2, lsl #2] 1250 cmp ip2, #7 1251 b.ne .Lloop_over_fast_path_range_\suffix 1252 1253.Lseven_args_fast_path_range_\suffix: 1254 ldr w7, [x8, #24] 1255.Lsix_args_fast_path_range_\suffix: 1256 ldr w6, [x8, #20] 1257.Lfive_args_fast_path_range_\suffix: 1258 ldr w5, [x8, #16] 1259.Lfour_args_fast_path_range_\suffix: 1260 ldr w4, [x8, #12] 1261.Lthree_args_fast_path_range_\suffix: 1262 ldr w3, [x8, #8] 1263.Ltwo_args_fast_path_range_\suffix: 1264 ldr w2, [x8, #4] 1265.Lone_arg_fast_path_range_\suffix: 1266 .if \is_static 1267 ldr w1, [x8, #0] 1268 .else 1269 // First argument already in w1. 1270 .endif 1271.Linvoke_fast_path_range_\suffix: 1272 .if \is_interface 1273 // Setup hidden argument. 1274 mov ip2, x26 1275 .endif 1276 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1277 blr lr 1278 FETCH_ADVANCE_INST 3 1279 GET_INST_OPCODE ip 1280 GOTO_OPCODE ip 1281 1282.Lfast_path_with_few_args_range_\suffix: 1283 // Fast path when we have zero or one argument (modulo 'this'). If there 1284 // is one argument, we can put it in both floating point and core register. 1285 FETCH_B w2, 0, 1 // number of arguments 1286 .if \is_static 1287 cmp w2, #1 1288 .else 1289 cmp w2, #2 1290 .endif 1291 b.lt .Linvoke_with_few_args_range_\suffix 1292 b.ne .Lget_shorty_range_\suffix 1293 FETCH w3, 2 // dex register of first argument 1294 .if \is_static 1295 GET_VREG w1, w3 1296 fmov s0, w1 1297 .else 1298 add w3, w3, #1 // Add 1 for next argument 1299 GET_VREG w2, w3 1300 fmov s0, w2 1301 .endif 1302.Linvoke_with_few_args_range_\suffix: 1303 // Check if the next instruction is move-result or move-result-wide. 1304 // If it is, we fetch the shorty and jump to the regular invocation. 1305 FETCH w27, 3 1306 and ip, x27, #0xfe 1307 cmp ip, #0x0a 1308 b.eq .Lget_shorty_and_invoke_range_\suffix 1309 .if \is_interface 1310 // Setup hidden argument. 1311 mov ip2, x26 1312 .endif 1313 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1314 blr lr 1315 mov xINST, x27 1316 ADVANCE 3 1317 GET_INST_OPCODE ip 1318 GOTO_OPCODE ip 1319.Lget_shorty_and_invoke_range_\suffix: 1320 GET_SHORTY_SLOW_PATH xINST, \is_interface 1321 b .Lgpr_setup_finished_range_\suffix 1322 .endif 1323 1324.Lget_shorty_range_\suffix: 1325 GET_SHORTY xINST, \is_interface, \is_polymorphic, \is_custom 1326 // From this point: 1327 // - xINST contains shorty (in callee-save to switch over return value after call). 1328 // - x0 contains method 1329 // - x1 contains 'this' pointer for instance method. 1330 // - for interface calls, x26 contains the interface method. 1331 add x9, xINST, #1 // shorty + 1 ; ie skip return arg character 1332 FETCH w10, 2 // arguments 1333 .if \is_string_init 1334 add x10, x10, #1 // arg start index 1335 mov x11, #1 // index in stack 1336 .elseif \is_static 1337 mov x11, xzr // index in stack 1338 .else 1339 add x10, x10, #1 // arg start index 1340 mov x11, #1 // index in stack 1341 .endif 1342 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d0, s0, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1343 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d1, s1, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1344 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d2, s2, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1345 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d3, s3, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1346 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d4, s4, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1347 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d5, s5, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1348 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d6, s6, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1349 LOOP_RANGE_OVER_SHORTY_LOADING_FPS d7, s7, x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1350 // Store in the outs array (stored above the ArtMethod in the stack) 1351 add x11, x11, #2 // Add two words for the ArtMethod stored before the outs. 1352 LOOP_RANGE_OVER_FPs x9, w10, w11, .Lxmm_setup_finished_range_\suffix 1353.Lxmm_setup_finished_range_\suffix: 1354 add x9, xINST, #1 // shorty + 1 ; ie skip return arg character 1355 FETCH w10, 2 // arguments 1356 .if \is_string_init 1357 add x10, x10, #1 // arg start index 1358 mov x11, #1 // index in stack 1359 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x1, w1, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1360 .elseif \is_static 1361 mov x11, xzr // index in stack 1362 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x1, w1, x9, w10, w11 .Lgpr_setup_finished_range_\suffix 1363 .else 1364 add x10, x10, #1 // arg start index 1365 mov x11, #1 // index in stack 1366 .endif 1367 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x2, w2, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1368 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x3, w3, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1369 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x4, w4, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1370 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x5, w5, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1371 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x6, w6, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1372 LOOP_RANGE_OVER_SHORTY_LOADING_GPRS x7, w7, x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1373 // Store in the outs array (stored above the ArtMethod in the stack) 1374 add x11, x11, #2 // Add two words for the ArtMethod stored before the outs. 1375 LOOP_RANGE_OVER_INTs x9, w10, w11, .Lgpr_setup_finished_range_\suffix 1376.Lgpr_setup_finished_range_\suffix: 1377 .if \is_polymorphic 1378 bl art_quick_invoke_polymorphic 1379 .elseif \is_custom 1380 bl art_quick_invoke_custom 1381 .else 1382 .if \is_interface 1383 // Setup hidden argument. 1384 mov ip2, x26 1385 .endif 1386 ldr lr, [x0, #ART_METHOD_QUICK_CODE_OFFSET_64] 1387 blr lr 1388 .endif 1389 SETUP_RETURN_VALUE xINST 1390.Ldone_return_range_\suffix: 1391 /* resume execution of caller */ 1392 .if \is_string_init 1393 FETCH w11, 2 // arguments 1394 GET_VREG w1, w11 1395 UPDATE_REGISTERS_FOR_STRING_INIT w1, w0 1396 .endif 1397 1398 .if \is_polymorphic 1399 FETCH_ADVANCE_INST 4 1400 .else 1401 FETCH_ADVANCE_INST 3 1402 .endif 1403 GET_INST_OPCODE ip 1404 GOTO_OPCODE ip 1405.endm 1406 1407.macro WRITE_BARRIER_IF_OBJECT is_object, value, holder, label 1408 .if \is_object 1409 cbz \value, \label 1410 ldr ip, [xSELF, #THREAD_CARD_TABLE_OFFSET] 1411 lsr wip2, \holder, #CARD_TABLE_CARD_SHIFT 1412 strb wip, [ip, ip2] 1413\label: 1414 .endif 1415.endm 1416 1417// Puts the next int/long/object parameter passed in physical register 1418// in the expected dex register array entry, and in case of object in the 1419// expected reference array entry. 1420.macro LOOP_OVER_SHORTY_STORING_GPRS gpr_64, gpr_32, shorty, arg_offset, regs, refs, finished 14211: // LOOP 1422 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1423 cbz wip, \finished // if (wip == '\0') goto finished 1424 cmp wip, #74 // if (wip == 'J') goto FOUND_LONG 1425 b.eq 2f 1426 cmp wip, #70 // if (wip == 'F') goto SKIP_FLOAT 1427 b.eq 3f 1428 cmp wip, #68 // if (wip == 'D') goto SKIP_DOUBLE 1429 b.eq 4f 1430 str \gpr_32, [\regs, \arg_offset] 1431 cmp wip, #76 // if (wip != 'L') goto NOT_REFERENCE 1432 b.ne 6f 1433 str \gpr_32, [\refs, \arg_offset] 14346: // NOT_REFERENCE 1435 add \arg_offset, \arg_offset, #4 1436 b 5f 14372: // FOUND_LONG 1438 str \gpr_64, [\regs, \arg_offset] 1439 add \arg_offset, \arg_offset, #8 1440 b 5f 14413: // SKIP_FLOAT 1442 add \arg_offset, \arg_offset, #4 1443 b 1b 14444: // SKIP_DOUBLE 1445 add \arg_offset, \arg_offset, #8 1446 b 1b 14475: 1448.endm 1449 1450// Puts the next floating point parameter passed in physical register 1451// in the expected dex register array entry. 1452// Uses ip as temporary. 1453.macro LOOP_OVER_SHORTY_STORING_FPS dreg, sreg, shorty, arg_offset, fp, finished 14541: // LOOP 1455 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1456 cbz wip, \finished // if (wip == '\0') goto finished 1457 cmp wip, #68 // if (wip == 'D') goto FOUND_DOUBLE 1458 b.eq 2f 1459 cmp wip, #70 // if (wip == 'F') goto FOUND_FLOAT 1460 b.eq 3f 1461 add \arg_offset, \arg_offset, #4 1462 // Handle extra argument in arg array taken by a long. 1463 cmp wip, #74 // if (wip != 'J') goto LOOP 1464 b.ne 1b 1465 add \arg_offset, \arg_offset, #4 1466 b 1b // goto LOOP 14672: // FOUND_DOUBLE 1468 str \dreg, [\fp, \arg_offset] 1469 add \arg_offset, \arg_offset, #8 1470 b 4f 14713: // FOUND_FLOAT 1472 str \sreg, [\fp, \arg_offset] 1473 add \arg_offset, \arg_offset, #4 14744: 1475.endm 1476 1477// Puts the next floating point parameter passed in stack 1478// in the expected dex register array entry. 1479// Uses ip as temporary. 1480// 1481// TODO: Or we could just spill regs to the reserved slots in the caller's 1482// frame and copy all regs in a simple loop. This time, however, we would 1483// need to look at the shorty anyway to look for the references. 1484// (The trade-off is different for passing arguments and receiving them.) 1485.macro LOOP_OVER_FPs shorty, arg_offset, regs, stack_ptr, finished 14861: // LOOP 1487 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1488 cbz wip, \finished // if (wip == '\0') goto finished 1489 cmp wip, #68 // if (wip == 'D') goto FOUND_DOUBLE 1490 b.eq 2f 1491 cmp wip, #70 // if (wip == 'F') goto FOUND_FLOAT 1492 b.eq 3f 1493 add \arg_offset, \arg_offset, #4 1494 // Handle extra argument in arg array taken by a long. 1495 cmp wip, #74 // if (wip != 'J') goto LOOP 1496 b.ne 1b 1497 add \arg_offset, \arg_offset, #4 1498 b 1b // goto LOOP 14992: // FOUND_DOUBLE 1500 add ip, \stack_ptr, \arg_offset 1501 ldr ip, [ip, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK] 1502 str ip, [\regs, \arg_offset] 1503 add \arg_offset, \arg_offset, #8 1504 b 1b 15053: // FOUND_FLOAT 1506 add ip, \stack_ptr, \arg_offset 1507 ldr wip, [ip, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK] 1508 str wip, [\regs, \arg_offset] 1509 add \arg_offset, \arg_offset, #4 1510 b 1b 1511.endm 1512 1513// Puts the next int/long/object parameter passed in stack 1514// in the expected dex register array entry, and in case of object in the 1515// expected reference array entry. 1516// Uses ip and ip2 as temporary. 1517.macro LOOP_OVER_INTs shorty, arg_offset, regs, refs, stack_ptr, finished 15181: // LOOP 1519 ldrb wip, [\shorty], #1 // Load next character in shorty, and increment. 1520 cbz wip, \finished // if (wip == '\0') goto finished 1521 cmp wip, #74 // if (wip == 'J') goto FOUND_LONG 1522 b.eq 2f 1523 cmp wip, #70 // if (wip == 'F') goto SKIP_FLOAT 1524 b.eq 3f 1525 cmp wip, #68 // if (wip == 'D') goto SKIP_DOUBLE 1526 b.eq 4f 1527 add ip2, \stack_ptr, \arg_offset 1528 ldr wip2, [ip2, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK] 1529 str wip2, [\regs, \arg_offset] 1530 cmp wip, #76 // if (wip != 'L') goto loop 1531 b.ne 3f 1532 str wip2, [\refs, \arg_offset] 1533 add \arg_offset, \arg_offset, #4 1534 b 1b 15352: // FOUND_LONG 1536 add ip, \stack_ptr, \arg_offset 1537 ldr ip, [ip, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK] 1538 str ip, [\regs, \arg_offset] 1539 add \arg_offset, \arg_offset, #8 1540 b 1b 15413: // SKIP_FLOAT 1542 add \arg_offset, \arg_offset, #4 1543 b 1b 15444: // SKIP_DOUBLE 1545 add \arg_offset, \arg_offset, #8 1546 b 1b 1547.endm 1548 1549.macro SETUP_REFERENCE_PARAMETER_IN_GPR gpr32, regs, refs, ins, arg_offset, finished 1550 str \gpr32, [\regs, \arg_offset] 1551 sub \ins, \ins, #1 1552 str \gpr32, [\refs, \arg_offset] 1553 add \arg_offset, \arg_offset, #4 1554 cbz \ins, \finished 1555.endm 1556 1557// Uses ip2 as temporary. 1558.macro SETUP_REFERENCE_PARAMETERS_IN_STACK regs, refs, ins, stack_ptr, arg_offset 15591: 1560 ldr wip2, [\stack_ptr, \arg_offset] 1561 sub \ins, \ins, #1 1562 str wip2, [\regs, \arg_offset] 1563 str wip2, [\refs, \arg_offset] 1564 add \arg_offset, \arg_offset, #4 1565 cbnz \ins, 1b 1566.endm 1567 1568.macro CHECK_AND_UPDATE_SHARED_MEMORY_METHOD if_hot, if_not_hot 1569 ldr wip, [x0, #ART_METHOD_ACCESS_FLAGS_OFFSET] 1570 tbz wip, #ART_METHOD_IS_MEMORY_SHARED_FLAG_BIT, \if_hot 1571 ldr wip, [xSELF, #THREAD_SHARED_METHOD_HOTNESS_OFFSET] 1572 cbz wip, \if_hot 1573 add wip, wip, #-1 1574 str wip, [xSELF, #THREAD_SHARED_METHOD_HOTNESS_OFFSET] 1575 b \if_not_hot 1576.endm 1577 1578.macro DO_SUSPEND_CHECK continue_label 1579 ldr wip, [xSELF, #THREAD_FLAGS_OFFSET] 1580 tst wip, #THREAD_SUSPEND_OR_CHECKPOINT_REQUEST 1581 b.eq \continue_label 1582 EXPORT_PC 1583 bl art_quick_test_suspend 1584.endm 1585 1586%def entry(): 1587/* 1588 * ArtMethod entry point. 1589 * 1590 * On entry: 1591 * x0 ArtMethod* callee 1592 * rest method parameters 1593 */ 1594 1595OAT_ENTRY ExecuteNterpWithClinitImpl, EndExecuteNterpWithClinitImpl 1596 // For simplicity, we don't do a read barrier here, but instead rely 1597 // on art_quick_resolution_trampoline to always have a suspend point before 1598 // calling back here. 1599 ldr wip, [x0, #ART_METHOD_DECLARING_CLASS_OFFSET] 1600 ldrb wip2, [ip, #MIRROR_CLASS_IS_VISIBLY_INITIALIZED_OFFSET] 1601 cmp ip2, #MIRROR_CLASS_IS_VISIBLY_INITIALIZED_VALUE 1602 b.hs ExecuteNterpImpl 1603 cmp ip2, #MIRROR_CLASS_IS_INITIALIZED_VALUE 1604 b.lo .Linitializing_check 1605 dmb ish 1606 b ExecuteNterpImpl 1607.Linitializing_check: 1608 cmp ip2, #MIRROR_CLASS_IS_INITIALIZING_VALUE 1609 b.lo .Lresolution_trampoline 1610 ldr wip2, [ip, #MIRROR_CLASS_CLINIT_THREAD_ID_OFFSET] 1611 ldr wip, [xSELF, #THREAD_TID_OFFSET] 1612 cmp wip, wip2 1613 b.eq ExecuteNterpImpl 1614.Lresolution_trampoline: 1615 b art_quick_resolution_trampoline 1616EndExecuteNterpWithClinitImpl: 1617 1618OAT_ENTRY ExecuteNterpImpl, EndExecuteNterpImpl 1619 .cfi_startproc 1620 sub x16, sp, #STACK_OVERFLOW_RESERVED_BYTES 1621 ldr wzr, [x16] 1622 /* Spill callee save regs */ 1623 SPILL_ALL_CALLEE_SAVES 1624 1625 ldr xPC, [x0, #ART_METHOD_DATA_OFFSET_64] 1626 // Setup the stack for executing the method. 1627 SETUP_STACK_FRAME xPC, xREFS, xFP, CFI_REFS, load_ins=1 1628 1629 // Setup the parameters 1630 cbz w15, .Lxmm_setup_finished 1631 1632 sub ip2, ip, x15 1633 ldr w26, [x0, #ART_METHOD_ACCESS_FLAGS_OFFSET] 1634 lsl x27, ip2, #2 // x27 is now the offset for inputs into the registers array. 1635 1636 tbz w26, #ART_METHOD_NTERP_ENTRY_POINT_FAST_PATH_FLAG_BIT, .Lsetup_slow_path 1637 // Setup pointer to inputs in FP and pointer to inputs in REFS 1638 add x10, xFP, x27 1639 add x11, xREFS, x27 1640 mov x12, #0 1641 SETUP_REFERENCE_PARAMETER_IN_GPR w1, x10, x11, w15, x12, .Lxmm_setup_finished 1642 SETUP_REFERENCE_PARAMETER_IN_GPR w2, x10, x11, w15, x12, .Lxmm_setup_finished 1643 SETUP_REFERENCE_PARAMETER_IN_GPR w3, x10, x11, w15, x12, .Lxmm_setup_finished 1644 SETUP_REFERENCE_PARAMETER_IN_GPR w4, x10, x11, w15, x12, .Lxmm_setup_finished 1645 SETUP_REFERENCE_PARAMETER_IN_GPR w5, x10, x11, w15, x12, .Lxmm_setup_finished 1646 SETUP_REFERENCE_PARAMETER_IN_GPR w6, x10, x11, w15, x12, .Lxmm_setup_finished 1647 SETUP_REFERENCE_PARAMETER_IN_GPR w7, x10, x11, w15, x12, .Lxmm_setup_finished 1648 add x28, x28, #OFFSET_TO_FIRST_ARGUMENT_IN_STACK 1649 SETUP_REFERENCE_PARAMETERS_IN_STACK x10, x11, w15, x28, x12 1650 b .Lxmm_setup_finished 1651 1652.Lsetup_slow_path: 1653 // If the method is not static and there is one argument ('this'), we don't need to fetch the 1654 // shorty. 1655 tbnz w26, #ART_METHOD_IS_STATIC_FLAG_BIT, .Lsetup_with_shorty 1656 str w1, [xFP, x27] 1657 str w1, [xREFS, x27] 1658 cmp w15, #1 1659 b.eq .Lxmm_setup_finished 1660 1661.Lsetup_with_shorty: 1662 // TODO: Get shorty in a better way and remove below 1663 SPILL_ALL_ARGUMENTS 1664 bl NterpGetShorty 1665 // Save shorty in callee-save xIBASE. 1666 mov xIBASE, x0 1667 RESTORE_ALL_ARGUMENTS 1668 1669 // Setup pointer to inputs in FP and pointer to inputs in REFS 1670 add x10, xFP, x27 1671 add x11, xREFS, x27 1672 mov x12, #0 1673 1674 add x9, xIBASE, #1 // shorty + 1 ; ie skip return arg character 1675 tbnz w26, #ART_METHOD_IS_STATIC_FLAG_BIT, .Lhandle_static_method 1676 add x10, x10, #4 1677 add x11, x11, #4 1678 add x28, x28, #4 1679 b .Lcontinue_setup_gprs 1680.Lhandle_static_method: 1681 LOOP_OVER_SHORTY_STORING_GPRS x1, w1, x9, x12, x10, x11, .Lgpr_setup_finished 1682.Lcontinue_setup_gprs: 1683 LOOP_OVER_SHORTY_STORING_GPRS x2, w2, x9, x12, x10, x11, .Lgpr_setup_finished 1684 LOOP_OVER_SHORTY_STORING_GPRS x3, w3, x9, x12, x10, x11, .Lgpr_setup_finished 1685 LOOP_OVER_SHORTY_STORING_GPRS x4, w4, x9, x12, x10, x11, .Lgpr_setup_finished 1686 LOOP_OVER_SHORTY_STORING_GPRS x5, w5, x9, x12, x10, x11, .Lgpr_setup_finished 1687 LOOP_OVER_SHORTY_STORING_GPRS x6, w6, x9, x12, x10, x11, .Lgpr_setup_finished 1688 LOOP_OVER_SHORTY_STORING_GPRS x7, w7, x9, x12, x10, x11, .Lgpr_setup_finished 1689 LOOP_OVER_INTs x9, x12, x10, x11, x28, .Lgpr_setup_finished 1690.Lgpr_setup_finished: 1691 add x9, xIBASE, #1 // shorty + 1 ; ie skip return arg character 1692 mov x12, #0 // reset counter 1693 LOOP_OVER_SHORTY_STORING_FPS d0, s0, x9, x12, x10, .Lxmm_setup_finished 1694 LOOP_OVER_SHORTY_STORING_FPS d1, s1, x9, x12, x10, .Lxmm_setup_finished 1695 LOOP_OVER_SHORTY_STORING_FPS d2, s2, x9, x12, x10, .Lxmm_setup_finished 1696 LOOP_OVER_SHORTY_STORING_FPS d3, s3, x9, x12, x10, .Lxmm_setup_finished 1697 LOOP_OVER_SHORTY_STORING_FPS d4, s4, x9, x12, x10, .Lxmm_setup_finished 1698 LOOP_OVER_SHORTY_STORING_FPS d5, s5, x9, x12, x10, .Lxmm_setup_finished 1699 LOOP_OVER_SHORTY_STORING_FPS d6, s6, x9, x12, x10, .Lxmm_setup_finished 1700 LOOP_OVER_SHORTY_STORING_FPS d7, s7, x9, x12, x10, .Lxmm_setup_finished 1701 LOOP_OVER_FPs x9, x12, x10, x28, .Lxmm_setup_finished 1702.Lxmm_setup_finished: 1703 CFI_DEFINE_DEX_PC_WITH_OFFSET(CFI_TMP, CFI_DEX, 0) 1704 1705 // Set rIBASE 1706 adr xIBASE, artNterpAsmInstructionStart 1707 /* start executing the instruction at xPC */ 1708 START_EXECUTING_INSTRUCTIONS 1709 /* NOTE: no fallthrough */ 1710 // cfi info continues, and covers the whole nterp implementation. 1711 SIZE ExecuteNterpImpl 1712 1713%def opcode_pre(): 1714 1715%def fetch_from_thread_cache(dest_reg, miss_label): 1716 // Fetch some information from the thread cache. 1717 // Uses ip and ip2 as temporaries. 1718 add ip, xSELF, #THREAD_INTERPRETER_CACHE_OFFSET // cache address 1719 ubfx ip2, xPC, #2, #THREAD_INTERPRETER_CACHE_SIZE_LOG2 // entry index 1720 add ip, ip, ip2, lsl #4 // entry address within the cache 1721 ldp ip, ${dest_reg}, [ip] // entry key (pc) and value (offset) 1722 cmp ip, xPC 1723 b.ne ${miss_label} 1724 1725%def footer(): 1726/* 1727 * =========================================================================== 1728 * Common subroutines and data 1729 * =========================================================================== 1730 */ 1731 1732 .text 1733 .align 2 1734 1735// Enclose all code below in a symbol (which gets printed in backtraces). 1736NAME_START nterp_helper 1737 1738// Note: mterp also uses the common_* names below for helpers, but that's OK 1739// as the assembler compiled each interpreter separately. 1740common_errDivideByZero: 1741 EXPORT_PC 1742 bl art_quick_throw_div_zero 1743 1744// Expect index in w1, length in w3. 1745common_errArrayIndex: 1746 EXPORT_PC 1747 mov x0, x1 1748 mov x1, x3 1749 bl art_quick_throw_array_bounds 1750 1751common_errNullObject: 1752 EXPORT_PC 1753 bl art_quick_throw_null_pointer_exception 1754 1755NterpCommonInvokeStatic: 1756 COMMON_INVOKE_NON_RANGE is_static=1, suffix="invokeStatic" 1757 1758NterpCommonInvokeStaticRange: 1759 COMMON_INVOKE_RANGE is_static=1, suffix="invokeStatic" 1760 1761NterpCommonInvokeInstance: 1762 COMMON_INVOKE_NON_RANGE suffix="invokeInstance" 1763 1764NterpCommonInvokeInstanceRange: 1765 COMMON_INVOKE_RANGE suffix="invokeInstance" 1766 1767NterpCommonInvokeInterface: 1768 COMMON_INVOKE_NON_RANGE is_interface=1, suffix="invokeInterface" 1769 1770NterpCommonInvokeInterfaceRange: 1771 COMMON_INVOKE_RANGE is_interface=1, suffix="invokeInterface" 1772 1773NterpCommonInvokePolymorphic: 1774 COMMON_INVOKE_NON_RANGE is_polymorphic=1, suffix="invokePolymorphic" 1775 1776NterpCommonInvokePolymorphicRange: 1777 COMMON_INVOKE_RANGE is_polymorphic=1, suffix="invokePolymorphic" 1778 1779NterpCommonInvokeCustom: 1780 COMMON_INVOKE_NON_RANGE is_static=1, is_custom=1, suffix="invokeCustom" 1781 1782NterpCommonInvokeCustomRange: 1783 COMMON_INVOKE_RANGE is_static=1, is_custom=1, suffix="invokeCustom" 1784 1785NterpHandleStringInit: 1786 COMMON_INVOKE_NON_RANGE is_string_init=1, suffix="stringInit" 1787 1788NterpHandleStringInitRange: 1789 COMMON_INVOKE_RANGE is_string_init=1, suffix="stringInit" 1790 1791NterpHandleHotnessOverflow: 1792 CHECK_AND_UPDATE_SHARED_MEMORY_METHOD if_hot=1f, if_not_hot=5f 17931: 1794 mov x1, xPC 1795 mov x2, xFP 1796 bl nterp_hot_method 1797 cbnz x0, 3f 17982: 1799 FETCH wINST, 0 // load wINST 1800 GET_INST_OPCODE ip // extract opcode from wINST 1801 GOTO_OPCODE ip // jump to next instruction 18023: 1803 // Drop the current frame. 1804 ldr ip, [xREFS, #-8] 1805 mov sp, ip 1806 .cfi_def_cfa sp, CALLEE_SAVES_SIZE 1807 1808 // The transition frame of type SaveAllCalleeSaves saves x19 and x20, 1809 // but not managed ABI. So we need to restore callee-saves of the nterp frame, 1810 // and save managed ABI callee saves, which will be restored by the callee upon 1811 // return. 1812 RESTORE_ALL_CALLEE_SAVES 1813 INCREASE_FRAME ((CALLEE_SAVES_SIZE) - 16) 1814 1815 // FP callee-saves 1816 stp d8, d9, [sp, #0] 1817 stp d10, d11, [sp, #16] 1818 stp d12, d13, [sp, #32] 1819 stp d14, d15, [sp, #48] 1820 1821 // GP callee-saves. 1822 SAVE_TWO_REGS x21, x22, 64 1823 SAVE_TWO_REGS x23, x24, 80 1824 SAVE_TWO_REGS x25, x26, 96 1825 SAVE_TWO_REGS x27, x28, 112 1826 SAVE_TWO_REGS x29, lr, 128 1827 1828 // Setup the new frame 1829 ldr x1, [x0, #OSR_DATA_FRAME_SIZE] 1830 // Given stack size contains all callee saved registers, remove them. 1831 sub x1, x1, #(CALLEE_SAVES_SIZE - 16) 1832 1833 // We know x1 cannot be 0, as it at least contains the ArtMethod. 1834 1835 // Remember CFA in a callee-save register. 1836 mov xINST, sp 1837 .cfi_def_cfa_register xINST 1838 1839 sub sp, sp, x1 1840 1841 add x2, x0, #OSR_DATA_MEMORY 18424: 1843 sub x1, x1, #8 1844 ldr ip, [x2, x1] 1845 str ip, [sp, x1] 1846 cbnz x1, 4b 1847 1848 // Fetch the native PC to jump to and save it in a callee-save register. 1849 ldr xFP, [x0, #OSR_DATA_NATIVE_PC] 1850 1851 // Free the memory holding OSR Data. 1852 bl free 1853 1854 // Jump to the compiled code. 1855 br xFP 18565: 1857 DO_SUSPEND_CHECK continue_label=2b 1858 b 2b 1859 1860// This is the logical end of ExecuteNterpImpl, where the frame info applies. 1861// EndExecuteNterpImpl includes the methods below as we want the runtime to 1862// see them as part of the Nterp PCs. 1863.cfi_endproc 1864 1865nterp_to_nterp_static_non_range: 1866 .cfi_startproc 1867 SETUP_STACK_FOR_INVOKE 1868 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=1, is_string_init=0 1869 .cfi_endproc 1870 1871nterp_to_nterp_string_init_non_range: 1872 .cfi_startproc 1873 SETUP_STACK_FOR_INVOKE 1874 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 1875 .cfi_endproc 1876 1877nterp_to_nterp_instance_non_range: 1878 .cfi_startproc 1879 SETUP_STACK_FOR_INVOKE 1880 SETUP_NON_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=0 1881 .cfi_endproc 1882 1883nterp_to_nterp_static_range: 1884 .cfi_startproc 1885 SETUP_STACK_FOR_INVOKE 1886 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=1 1887 .cfi_endproc 1888 1889nterp_to_nterp_instance_range: 1890 .cfi_startproc 1891 SETUP_STACK_FOR_INVOKE 1892 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0 1893 .cfi_endproc 1894 1895nterp_to_nterp_string_init_range: 1896 .cfi_startproc 1897 SETUP_STACK_FOR_INVOKE 1898 SETUP_RANGE_ARGUMENTS_AND_EXECUTE is_static=0, is_string_init=1 1899 .cfi_endproc 1900 1901NAME_END nterp_helper 1902 1903// This is the end of PCs contained by the OatQuickMethodHeader created for the interpreter 1904// entry point. 1905 .type EndExecuteNterpImpl, #function 1906 .hidden EndExecuteNterpImpl 1907 .global EndExecuteNterpImpl 1908EndExecuteNterpImpl: 1909 1910// Entrypoints into runtime. 1911NTERP_TRAMPOLINE nterp_get_static_field, NterpGetStaticField 1912NTERP_TRAMPOLINE nterp_get_instance_field_offset, NterpGetInstanceFieldOffset 1913NTERP_TRAMPOLINE nterp_filled_new_array, NterpFilledNewArray 1914NTERP_TRAMPOLINE nterp_filled_new_array_range, NterpFilledNewArrayRange 1915NTERP_TRAMPOLINE nterp_get_class, NterpGetClass 1916NTERP_TRAMPOLINE nterp_allocate_object, NterpAllocateObject 1917NTERP_TRAMPOLINE nterp_get_method, NterpGetMethod 1918NTERP_TRAMPOLINE nterp_hot_method, NterpHotMethod 1919NTERP_TRAMPOLINE nterp_load_object, NterpLoadObject 1920 1921ENTRY nterp_deliver_pending_exception 1922 DELIVER_PENDING_EXCEPTION 1923END nterp_deliver_pending_exception 1924 1925// gen_mterp.py will inline the following definitions 1926// within [ExecuteNterpImpl, EndExecuteNterpImpl). 1927%def instruction_end(): 1928 1929 .type artNterpAsmInstructionEnd, #function 1930 .hidden artNterpAsmInstructionEnd 1931 .global artNterpAsmInstructionEnd 1932artNterpAsmInstructionEnd: 1933 // artNterpAsmInstructionEnd is used as landing pad for exception handling. 1934 FETCH_INST 1935 GET_INST_OPCODE ip 1936 GOTO_OPCODE ip 1937 1938%def instruction_start(): 1939 1940 .type artNterpAsmInstructionStart, #function 1941 .hidden artNterpAsmInstructionStart 1942 .global artNterpAsmInstructionStart 1943artNterpAsmInstructionStart = .L_op_nop 1944 .text 1945 1946%def opcode_name_prefix(): 1947% return "nterp_" 1948%def opcode_start(): 1949 NAME_START nterp_${opcode} 1950 # Explicitly restore CFA, just in case the previous opcode clobbered it (by .cfi_def_*). 1951 CFI_DEF_CFA_BREG_PLUS_UCONST CFI_REFS, -8, CALLEE_SAVES_SIZE 1952%def opcode_end(): 1953 NAME_END nterp_${opcode} 1954 // Advance to the end of this handler. Causes error if we are past that point. 1955 .org nterp_${opcode} + NTERP_HANDLER_SIZE // ${opcode} handler is too big! 1956%def opcode_slow_path_start(name): 1957 NAME_START ${name} 1958%def opcode_slow_path_end(name): 1959 NAME_END ${name} 1960