1/** 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16#include "arch/aarch64/helpers_aarch64.S" 17#include "arch/asm_support.h" 18 19#define REGS_COUNT 30 20#define FP_REGS_COUNT 32 21#define REGS_BUFFER_SIZE (REGS_COUNT * 8) 22#define FP_REGS_BUFFER_SIZE (FP_REGS_COUNT * 8) 23 24.extern ConvertIFrameToCFrame 25.extern PrepareOsrEntry 26.extern SetResult 27.extern GetCurrentManagedThread 28 29.macro OSR_ENTRY 30 // Prepare buffer for scalar registers 31 sub sp, sp, REGS_BUFFER_SIZE 32 mov x4, sp 33 // Prepare buffer for vector registers 34 sub sp, sp, FP_REGS_BUFFER_SIZE 35 mov x5, sp 36 37 // x0 - pointer to iframe 38 // x1 - bytecode offset 39 // x2 - osr code pointer 40 // x3 - pointer to cframe 41 // x4 - buffer for scalar registers 42 // x5 - buffer for vector registers 43 bl PrepareOsrEntry 44 mov x17, x0 45 46 mov x16, sp 47 stp x16, x17, [sp, #-16]! 48 49 bl GetCurrentManagedThread 50 mov THREAD_REG, x0 51 52 ldp x16, x17, [sp], #16 53 54 // Restore registers from buffers prepared by PrepareOsrEntry 55 ldp d0, d1, [x16], #16 56 ldp d2, d3, [x16], #16 57 ldp d4, d5, [x16], #16 58 ldp d6, d7, [x16], #16 59 ldp d8, d9, [x16], #16 60 ldp d10, d11, [x16], #16 61 ldp d12, d13, [x16], #16 62 ldp d14, d15, [x16], #16 63 ldp d16, d17, [x16], #16 64 ldp d18, d19, [x16], #16 65 ldp d20, d21, [x16], #16 66 ldp d22, d23, [x16], #16 67 ldp d24, d25, [x16], #16 68 ldp d26, d27, [x16], #16 69 ldp d28, d29, [x16], #16 70 ldp d30, d31, [x16], #16 71 72 ldp x0, x1, [x16], #16 73 ldp x2, x3, [x16], #16 74 ldp x4, x5, [x16], #16 75 ldp x6, x7, [x16], #16 76 ldp x8, x9, [x16], #16 77 ldp x10, x11, [x16], #16 78 ldp x12, x13, [x16], #16 79 ldp x14, x15, [x16], #16 80 add x16, x16 ,16 81 ldp x18, x19, [x16], #16 82 ldp x20, x21, [x16], #16 83 ldp x22, x23, [x16], #16 84 ldp x24, x25, [x16], #16 85 ldp x26, x27, [x16], #16 86 // X28 is a THREAD_REG, so skip it 87 add sp, sp, (REGS_BUFFER_SIZE + FP_REGS_BUFFER_SIZE) 88 89 blr x17 90.endm 91 92.global OsrEntryAfterCFrame 93.type OsrEntryAfterCFrame, %function 94OsrEntryAfterCFrame: 95 CFI_STARTPROC 96 CFI_DEF_CFA(sp, 0) 97#ifndef NDEBUG 98 stp fp, lr, [sp, #-16]! 99 CFI_ADJUST_CFA_OFFSET(2 * 8) 100 CFI_REL_OFFSET(lr, 8) 101 CFI_REL_OFFSET(fp, 0) 102#endif 103 104 // x0 - pointer to Frame 105 // x1 - bytecode offset 106 // x2 - pointer to compiled code 107 // x3 - frame size 108 109 // We're going to replace C2I frame by new cframe 110 ldr x16, [x0, #FRAME_PREV_FRAME_OFFSET] 111 add x18, x16, #24 112 113 // We will return to the caller cframe skipping interpreter and C2I bridge 114 ldr fp, [x16] 115 ldr lr, [x16, #16] 116 117 // Copy callee saved registers from boundary frame into cframe. 118 // Destination - callee saved stack in the cframe 119 sub x17, x18, #((CFRAME_HEADER_SIZE + CFRAME_LOCALS_COUNT) * 8 + CALLEE_SAVED_SIZE) 120 // Source - callee saved stack in the bounadary frame 121 sub x16, x18, #((CFRAME_HEADER_SIZE * 8) + CALLEE_SAVED_SIZE) 122 123 ldp x14, x15, [x16], #16 124 stp x14, x15, [x17], #16 125 ldp x14, x15, [x16], #16 126 stp x14, x15, [x17], #16 127 ldp x14, x15, [x16], #16 128 stp x14, x15, [x17], #16 129 ldp x14, x15, [x16], #16 130 stp x14, x15, [x17], #16 131 ldp x14, x15, [x16], #16 132 stp x14, x15, [x17], #16 133 ldp x14, x15, [x16], #16 134 stp x14, x15, [x17], #16 135 ldp x14, x15, [x16], #16 136 stp x14, x15, [x17], #16 137 ldp x14, x15, [x16], #16 138 stp x14, x15, [x17], #16 139 ldp x14, x15, [x16], #16 140 stp x14, x15, [x17], #16 141 142 // Create CFrame 143 stp fp, lr, [x18, #-16] 144 ldr x16, [x0, #FRAME_METHOD_OFFSET] 145 str x16, [x18, #-24] 146 147 mov x4, x3 148 // Set x3 to created cframe 149 sub x3, x18, #16 150 mov fp, x3 151 152 CFI_DEF_CFA(fp, (2 * 8)) 153 CFI_REL_OFFSET(lr, 8) 154 CFI_REL_OFFSET(fp, 0) 155 CFI_REL_OFFSET(THREAD_REG, -((CFRAME_CALLEE_REGS_START_SLOT + 0) * 8)) 156 CFI_REL_OFFSET(x27, -((CFRAME_CALLEE_REGS_START_SLOT + 1) * 8)) 157 CFI_REL_OFFSET(x26, -((CFRAME_CALLEE_REGS_START_SLOT + 2) * 8)) 158 CFI_REL_OFFSET(x25, -((CFRAME_CALLEE_REGS_START_SLOT + 3) * 8)) 159 CFI_REL_OFFSET(x24, -((CFRAME_CALLEE_REGS_START_SLOT + 4) * 8)) 160 CFI_REL_OFFSET(x23, -((CFRAME_CALLEE_REGS_START_SLOT + 5) * 8)) 161 CFI_REL_OFFSET(x22, -((CFRAME_CALLEE_REGS_START_SLOT + 6) * 8)) 162 CFI_REL_OFFSET(x21, -((CFRAME_CALLEE_REGS_START_SLOT + 7) * 8)) 163 CFI_REL_OFFSET(x20, -((CFRAME_CALLEE_REGS_START_SLOT + 8) * 8)) 164 CFI_REL_OFFSET(x19, -((CFRAME_CALLEE_REGS_START_SLOT + 9) * 8)) 165 CFI_REL_OFFSET(d15, -((CFRAME_CALLEE_REGS_START_SLOT + 10) * 8)) 166 CFI_REL_OFFSET(d14, -((CFRAME_CALLEE_REGS_START_SLOT + 11) * 8)) 167 CFI_REL_OFFSET(d13, -((CFRAME_CALLEE_REGS_START_SLOT + 12) * 8)) 168 CFI_REL_OFFSET(d12, -((CFRAME_CALLEE_REGS_START_SLOT + 13) * 8)) 169 CFI_REL_OFFSET(d11, -((CFRAME_CALLEE_REGS_START_SLOT + 14) * 8)) 170 CFI_REL_OFFSET(d10, -((CFRAME_CALLEE_REGS_START_SLOT + 15) * 8)) 171 CFI_REL_OFFSET(d9, -((CFRAME_CALLEE_REGS_START_SLOT + 16) * 8)) 172 CFI_REL_OFFSET(d8, -((CFRAME_CALLEE_REGS_START_SLOT + 17) * 8)) 173 174 // Adjust stack for the whole cframe 175 sub sp, x18, x4 176 177 OSR_ENTRY 178 179 // We need to presrve lr/fp stored in the cframe, since GetCurrentManagedThread will modify stack memory. 180 sub sp, sp, #16 181 182 // Restore THREAD REG 183 str x0, [sp, #-16]! 184 bl GetCurrentManagedThread 185 mov THREAD_REG, x0 186 ldr x0, [sp], #16 187 188 // Load lr/fp and restore sp 189 ldp fp, lr, [sp], #16 190 CFI_RESTORE(lr) 191 CFI_RESTORE(fp) 192 CFI_DEF_CFA(sp, 0) 193 ret 194 CFI_ENDPROC 195 196.global OsrEntryAfterIFrame 197.type OsrEntryAfterIFrame, %function 198OsrEntryAfterIFrame: 199 CFI_STARTPROC 200 CFI_DEF_CFA(sp, 0) 201 202 // x0 - pointer to Frame 203 // x1 - bytecode offset 204 // x2 - pointer to compiled code 205 // x3 - frame size 206 207 // Save original lr and fp before cframe 208 stp fp, lr, [sp, #-16]! 209 CFI_ADJUST_CFA_OFFSET(16) 210 CFI_REL_OFFSET(lr, 8) 211 CFI_REL_OFFSET(fp, 0) 212 stp x0, xzr, [sp, #-16]! 213 CFI_ADJUST_CFA_OFFSET(16) 214 215 // Create I2C bridge 216 ldr x16, [x0, #FRAME_PREV_FRAME_OFFSET] 217 stp x16, lr, [sp, #-16]! 218 CFI_ADJUST_CFA_OFFSET(16) 219 mov x16, #INTERPRETER_TO_COMPILED_CODE_BRIDGE 220 sub sp, sp, #16 221 CFI_ADJUST_CFA_OFFSET(16) 222 str x16, [sp, #8] 223 224 // Create CFrame 225 add x16, sp, #16 226 stp x16, lr, [sp, #-16] 227 ldr x16, [x0, #FRAME_METHOD_OFFSET] 228 str x16, [sp, #-24] 229 230 // Push callee saved registers into cframe. They will be restored by epilogue of the compiled code. 231 sub x16, sp, #((CFRAME_HEADER_SIZE + CFRAME_LOCALS_COUNT) * 8) 232 PUSH_CALLEE_REGS x16 233 234 mov x4, x3 235 // Set x3 to created cframe 236 sub x3, sp, #16 237 mov fp, x3 238 239 CFI_DEF_CFA_REGISTER(fp) 240 CFI_ADJUST_CFA_OFFSET(16) 241 CFI_REL_OFFSET(x28, -((CFRAME_CALLEE_REGS_START_SLOT + 0) * 8)) 242 CFI_REL_OFFSET(x27, -((CFRAME_CALLEE_REGS_START_SLOT + 1) * 8)) 243 CFI_REL_OFFSET(x26, -((CFRAME_CALLEE_REGS_START_SLOT + 2) * 8)) 244 CFI_REL_OFFSET(x25, -((CFRAME_CALLEE_REGS_START_SLOT + 3) * 8)) 245 CFI_REL_OFFSET(x24, -((CFRAME_CALLEE_REGS_START_SLOT + 4) * 8)) 246 CFI_REL_OFFSET(x23, -((CFRAME_CALLEE_REGS_START_SLOT + 5) * 8)) 247 CFI_REL_OFFSET(x22, -((CFRAME_CALLEE_REGS_START_SLOT + 6) * 8)) 248 CFI_REL_OFFSET(x21, -((CFRAME_CALLEE_REGS_START_SLOT + 7) * 8)) 249 CFI_REL_OFFSET(x20, -((CFRAME_CALLEE_REGS_START_SLOT + 8) * 8)) 250 CFI_REL_OFFSET(x19, -((CFRAME_CALLEE_REGS_START_SLOT + 9) * 8)) 251 CFI_REL_OFFSET(d15, -((CFRAME_CALLEE_REGS_START_SLOT + 10) * 8)) 252 CFI_REL_OFFSET(d14, -((CFRAME_CALLEE_REGS_START_SLOT + 11) * 8)) 253 CFI_REL_OFFSET(d13, -((CFRAME_CALLEE_REGS_START_SLOT + 12) * 8)) 254 CFI_REL_OFFSET(d12, -((CFRAME_CALLEE_REGS_START_SLOT + 13) * 8)) 255 CFI_REL_OFFSET(d11, -((CFRAME_CALLEE_REGS_START_SLOT + 14) * 8)) 256 CFI_REL_OFFSET(d10, -((CFRAME_CALLEE_REGS_START_SLOT + 15) * 8)) 257 CFI_REL_OFFSET(d9, -((CFRAME_CALLEE_REGS_START_SLOT + 16) * 8)) 258 CFI_REL_OFFSET(d8, -((CFRAME_CALLEE_REGS_START_SLOT + 17) * 8)) 259 260 // Adjust stack for the whole cframe 261 sub sp, sp, x4 262 263 OSR_ENTRY 264 265 // Set frame kind in the TLS 266 stp x0, xzr, [sp, #-16]! 267 bl GetCurrentManagedThread 268 strb wzr, [x0, #MANAGED_THREAD_FRAME_KIND_OFFSET] 269 ldp x0, xzr, [sp], #16 270 271 // Remove I2C bridge 272 add sp, sp, #32 273 274 // Restore interpreter frame 275 ldp x16, xzr, [sp], #16 276 277 // Call SetOsrResult for writing the result to the interpreter frame's accumulator 278 mov x1, x0 279 mov x0, x16 280 bl SetOsrResult 281 282 // Restore original lr and fp 283 ldp fp, lr, [sp], #16 284 CFI_RESTORE(lr) 285 CFI_RESTORE(fp) 286 CFI_DEF_CFA(sp, 0) 287 ret 288 CFI_ENDPROC 289 290.global OsrEntryTopFrame 291.type OsrEntryTopFrame, %function 292OsrEntryTopFrame: 293 CFI_STARTPROC 294 CFI_DEF_CFA(sp, 0) 295 296 // x0 - pointer to Frame 297 // x1 - bytecode offset 298 // x2 - pointer to compiled code 299 // x3 - frame size 300 301 // Save original lr and fp before cframe 302 stp fp, lr, [sp, #-16]! 303 CFI_ADJUST_CFA_OFFSET(16) 304 CFI_REL_OFFSET(lr, 8) 305 CFI_REL_OFFSET(fp, 0) 306 // Save pointer to IFrame, we'll need it later 307 stp x0, xzr, [sp, #-16]! 308 CFI_ADJUST_CFA_OFFSET(16) 309 310 // Save lr and fp, fp is null because it is the top frame 311 str lr, [sp, #-8] 312 str xzr, [sp, #-16] 313 314 // Save method 315 ldr x16, [x0, #FRAME_METHOD_OFFSET] 316 str x16, [sp, #-24] 317 318 // Push callee saved registers into cframe. They will be restored by epilogue of the compiled code. 319 sub x4, sp, #((CFRAME_HEADER_SIZE + CFRAME_LOCALS_COUNT) * 8) 320 PUSH_CALLEE_REGS x4 321 322 mov x4, x3 323 // Set x3 to created cframe 324 sub x3, sp, #16 325 mov fp, x3 326 327 CFI_DEF_CFA_REGISTER(fp) 328 CFI_ADJUST_CFA_OFFSET(16) 329 CFI_REL_OFFSET(x28, -((CFRAME_CALLEE_REGS_START_SLOT + 0) * 8)) 330 CFI_REL_OFFSET(x27, -((CFRAME_CALLEE_REGS_START_SLOT + 1) * 8)) 331 CFI_REL_OFFSET(x26, -((CFRAME_CALLEE_REGS_START_SLOT + 2) * 8)) 332 CFI_REL_OFFSET(x25, -((CFRAME_CALLEE_REGS_START_SLOT + 3) * 8)) 333 CFI_REL_OFFSET(x24, -((CFRAME_CALLEE_REGS_START_SLOT + 4) * 8)) 334 CFI_REL_OFFSET(x23, -((CFRAME_CALLEE_REGS_START_SLOT + 5) * 8)) 335 CFI_REL_OFFSET(x22, -((CFRAME_CALLEE_REGS_START_SLOT + 6) * 8)) 336 CFI_REL_OFFSET(x21, -((CFRAME_CALLEE_REGS_START_SLOT + 7) * 8)) 337 CFI_REL_OFFSET(x20, -((CFRAME_CALLEE_REGS_START_SLOT + 8) * 8)) 338 CFI_REL_OFFSET(x19, -((CFRAME_CALLEE_REGS_START_SLOT + 9) * 8)) 339 CFI_REL_OFFSET(d15, -((CFRAME_CALLEE_REGS_START_SLOT + 10) * 8)) 340 CFI_REL_OFFSET(d14, -((CFRAME_CALLEE_REGS_START_SLOT + 11) * 8)) 341 CFI_REL_OFFSET(d13, -((CFRAME_CALLEE_REGS_START_SLOT + 12) * 8)) 342 CFI_REL_OFFSET(d12, -((CFRAME_CALLEE_REGS_START_SLOT + 13) * 8)) 343 CFI_REL_OFFSET(d11, -((CFRAME_CALLEE_REGS_START_SLOT + 14) * 8)) 344 CFI_REL_OFFSET(d10, -((CFRAME_CALLEE_REGS_START_SLOT + 15) * 8)) 345 CFI_REL_OFFSET(d9, -((CFRAME_CALLEE_REGS_START_SLOT + 16) * 8)) 346 CFI_REL_OFFSET(d8, -((CFRAME_CALLEE_REGS_START_SLOT + 17) * 8)) 347 348 // Adjust stack for the whole cframe 349 sub sp, sp, x4 350 351 OSR_ENTRY 352 353 // Set frame kind in the TLS 354 stp x0, xzr, [sp, #-16]! 355 bl GetCurrentManagedThread 356 strb wzr, [x0, #MANAGED_THREAD_FRAME_KIND_OFFSET] 357 ldp x0, xzr, [sp], #16 358 359 // Restore interpreter frame 360 ldp x16, xzr, [sp], #16 361 362 // Call SetOsrResult to write result to the interpreter frame's accumulator 363 mov x1, x0 364 mov x0, x16 365 bl SetOsrResult 366 367 // Restore original lr and fp 368 ldp fp, lr, [sp], #16 369 CFI_RESTORE(lr) 370 CFI_RESTORE(fp) 371 CFI_DEF_CFA(sp, 0) 372 ret 373 CFI_ENDPROC 374