1/* Copyright (c) 2009, 2010, 2011, 2012 ARM Ltd. 2 3Permission is hereby granted, free of charge, to any person obtaining 4a copy of this software and associated documentation files (the 5``Software''), to deal in the Software without restriction, including 6without limitation the rights to use, copy, modify, merge, publish, 7distribute, sublicense, and/or sell copies of the Software, and to 8permit persons to whom the Software is furnished to do so, subject to 9the following conditions: 10 11The above copyright notice and this permission notice shall be 12included in all copies or substantial portions of the Software. 13 14THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, 15EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 17IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 18CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 19TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 20SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ 21 22#if defined(__aarch64__) || defined(__arm64__) 23#define LIBFFI_ASM 24#include <fficonfig.h> 25#include <ffi.h> 26#include <ffi_cfi.h> 27#include "internal.h" 28 29#ifdef HAVE_MACHINE_ASM_H 30#include <machine/asm.h> 31#else 32#ifdef __USER_LABEL_PREFIX__ 33#define CONCAT1(a, b) CONCAT2(a, b) 34#define CONCAT2(a, b) a ## b 35 36/* Use the right prefix for global labels. */ 37#define CNAME(x) CONCAT1 (__USER_LABEL_PREFIX__, x) 38#else 39#define CNAME(x) x 40#endif 41#endif 42 43#ifdef __AARCH64EB__ 44# define BE(X) X 45#else 46# define BE(X) 0 47#endif 48 49#ifdef __ILP32__ 50#define PTR_REG(n) w##n 51#else 52#define PTR_REG(n) x##n 53#endif 54 55#ifdef __ILP32__ 56#define PTR_SIZE 4 57#else 58#define PTR_SIZE 8 59#endif 60 61 .text 62 .align 4 63 64/* ffi_call_SYSV 65 extern void ffi_call_SYSV (void *stack, void *frame, 66 void (*fn)(void), void *rvalue, 67 int flags, void *closure); 68 69 Therefore on entry we have: 70 71 x0 stack 72 x1 frame 73 x2 fn 74 x3 rvalue 75 x4 flags 76 x5 closure 77*/ 78 79 cfi_startproc 80CNAME(ffi_call_SYSV): 81 /* Use a stack frame allocated by our caller. */ 82 cfi_def_cfa(x1, 32); 83 stp x29, x30, [x1] 84 mov x29, x1 85 mov sp, x0 86 cfi_def_cfa_register(x29) 87 cfi_rel_offset (x29, 0) 88 cfi_rel_offset (x30, 8) 89 90 mov x9, x2 /* save fn */ 91 mov x8, x3 /* install structure return */ 92#ifdef FFI_GO_CLOSURES 93 mov x18, x5 /* install static chain */ 94#endif 95 stp x3, x4, [x29, #16] /* save rvalue and flags */ 96 97 /* Load the vector argument passing registers, if necessary. */ 98 tbz w4, #AARCH64_FLAG_ARG_V_BIT, 1f 99 ldp q0, q1, [sp, #0] 100 ldp q2, q3, [sp, #32] 101 ldp q4, q5, [sp, #64] 102 ldp q6, q7, [sp, #96] 1031: 104 /* Load the core argument passing registers, including 105 the structure return pointer. */ 106 ldp x0, x1, [sp, #16*N_V_ARG_REG + 0] 107 ldp x2, x3, [sp, #16*N_V_ARG_REG + 16] 108 ldp x4, x5, [sp, #16*N_V_ARG_REG + 32] 109 ldp x6, x7, [sp, #16*N_V_ARG_REG + 48] 110 111 /* Deallocate the context, leaving the stacked arguments. */ 112 add sp, sp, #CALL_CONTEXT_SIZE 113 114 blr x9 /* call fn */ 115 116 ldp x3, x4, [x29, #16] /* reload rvalue and flags */ 117 118 /* Partially deconstruct the stack frame. */ 119 mov sp, x29 120 cfi_def_cfa_register (sp) 121 ldp x29, x30, [x29] 122 123 /* Save the return value as directed. */ 124 adr x5, 0f 125 and w4, w4, #AARCH64_RET_MASK 126 add x5, x5, x4, lsl #3 127 br x5 128 129 /* Note that each table entry is 2 insns, and thus 8 bytes. 130 For integer data, note that we're storing into ffi_arg 131 and therefore we want to extend to 64 bits; these types 132 have two consecutive entries allocated for them. */ 133 .align 4 1340: ret /* VOID */ 135 nop 1361: str x0, [x3] /* INT64 */ 137 ret 1382: stp x0, x1, [x3] /* INT128 */ 139 ret 1403: brk #1000 /* UNUSED */ 141 ret 1424: brk #1000 /* UNUSED */ 143 ret 1445: brk #1000 /* UNUSED */ 145 ret 1466: brk #1000 /* UNUSED */ 147 ret 1487: brk #1000 /* UNUSED */ 149 ret 1508: st4 { v0.s, v1.s, v2.s, v3.s }[0], [x3] /* S4 */ 151 ret 1529: st3 { v0.s, v1.s, v2.s }[0], [x3] /* S3 */ 153 ret 15410: stp s0, s1, [x3] /* S2 */ 155 ret 15611: str s0, [x3] /* S1 */ 157 ret 15812: st4 { v0.d, v1.d, v2.d, v3.d }[0], [x3] /* D4 */ 159 ret 16013: st3 { v0.d, v1.d, v2.d }[0], [x3] /* D3 */ 161 ret 16214: stp d0, d1, [x3] /* D2 */ 163 ret 16415: str d0, [x3] /* D1 */ 165 ret 16616: str q3, [x3, #48] /* Q4 */ 167 nop 16817: str q2, [x3, #32] /* Q3 */ 169 nop 17018: stp q0, q1, [x3] /* Q2 */ 171 ret 17219: str q0, [x3] /* Q1 */ 173 ret 17420: uxtb w0, w0 /* UINT8 */ 175 str x0, [x3] 17621: ret /* reserved */ 177 nop 17822: uxth w0, w0 /* UINT16 */ 179 str x0, [x3] 18023: ret /* reserved */ 181 nop 18224: mov w0, w0 /* UINT32 */ 183 str x0, [x3] 18425: ret /* reserved */ 185 nop 18626: sxtb x0, w0 /* SINT8 */ 187 str x0, [x3] 18827: ret /* reserved */ 189 nop 19028: sxth x0, w0 /* SINT16 */ 191 str x0, [x3] 19229: ret /* reserved */ 193 nop 19430: sxtw x0, w0 /* SINT32 */ 195 str x0, [x3] 19631: ret /* reserved */ 197 nop 198 199 cfi_endproc 200 201 .globl CNAME(ffi_call_SYSV) 202 FFI_HIDDEN(CNAME(ffi_call_SYSV)) 203#ifdef __ELF__ 204 .type CNAME(ffi_call_SYSV), #function 205 .size CNAME(ffi_call_SYSV), .-CNAME(ffi_call_SYSV) 206#endif 207 208/* ffi_closure_SYSV 209 210 Closure invocation glue. This is the low level code invoked directly by 211 the closure trampoline to setup and call a closure. 212 213 On entry x17 points to a struct ffi_closure, x16 has been clobbered 214 all other registers are preserved. 215 216 We allocate a call context and save the argument passing registers, 217 then invoked the generic C ffi_closure_SYSV_inner() function to do all 218 the real work, on return we load the result passing registers back from 219 the call context. 220*/ 221 222#define ffi_closure_SYSV_FS (8*2 + CALL_CONTEXT_SIZE + 64) 223 224 .align 4 225CNAME(ffi_closure_SYSV_V): 226 cfi_startproc 227 stp x29, x30, [sp, #-ffi_closure_SYSV_FS]! 228 cfi_adjust_cfa_offset (ffi_closure_SYSV_FS) 229 cfi_rel_offset (x29, 0) 230 cfi_rel_offset (x30, 8) 231 232 /* Save the argument passing vector registers. */ 233 stp q0, q1, [sp, #16 + 0] 234 stp q2, q3, [sp, #16 + 32] 235 stp q4, q5, [sp, #16 + 64] 236 stp q6, q7, [sp, #16 + 96] 237 b 0f 238 cfi_endproc 239 240 .globl CNAME(ffi_closure_SYSV_V) 241 FFI_HIDDEN(CNAME(ffi_closure_SYSV_V)) 242#ifdef __ELF__ 243 .type CNAME(ffi_closure_SYSV_V), #function 244 .size CNAME(ffi_closure_SYSV_V), . - CNAME(ffi_closure_SYSV_V) 245#endif 246 247 .align 4 248 cfi_startproc 249CNAME(ffi_closure_SYSV): 250 stp x29, x30, [sp, #-ffi_closure_SYSV_FS]! 251 cfi_adjust_cfa_offset (ffi_closure_SYSV_FS) 252 cfi_rel_offset (x29, 0) 253 cfi_rel_offset (x30, 8) 2540: 255 mov x29, sp 256 257 /* Save the argument passing core registers. */ 258 stp x0, x1, [sp, #16 + 16*N_V_ARG_REG + 0] 259 stp x2, x3, [sp, #16 + 16*N_V_ARG_REG + 16] 260 stp x4, x5, [sp, #16 + 16*N_V_ARG_REG + 32] 261 stp x6, x7, [sp, #16 + 16*N_V_ARG_REG + 48] 262 263 /* Load ffi_closure_inner arguments. */ 264 ldp PTR_REG(0), PTR_REG(1), [x17, #FFI_TRAMPOLINE_CLOSURE_OFFSET] /* load cif, fn */ 265 ldr PTR_REG(2), [x17, #FFI_TRAMPOLINE_CLOSURE_OFFSET+PTR_SIZE*2] /* load user_data */ 266.Ldo_closure: 267 add x3, sp, #16 /* load context */ 268 add x4, sp, #ffi_closure_SYSV_FS /* load stack */ 269 add x5, sp, #16+CALL_CONTEXT_SIZE /* load rvalue */ 270 mov x6, x8 /* load struct_rval */ 271 bl CNAME(ffi_closure_SYSV_inner) 272 273 /* Load the return value as directed. */ 274 adr x1, 0f 275 and w0, w0, #AARCH64_RET_MASK 276 add x1, x1, x0, lsl #3 277 add x3, sp, #16+CALL_CONTEXT_SIZE 278 br x1 279 280 /* Note that each table entry is 2 insns, and thus 8 bytes. */ 281 .align 4 2820: b 99f /* VOID */ 283 nop 2841: ldr x0, [x3] /* INT64 */ 285 b 99f 2862: ldp x0, x1, [x3] /* INT128 */ 287 b 99f 2883: brk #1000 /* UNUSED */ 289 nop 2904: brk #1000 /* UNUSED */ 291 nop 2925: brk #1000 /* UNUSED */ 293 nop 2946: brk #1000 /* UNUSED */ 295 nop 2967: brk #1000 /* UNUSED */ 297 nop 2988: ldr s3, [x3, #12] /* S4 */ 299 nop 3009: ldr s2, [x3, #8] /* S3 */ 301 nop 30210: ldp s0, s1, [x3] /* S2 */ 303 b 99f 30411: ldr s0, [x3] /* S1 */ 305 b 99f 30612: ldr d3, [x3, #24] /* D4 */ 307 nop 30813: ldr d2, [x3, #16] /* D3 */ 309 nop 31014: ldp d0, d1, [x3] /* D2 */ 311 b 99f 31215: ldr d0, [x3] /* D1 */ 313 b 99f 31416: ldr q3, [x3, #48] /* Q4 */ 315 nop 31617: ldr q2, [x3, #32] /* Q3 */ 317 nop 31818: ldp q0, q1, [x3] /* Q2 */ 319 b 99f 32019: ldr q0, [x3] /* Q1 */ 321 b 99f 32220: ldrb w0, [x3, #BE(7)] /* UINT8 */ 323 b 99f 32421: brk #1000 /* reserved */ 325 nop 32622: ldrh w0, [x3, #BE(6)] /* UINT16 */ 327 b 99f 32823: brk #1000 /* reserved */ 329 nop 33024: ldr w0, [x3, #BE(4)] /* UINT32 */ 331 b 99f 33225: brk #1000 /* reserved */ 333 nop 33426: ldrsb x0, [x3, #BE(7)] /* SINT8 */ 335 b 99f 33627: brk #1000 /* reserved */ 337 nop 33828: ldrsh x0, [x3, #BE(6)] /* SINT16 */ 339 b 99f 34029: brk #1000 /* reserved */ 341 nop 34230: ldrsw x0, [x3, #BE(4)] /* SINT32 */ 343 nop 34431: /* reserved */ 34599: ldp x29, x30, [sp], #ffi_closure_SYSV_FS 346 cfi_adjust_cfa_offset (-ffi_closure_SYSV_FS) 347 cfi_restore (x29) 348 cfi_restore (x30) 349 ret 350 cfi_endproc 351 352 .globl CNAME(ffi_closure_SYSV) 353 FFI_HIDDEN(CNAME(ffi_closure_SYSV)) 354#ifdef __ELF__ 355 .type CNAME(ffi_closure_SYSV), #function 356 .size CNAME(ffi_closure_SYSV), . - CNAME(ffi_closure_SYSV) 357#endif 358 359#if FFI_EXEC_TRAMPOLINE_TABLE 360 361#ifdef __MACH__ 362#include <mach/machine/vm_param.h> 363 .align PAGE_MAX_SHIFT 364CNAME(ffi_closure_trampoline_table_page): 365 .rept PAGE_MAX_SIZE / FFI_TRAMPOLINE_SIZE 366 adr x16, -PAGE_MAX_SIZE 367 ldp x17, x16, [x16] 368 br x16 369 nop /* each entry in the trampoline config page is 2*sizeof(void*) so the trampoline itself cannot be smaller that 16 bytes */ 370 .endr 371 372 .globl CNAME(ffi_closure_trampoline_table_page) 373 FFI_HIDDEN(CNAME(ffi_closure_trampoline_table_page)) 374 #ifdef __ELF__ 375 .type CNAME(ffi_closure_trampoline_table_page), #function 376 .size CNAME(ffi_closure_trampoline_table_page), . - CNAME(ffi_closure_trampoline_table_page) 377 #endif 378#endif 379 380#endif /* FFI_EXEC_TRAMPOLINE_TABLE */ 381 382#ifdef FFI_GO_CLOSURES 383 .align 4 384CNAME(ffi_go_closure_SYSV_V): 385 cfi_startproc 386 stp x29, x30, [sp, #-ffi_closure_SYSV_FS]! 387 cfi_adjust_cfa_offset (ffi_closure_SYSV_FS) 388 cfi_rel_offset (x29, 0) 389 cfi_rel_offset (x30, 8) 390 391 /* Save the argument passing vector registers. */ 392 stp q0, q1, [sp, #16 + 0] 393 stp q2, q3, [sp, #16 + 32] 394 stp q4, q5, [sp, #16 + 64] 395 stp q6, q7, [sp, #16 + 96] 396 b 0f 397 cfi_endproc 398 399 .globl CNAME(ffi_go_closure_SYSV_V) 400 FFI_HIDDEN(CNAME(ffi_go_closure_SYSV_V)) 401#ifdef __ELF__ 402 .type CNAME(ffi_go_closure_SYSV_V), #function 403 .size CNAME(ffi_go_closure_SYSV_V), . - CNAME(ffi_go_closure_SYSV_V) 404#endif 405 406 .align 4 407 cfi_startproc 408CNAME(ffi_go_closure_SYSV): 409 stp x29, x30, [sp, #-ffi_closure_SYSV_FS]! 410 cfi_adjust_cfa_offset (ffi_closure_SYSV_FS) 411 cfi_rel_offset (x29, 0) 412 cfi_rel_offset (x30, 8) 4130: 414 mov x29, sp 415 416 /* Save the argument passing core registers. */ 417 stp x0, x1, [sp, #16 + 16*N_V_ARG_REG + 0] 418 stp x2, x3, [sp, #16 + 16*N_V_ARG_REG + 16] 419 stp x4, x5, [sp, #16 + 16*N_V_ARG_REG + 32] 420 stp x6, x7, [sp, #16 + 16*N_V_ARG_REG + 48] 421 422 /* Load ffi_closure_inner arguments. */ 423 ldp PTR_REG(0), PTR_REG(1), [x18, #PTR_SIZE]/* load cif, fn */ 424 mov x2, x18 /* load user_data */ 425 b .Ldo_closure 426 cfi_endproc 427 428 .globl CNAME(ffi_go_closure_SYSV) 429 FFI_HIDDEN(CNAME(ffi_go_closure_SYSV)) 430#ifdef __ELF__ 431 .type CNAME(ffi_go_closure_SYSV), #function 432 .size CNAME(ffi_go_closure_SYSV), . - CNAME(ffi_go_closure_SYSV) 433#endif 434#endif /* FFI_GO_CLOSURES */ 435#endif /* __arm64__ */ 436 437#if defined __ELF__ && defined __linux__ 438 .section .note.GNU-stack,"",%progbits 439#endif 440 441