1 /* -----------------------------------------------------------------------
2 ffi_linux64.c - Copyright (C) 2013 IBM
3 Copyright (C) 2011 Anthony Green
4 Copyright (C) 2011 Kyle Moffett
5 Copyright (C) 2008 Red Hat, Inc
6 Copyright (C) 2007, 2008 Free Software Foundation, Inc
7 Copyright (c) 1998 Geoffrey Keating
8
9 PowerPC Foreign Function Interface
10
11 Permission is hereby granted, free of charge, to any person obtaining
12 a copy of this software and associated documentation files (the
13 ``Software''), to deal in the Software without restriction, including
14 without limitation the rights to use, copy, modify, merge, publish,
15 distribute, sublicense, and/or sell copies of the Software, and to
16 permit persons to whom the Software is furnished to do so, subject to
17 the following conditions:
18
19 The above copyright notice and this permission notice shall be included
20 in all copies or substantial portions of the Software.
21
22 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
25 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR
26 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
27 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 OTHER DEALINGS IN THE SOFTWARE.
29 ----------------------------------------------------------------------- */
30
31 #include "ffi.h"
32
33 #ifdef POWERPC64
34 #include "ffi_common.h"
35 #include "ffi_powerpc.h"
36
37
38 /* About the LINUX64 ABI. */
39 enum {
40 NUM_GPR_ARG_REGISTERS64 = 8,
41 NUM_FPR_ARG_REGISTERS64 = 13
42 };
43 enum { ASM_NEEDS_REGISTERS64 = 4 };
44
45
46 #if HAVE_LONG_DOUBLE_VARIANT && FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
47 /* Adjust size of ffi_type_longdouble. */
48 void FFI_HIDDEN
ffi_prep_types_linux64(ffi_abi abi)49 ffi_prep_types_linux64 (ffi_abi abi)
50 {
51 if ((abi & (FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128)) == FFI_LINUX)
52 {
53 ffi_type_longdouble.size = 8;
54 ffi_type_longdouble.alignment = 8;
55 }
56 else
57 {
58 ffi_type_longdouble.size = 16;
59 ffi_type_longdouble.alignment = 16;
60 }
61 }
62 #endif
63
64
65 #if _CALL_ELF == 2
66 static unsigned int
discover_homogeneous_aggregate(const ffi_type * t,unsigned int * elnum)67 discover_homogeneous_aggregate (const ffi_type *t, unsigned int *elnum)
68 {
69 switch (t->type)
70 {
71 case FFI_TYPE_FLOAT:
72 case FFI_TYPE_DOUBLE:
73 *elnum = 1;
74 return (int) t->type;
75
76 case FFI_TYPE_STRUCT:;
77 {
78 unsigned int base_elt = 0, total_elnum = 0;
79 ffi_type **el = t->elements;
80 while (*el)
81 {
82 unsigned int el_elt, el_elnum = 0;
83 el_elt = discover_homogeneous_aggregate (*el, &el_elnum);
84 if (el_elt == 0
85 || (base_elt && base_elt != el_elt))
86 return 0;
87 base_elt = el_elt;
88 total_elnum += el_elnum;
89 if (total_elnum > 8)
90 return 0;
91 el++;
92 }
93 *elnum = total_elnum;
94 return base_elt;
95 }
96
97 default:
98 return 0;
99 }
100 }
101 #endif
102
103
104 /* Perform machine dependent cif processing */
105 static ffi_status
ffi_prep_cif_linux64_core(ffi_cif * cif)106 ffi_prep_cif_linux64_core (ffi_cif *cif)
107 {
108 ffi_type **ptr;
109 unsigned bytes;
110 unsigned i, fparg_count = 0, intarg_count = 0;
111 unsigned flags = cif->flags;
112 #if _CALL_ELF == 2
113 unsigned int elt, elnum;
114 #endif
115
116 #if FFI_TYPE_LONGDOUBLE == FFI_TYPE_DOUBLE
117 /* If compiled without long double support.. */
118 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
119 return FFI_BAD_ABI;
120 #endif
121
122 /* The machine-independent calculation of cif->bytes doesn't work
123 for us. Redo the calculation. */
124 #if _CALL_ELF == 2
125 /* Space for backchain, CR, LR, TOC and the asm's temp regs. */
126 bytes = (4 + ASM_NEEDS_REGISTERS64) * sizeof (long);
127
128 /* Space for the general registers. */
129 bytes += NUM_GPR_ARG_REGISTERS64 * sizeof (long);
130 #else
131 /* Space for backchain, CR, LR, cc/ld doubleword, TOC and the asm's temp
132 regs. */
133 bytes = (6 + ASM_NEEDS_REGISTERS64) * sizeof (long);
134
135 /* Space for the mandatory parm save area and general registers. */
136 bytes += 2 * NUM_GPR_ARG_REGISTERS64 * sizeof (long);
137 #endif
138
139 /* Return value handling. */
140 switch (cif->rtype->type)
141 {
142 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
143 case FFI_TYPE_LONGDOUBLE:
144 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
145 flags |= FLAG_RETURNS_128BITS;
146 /* Fall through. */
147 #endif
148 case FFI_TYPE_DOUBLE:
149 flags |= FLAG_RETURNS_64BITS;
150 /* Fall through. */
151 case FFI_TYPE_FLOAT:
152 flags |= FLAG_RETURNS_FP;
153 break;
154
155 case FFI_TYPE_UINT128:
156 flags |= FLAG_RETURNS_128BITS;
157 /* Fall through. */
158 case FFI_TYPE_UINT64:
159 case FFI_TYPE_SINT64:
160 flags |= FLAG_RETURNS_64BITS;
161 break;
162
163 case FFI_TYPE_STRUCT:
164 #if _CALL_ELF == 2
165 elt = discover_homogeneous_aggregate (cif->rtype, &elnum);
166 if (elt)
167 {
168 if (elt == FFI_TYPE_DOUBLE)
169 flags |= FLAG_RETURNS_64BITS;
170 flags |= FLAG_RETURNS_FP | FLAG_RETURNS_SMST;
171 break;
172 }
173 if (cif->rtype->size <= 16)
174 {
175 flags |= FLAG_RETURNS_SMST;
176 break;
177 }
178 #endif
179 intarg_count++;
180 flags |= FLAG_RETVAL_REFERENCE;
181 /* Fall through. */
182 case FFI_TYPE_VOID:
183 flags |= FLAG_RETURNS_NOTHING;
184 break;
185
186 default:
187 /* Returns 32-bit integer, or similar. Nothing to do here. */
188 break;
189 }
190
191 for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++)
192 {
193 unsigned int align;
194
195 switch ((*ptr)->type)
196 {
197 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
198 case FFI_TYPE_LONGDOUBLE:
199 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
200 {
201 fparg_count++;
202 intarg_count++;
203 }
204 /* Fall through. */
205 #endif
206 case FFI_TYPE_DOUBLE:
207 case FFI_TYPE_FLOAT:
208 fparg_count++;
209 intarg_count++;
210 if (fparg_count > NUM_FPR_ARG_REGISTERS64)
211 flags |= FLAG_ARG_NEEDS_PSAVE;
212 break;
213
214 case FFI_TYPE_STRUCT:
215 if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
216 {
217 align = (*ptr)->alignment;
218 if (align > 16)
219 align = 16;
220 align = align / 8;
221 if (align > 1)
222 intarg_count = ALIGN (intarg_count, align);
223 }
224 intarg_count += ((*ptr)->size + 7) / 8;
225 #if _CALL_ELF == 2
226 elt = discover_homogeneous_aggregate (*ptr, &elnum);
227 if (elt)
228 {
229 fparg_count += elnum;
230 if (fparg_count > NUM_FPR_ARG_REGISTERS64)
231 flags |= FLAG_ARG_NEEDS_PSAVE;
232 }
233 else
234 #endif
235 {
236 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
237 flags |= FLAG_ARG_NEEDS_PSAVE;
238 }
239 break;
240
241 case FFI_TYPE_POINTER:
242 case FFI_TYPE_UINT64:
243 case FFI_TYPE_SINT64:
244 case FFI_TYPE_INT:
245 case FFI_TYPE_UINT32:
246 case FFI_TYPE_SINT32:
247 case FFI_TYPE_UINT16:
248 case FFI_TYPE_SINT16:
249 case FFI_TYPE_UINT8:
250 case FFI_TYPE_SINT8:
251 /* Everything else is passed as a 8-byte word in a GPR, either
252 the object itself or a pointer to it. */
253 intarg_count++;
254 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
255 flags |= FLAG_ARG_NEEDS_PSAVE;
256 break;
257 default:
258 FFI_ASSERT (0);
259 }
260 }
261
262 if (fparg_count != 0)
263 flags |= FLAG_FP_ARGUMENTS;
264 if (intarg_count > 4)
265 flags |= FLAG_4_GPR_ARGUMENTS;
266
267 /* Space for the FPR registers, if needed. */
268 if (fparg_count != 0)
269 bytes += NUM_FPR_ARG_REGISTERS64 * sizeof (double);
270
271 /* Stack space. */
272 #if _CALL_ELF == 2
273 if ((flags & FLAG_ARG_NEEDS_PSAVE) != 0)
274 bytes += intarg_count * sizeof (long);
275 #else
276 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
277 bytes += (intarg_count - NUM_GPR_ARG_REGISTERS64) * sizeof (long);
278 #endif
279
280 /* The stack space allocated needs to be a multiple of 16 bytes. */
281 bytes = (bytes + 15) & ~0xF;
282
283 cif->flags = flags;
284 cif->bytes = bytes;
285
286 return FFI_OK;
287 }
288
289 ffi_status FFI_HIDDEN
ffi_prep_cif_linux64(ffi_cif * cif)290 ffi_prep_cif_linux64 (ffi_cif *cif)
291 {
292 if ((cif->abi & FFI_LINUX) != 0)
293 cif->nfixedargs = cif->nargs;
294 #if _CALL_ELF != 2
295 else if (cif->abi == FFI_COMPAT_LINUX64)
296 {
297 /* This call is from old code. Don't touch cif->nfixedargs
298 since old code will be using a smaller cif. */
299 cif->flags |= FLAG_COMPAT;
300 /* Translate to new abi value. */
301 cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
302 }
303 #endif
304 else
305 return FFI_BAD_ABI;
306 return ffi_prep_cif_linux64_core (cif);
307 }
308
309 ffi_status FFI_HIDDEN
ffi_prep_cif_linux64_var(ffi_cif * cif,unsigned int nfixedargs,unsigned int ntotalargs MAYBE_UNUSED)310 ffi_prep_cif_linux64_var (ffi_cif *cif,
311 unsigned int nfixedargs,
312 unsigned int ntotalargs MAYBE_UNUSED)
313 {
314 if ((cif->abi & FFI_LINUX) != 0)
315 cif->nfixedargs = nfixedargs;
316 #if _CALL_ELF != 2
317 else if (cif->abi == FFI_COMPAT_LINUX64)
318 {
319 /* This call is from old code. Don't touch cif->nfixedargs
320 since old code will be using a smaller cif. */
321 cif->flags |= FLAG_COMPAT;
322 /* Translate to new abi value. */
323 cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
324 }
325 #endif
326 else
327 return FFI_BAD_ABI;
328 #if _CALL_ELF == 2
329 cif->flags |= FLAG_ARG_NEEDS_PSAVE;
330 #endif
331 return ffi_prep_cif_linux64_core (cif);
332 }
333
334
335 /* ffi_prep_args64 is called by the assembly routine once stack space
336 has been allocated for the function's arguments.
337
338 The stack layout we want looks like this:
339
340 | Ret addr from ffi_call_LINUX64 8bytes | higher addresses
341 |--------------------------------------------|
342 | CR save area 8bytes |
343 |--------------------------------------------|
344 | Previous backchain pointer 8 | stack pointer here
345 |--------------------------------------------|<+ <<< on entry to
346 | Saved r28-r31 4*8 | | ffi_call_LINUX64
347 |--------------------------------------------| |
348 | GPR registers r3-r10 8*8 | |
349 |--------------------------------------------| |
350 | FPR registers f1-f13 (optional) 13*8 | |
351 |--------------------------------------------| |
352 | Parameter save area | |
353 |--------------------------------------------| |
354 | TOC save area 8 | |
355 |--------------------------------------------| | stack |
356 | Linker doubleword 8 | | grows |
357 |--------------------------------------------| | down V
358 | Compiler doubleword 8 | |
359 |--------------------------------------------| | lower addresses
360 | Space for callee's LR 8 | |
361 |--------------------------------------------| |
362 | CR save area 8 | |
363 |--------------------------------------------| | stack pointer here
364 | Current backchain pointer 8 |-/ during
365 |--------------------------------------------| <<< ffi_call_LINUX64
366
367 */
368
369 void FFI_HIDDEN
ffi_prep_args64(extended_cif * ecif,unsigned long * const stack)370 ffi_prep_args64 (extended_cif *ecif, unsigned long *const stack)
371 {
372 const unsigned long bytes = ecif->cif->bytes;
373 const unsigned long flags = ecif->cif->flags;
374
375 typedef union
376 {
377 char *c;
378 unsigned long *ul;
379 float *f;
380 double *d;
381 size_t p;
382 } valp;
383
384 /* 'stacktop' points at the previous backchain pointer. */
385 valp stacktop;
386
387 /* 'next_arg' points at the space for gpr3, and grows upwards as
388 we use GPR registers, then continues at rest. */
389 valp gpr_base;
390 valp gpr_end;
391 valp rest;
392 valp next_arg;
393
394 /* 'fpr_base' points at the space for fpr3, and grows upwards as
395 we use FPR registers. */
396 valp fpr_base;
397 unsigned int fparg_count;
398
399 unsigned int i, words, nargs, nfixedargs;
400 ffi_type **ptr;
401 double double_tmp;
402 union
403 {
404 void **v;
405 char **c;
406 signed char **sc;
407 unsigned char **uc;
408 signed short **ss;
409 unsigned short **us;
410 signed int **si;
411 unsigned int **ui;
412 unsigned long **ul;
413 float **f;
414 double **d;
415 } p_argv;
416 unsigned long gprvalue;
417 unsigned long align;
418
419 stacktop.c = (char *) stack + bytes;
420 gpr_base.ul = stacktop.ul - ASM_NEEDS_REGISTERS64 - NUM_GPR_ARG_REGISTERS64;
421 gpr_end.ul = gpr_base.ul + NUM_GPR_ARG_REGISTERS64;
422 #if _CALL_ELF == 2
423 rest.ul = stack + 4 + NUM_GPR_ARG_REGISTERS64;
424 #else
425 rest.ul = stack + 6 + NUM_GPR_ARG_REGISTERS64;
426 #endif
427 fpr_base.d = gpr_base.d - NUM_FPR_ARG_REGISTERS64;
428 fparg_count = 0;
429 next_arg.ul = gpr_base.ul;
430
431 /* Check that everything starts aligned properly. */
432 FFI_ASSERT (((unsigned long) (char *) stack & 0xF) == 0);
433 FFI_ASSERT (((unsigned long) stacktop.c & 0xF) == 0);
434 FFI_ASSERT ((bytes & 0xF) == 0);
435
436 /* Deal with return values that are actually pass-by-reference. */
437 if (flags & FLAG_RETVAL_REFERENCE)
438 *next_arg.ul++ = (unsigned long) (char *) ecif->rvalue;
439
440 /* Now for the arguments. */
441 p_argv.v = ecif->avalue;
442 nargs = ecif->cif->nargs;
443 #if _CALL_ELF != 2
444 nfixedargs = (unsigned) -1;
445 if ((flags & FLAG_COMPAT) == 0)
446 #endif
447 nfixedargs = ecif->cif->nfixedargs;
448 for (ptr = ecif->cif->arg_types, i = 0;
449 i < nargs;
450 i++, ptr++, p_argv.v++)
451 {
452 #if _CALL_ELF == 2
453 unsigned int elt, elnum;
454 #endif
455
456 switch ((*ptr)->type)
457 {
458 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
459 case FFI_TYPE_LONGDOUBLE:
460 if ((ecif->cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
461 {
462 double_tmp = (*p_argv.d)[0];
463 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
464 {
465 *fpr_base.d++ = double_tmp;
466 # if _CALL_ELF != 2
467 if ((flags & FLAG_COMPAT) != 0)
468 *next_arg.d = double_tmp;
469 # endif
470 }
471 else
472 *next_arg.d = double_tmp;
473 if (++next_arg.ul == gpr_end.ul)
474 next_arg.ul = rest.ul;
475 fparg_count++;
476 double_tmp = (*p_argv.d)[1];
477 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
478 {
479 *fpr_base.d++ = double_tmp;
480 # if _CALL_ELF != 2
481 if ((flags & FLAG_COMPAT) != 0)
482 *next_arg.d = double_tmp;
483 # endif
484 }
485 else
486 *next_arg.d = double_tmp;
487 if (++next_arg.ul == gpr_end.ul)
488 next_arg.ul = rest.ul;
489 fparg_count++;
490 FFI_ASSERT (__LDBL_MANT_DIG__ == 106);
491 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
492 break;
493 }
494 /* Fall through. */
495 #endif
496 case FFI_TYPE_DOUBLE:
497 double_tmp = **p_argv.d;
498 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
499 {
500 *fpr_base.d++ = double_tmp;
501 #if _CALL_ELF != 2
502 if ((flags & FLAG_COMPAT) != 0)
503 *next_arg.d = double_tmp;
504 #endif
505 }
506 else
507 *next_arg.d = double_tmp;
508 if (++next_arg.ul == gpr_end.ul)
509 next_arg.ul = rest.ul;
510 fparg_count++;
511 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
512 break;
513
514 case FFI_TYPE_FLOAT:
515 double_tmp = **p_argv.f;
516 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
517 {
518 *fpr_base.d++ = double_tmp;
519 #if _CALL_ELF != 2
520 if ((flags & FLAG_COMPAT) != 0)
521 *next_arg.f = (float) double_tmp;
522 #endif
523 }
524 else
525 *next_arg.f = (float) double_tmp;
526 if (++next_arg.ul == gpr_end.ul)
527 next_arg.ul = rest.ul;
528 fparg_count++;
529 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
530 break;
531
532 case FFI_TYPE_STRUCT:
533 if ((ecif->cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
534 {
535 align = (*ptr)->alignment;
536 if (align > 16)
537 align = 16;
538 if (align > 1)
539 next_arg.p = ALIGN (next_arg.p, align);
540 }
541 #if _CALL_ELF == 2
542 elt = discover_homogeneous_aggregate (*ptr, &elnum);
543 if (elt)
544 {
545 union {
546 void *v;
547 float *f;
548 double *d;
549 } arg;
550
551 arg.v = *p_argv.v;
552 if (elt == FFI_TYPE_FLOAT)
553 {
554 do
555 {
556 double_tmp = *arg.f++;
557 if (fparg_count < NUM_FPR_ARG_REGISTERS64
558 && i < nfixedargs)
559 *fpr_base.d++ = double_tmp;
560 else
561 *next_arg.f = (float) double_tmp;
562 if (++next_arg.f == gpr_end.f)
563 next_arg.f = rest.f;
564 fparg_count++;
565 }
566 while (--elnum != 0);
567 if ((next_arg.p & 3) != 0)
568 {
569 if (++next_arg.f == gpr_end.f)
570 next_arg.f = rest.f;
571 }
572 }
573 else
574 do
575 {
576 double_tmp = *arg.d++;
577 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
578 *fpr_base.d++ = double_tmp;
579 else
580 *next_arg.d = double_tmp;
581 if (++next_arg.d == gpr_end.d)
582 next_arg.d = rest.d;
583 fparg_count++;
584 }
585 while (--elnum != 0);
586 }
587 else
588 #endif
589 {
590 words = ((*ptr)->size + 7) / 8;
591 if (next_arg.ul >= gpr_base.ul && next_arg.ul + words > gpr_end.ul)
592 {
593 size_t first = gpr_end.c - next_arg.c;
594 memcpy (next_arg.c, *p_argv.c, first);
595 memcpy (rest.c, *p_argv.c + first, (*ptr)->size - first);
596 next_arg.c = rest.c + words * 8 - first;
597 }
598 else
599 {
600 char *where = next_arg.c;
601
602 #ifndef __LITTLE_ENDIAN__
603 /* Structures with size less than eight bytes are passed
604 left-padded. */
605 if ((*ptr)->size < 8)
606 where += 8 - (*ptr)->size;
607 #endif
608 memcpy (where, *p_argv.c, (*ptr)->size);
609 next_arg.ul += words;
610 if (next_arg.ul == gpr_end.ul)
611 next_arg.ul = rest.ul;
612 }
613 }
614 break;
615
616 case FFI_TYPE_UINT8:
617 gprvalue = **p_argv.uc;
618 goto putgpr;
619 case FFI_TYPE_SINT8:
620 gprvalue = **p_argv.sc;
621 goto putgpr;
622 case FFI_TYPE_UINT16:
623 gprvalue = **p_argv.us;
624 goto putgpr;
625 case FFI_TYPE_SINT16:
626 gprvalue = **p_argv.ss;
627 goto putgpr;
628 case FFI_TYPE_UINT32:
629 gprvalue = **p_argv.ui;
630 goto putgpr;
631 case FFI_TYPE_INT:
632 case FFI_TYPE_SINT32:
633 gprvalue = **p_argv.si;
634 goto putgpr;
635
636 case FFI_TYPE_UINT64:
637 case FFI_TYPE_SINT64:
638 case FFI_TYPE_POINTER:
639 gprvalue = **p_argv.ul;
640 putgpr:
641 *next_arg.ul++ = gprvalue;
642 if (next_arg.ul == gpr_end.ul)
643 next_arg.ul = rest.ul;
644 break;
645 }
646 }
647
648 FFI_ASSERT (flags & FLAG_4_GPR_ARGUMENTS
649 || (next_arg.ul >= gpr_base.ul
650 && next_arg.ul <= gpr_base.ul + 4));
651 }
652
653
654 #if _CALL_ELF == 2
655 #define MIN_CACHE_LINE_SIZE 8
656
657 static void
flush_icache(char * wraddr,char * xaddr,int size)658 flush_icache (char *wraddr, char *xaddr, int size)
659 {
660 int i;
661 for (i = 0; i < size; i += MIN_CACHE_LINE_SIZE)
662 __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;"
663 : : "r" (xaddr + i), "r" (wraddr + i) : "memory");
664 __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;" "sync;" "isync;"
665 : : "r"(xaddr + size - 1), "r"(wraddr + size - 1)
666 : "memory");
667 }
668 #endif
669
670 ffi_status
ffi_prep_closure_loc_linux64(ffi_closure * closure,ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data,void * codeloc)671 ffi_prep_closure_loc_linux64 (ffi_closure *closure,
672 ffi_cif *cif,
673 void (*fun) (ffi_cif *, void *, void **, void *),
674 void *user_data,
675 void *codeloc)
676 {
677 #if _CALL_ELF == 2
678 unsigned int *tramp = (unsigned int *) &closure->tramp[0];
679
680 if (cif->abi < FFI_LINUX || cif->abi >= FFI_LAST_ABI)
681 return FFI_BAD_ABI;
682
683 tramp[0] = 0xe96c0018; /* 0: ld 11,2f-0b(12) */
684 tramp[1] = 0xe98c0010; /* ld 12,1f-0b(12) */
685 tramp[2] = 0x7d8903a6; /* mtctr 12 */
686 tramp[3] = 0x4e800420; /* bctr */
687 /* 1: .quad function_addr */
688 /* 2: .quad context */
689 *(void **) &tramp[4] = (void *) ffi_closure_LINUX64;
690 *(void **) &tramp[6] = codeloc;
691 flush_icache ((char *)tramp, (char *)codeloc, FFI_TRAMPOLINE_SIZE);
692 #else
693 void **tramp = (void **) &closure->tramp[0];
694
695 if (cif->abi < FFI_LINUX || cif->abi >= FFI_LAST_ABI)
696 return FFI_BAD_ABI;
697
698 /* Copy function address and TOC from ffi_closure_LINUX64. */
699 memcpy (tramp, (char *) ffi_closure_LINUX64, 16);
700 tramp[2] = tramp[1];
701 tramp[1] = codeloc;
702 #endif
703
704 closure->cif = cif;
705 closure->fun = fun;
706 closure->user_data = user_data;
707
708 return FFI_OK;
709 }
710
711
712 int FFI_HIDDEN
ffi_closure_helper_LINUX64(ffi_closure * closure,void * rvalue,unsigned long * pst,ffi_dblfl * pfr)713 ffi_closure_helper_LINUX64 (ffi_closure *closure, void *rvalue,
714 unsigned long *pst, ffi_dblfl *pfr)
715 {
716 /* rvalue is the pointer to space for return value in closure assembly */
717 /* pst is the pointer to parameter save area
718 (r3-r10 are stored into its first 8 slots by ffi_closure_LINUX64) */
719 /* pfr is the pointer to where f1-f13 are stored in ffi_closure_LINUX64 */
720
721 void **avalue;
722 ffi_type **arg_types;
723 unsigned long i, avn, nfixedargs;
724 ffi_cif *cif;
725 ffi_dblfl *end_pfr = pfr + NUM_FPR_ARG_REGISTERS64;
726 unsigned long align;
727
728 cif = closure->cif;
729 avalue = alloca (cif->nargs * sizeof (void *));
730
731 /* Copy the caller's structure return value address so that the
732 closure returns the data directly to the caller. */
733 if (cif->rtype->type == FFI_TYPE_STRUCT
734 && (cif->flags & FLAG_RETURNS_SMST) == 0)
735 {
736 rvalue = (void *) *pst;
737 pst++;
738 }
739
740 i = 0;
741 avn = cif->nargs;
742 #if _CALL_ELF != 2
743 nfixedargs = (unsigned) -1;
744 if ((cif->flags & FLAG_COMPAT) == 0)
745 #endif
746 nfixedargs = cif->nfixedargs;
747 arg_types = cif->arg_types;
748
749 /* Grab the addresses of the arguments from the stack frame. */
750 while (i < avn)
751 {
752 unsigned int elt, elnum;
753
754 switch (arg_types[i]->type)
755 {
756 case FFI_TYPE_SINT8:
757 case FFI_TYPE_UINT8:
758 #ifndef __LITTLE_ENDIAN__
759 avalue[i] = (char *) pst + 7;
760 pst++;
761 break;
762 #endif
763
764 case FFI_TYPE_SINT16:
765 case FFI_TYPE_UINT16:
766 #ifndef __LITTLE_ENDIAN__
767 avalue[i] = (char *) pst + 6;
768 pst++;
769 break;
770 #endif
771
772 case FFI_TYPE_SINT32:
773 case FFI_TYPE_UINT32:
774 #ifndef __LITTLE_ENDIAN__
775 avalue[i] = (char *) pst + 4;
776 pst++;
777 break;
778 #endif
779
780 case FFI_TYPE_SINT64:
781 case FFI_TYPE_UINT64:
782 case FFI_TYPE_POINTER:
783 avalue[i] = pst;
784 pst++;
785 break;
786
787 case FFI_TYPE_STRUCT:
788 if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
789 {
790 align = arg_types[i]->alignment;
791 if (align > 16)
792 align = 16;
793 if (align > 1)
794 pst = (unsigned long *) ALIGN ((size_t) pst, align);
795 }
796 elt = 0;
797 #if _CALL_ELF == 2
798 elt = discover_homogeneous_aggregate (arg_types[i], &elnum);
799 #endif
800 if (elt)
801 {
802 union {
803 void *v;
804 unsigned long *ul;
805 float *f;
806 double *d;
807 size_t p;
808 } to, from;
809
810 /* Repackage the aggregate from its parts. The
811 aggregate size is not greater than the space taken by
812 the registers so store back to the register/parameter
813 save arrays. */
814 if (pfr + elnum <= end_pfr)
815 to.v = pfr;
816 else
817 to.v = pst;
818
819 avalue[i] = to.v;
820 from.ul = pst;
821 if (elt == FFI_TYPE_FLOAT)
822 {
823 do
824 {
825 if (pfr < end_pfr && i < nfixedargs)
826 {
827 *to.f = (float) pfr->d;
828 pfr++;
829 }
830 else
831 *to.f = *from.f;
832 to.f++;
833 from.f++;
834 }
835 while (--elnum != 0);
836 }
837 else
838 {
839 do
840 {
841 if (pfr < end_pfr && i < nfixedargs)
842 {
843 *to.d = pfr->d;
844 pfr++;
845 }
846 else
847 *to.d = *from.d;
848 to.d++;
849 from.d++;
850 }
851 while (--elnum != 0);
852 }
853 }
854 else
855 {
856 #ifndef __LITTLE_ENDIAN__
857 /* Structures with size less than eight bytes are passed
858 left-padded. */
859 if (arg_types[i]->size < 8)
860 avalue[i] = (char *) pst + 8 - arg_types[i]->size;
861 else
862 #endif
863 avalue[i] = pst;
864 }
865 pst += (arg_types[i]->size + 7) / 8;
866 break;
867
868 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
869 case FFI_TYPE_LONGDOUBLE:
870 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
871 {
872 if (pfr + 1 < end_pfr && i + 1 < nfixedargs)
873 {
874 avalue[i] = pfr;
875 pfr += 2;
876 }
877 else
878 {
879 if (pfr < end_pfr && i < nfixedargs)
880 {
881 /* Passed partly in f13 and partly on the stack.
882 Move it all to the stack. */
883 *pst = *(unsigned long *) pfr;
884 pfr++;
885 }
886 avalue[i] = pst;
887 }
888 pst += 2;
889 break;
890 }
891 /* Fall through. */
892 #endif
893 case FFI_TYPE_DOUBLE:
894 /* On the outgoing stack all values are aligned to 8 */
895 /* there are 13 64bit floating point registers */
896
897 if (pfr < end_pfr && i < nfixedargs)
898 {
899 avalue[i] = pfr;
900 pfr++;
901 }
902 else
903 avalue[i] = pst;
904 pst++;
905 break;
906
907 case FFI_TYPE_FLOAT:
908 if (pfr < end_pfr && i < nfixedargs)
909 {
910 /* Float values are stored as doubles in the
911 ffi_closure_LINUX64 code. Fix them here. */
912 pfr->f = (float) pfr->d;
913 avalue[i] = pfr;
914 pfr++;
915 }
916 else
917 avalue[i] = pst;
918 pst++;
919 break;
920
921 default:
922 FFI_ASSERT (0);
923 }
924
925 i++;
926 }
927
928
929 (closure->fun) (cif, rvalue, avalue, closure->user_data);
930
931 /* Tell ffi_closure_LINUX64 how to perform return type promotions. */
932 if ((cif->flags & FLAG_RETURNS_SMST) != 0)
933 {
934 if ((cif->flags & FLAG_RETURNS_FP) == 0)
935 return FFI_V2_TYPE_SMALL_STRUCT + cif->rtype->size - 1;
936 else if ((cif->flags & FLAG_RETURNS_64BITS) != 0)
937 return FFI_V2_TYPE_DOUBLE_HOMOG;
938 else
939 return FFI_V2_TYPE_FLOAT_HOMOG;
940 }
941 return cif->rtype->type;
942 }
943 #endif
944