1 /* -----------------------------------------------------------------------
2 ffi64.c - Copyright (c) 2011, 2018 Anthony Green
3 Copyright (c) 2013 The Written Word, Inc.
4 Copyright (c) 2008, 2010 Red Hat, Inc.
5 Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
6
7 x86-64 Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdarg.h>
35 #include <stdint.h>
36 #include "internal64.h"
37
38 #ifdef __x86_64__
39
40 #define MAX_GPR_REGS 6
41 #define MAX_SSE_REGS 8
42
43 #if defined(__INTEL_COMPILER)
44 #include "xmmintrin.h"
45 #define UINT128 __m128
46 #else
47 #if defined(__SUNPRO_C)
48 #include <sunmedia_types.h>
49 #define UINT128 __m128i
50 #else
51 #define UINT128 __int128_t
52 #endif
53 #endif
54
55 union big_int_union
56 {
57 UINT32 i32;
58 UINT64 i64;
59 UINT128 i128;
60 };
61
62 struct register_args
63 {
64 /* Registers for argument passing. */
65 UINT64 gpr[MAX_GPR_REGS];
66 union big_int_union sse[MAX_SSE_REGS];
67 UINT64 rax; /* ssecount */
68 UINT64 r10; /* static chain */
69 };
70
71 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
72 void *raddr, void (*fnaddr)(void)) FFI_HIDDEN;
73
74 /* All reference to register classes here is identical to the code in
75 gcc/config/i386/i386.c. Do *not* change one without the other. */
76
77 /* Register class used for passing given 64bit part of the argument.
78 These represent classes as documented by the PS ABI, with the
79 exception of SSESF, SSEDF classes, that are basically SSE class,
80 just gcc will use SF or DFmode move instead of DImode to avoid
81 reformatting penalties.
82
83 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
84 whenever possible (upper half does contain padding). */
85 enum x86_64_reg_class
86 {
87 X86_64_NO_CLASS,
88 X86_64_INTEGER_CLASS,
89 X86_64_INTEGERSI_CLASS,
90 X86_64_SSE_CLASS,
91 X86_64_SSESF_CLASS,
92 X86_64_SSEDF_CLASS,
93 X86_64_SSEUP_CLASS,
94 X86_64_X87_CLASS,
95 X86_64_X87UP_CLASS,
96 X86_64_COMPLEX_X87_CLASS,
97 X86_64_MEMORY_CLASS
98 };
99
100 #define MAX_CLASSES 4
101
102 #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
103
104 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
105 of this code is to classify each 8bytes of incoming argument by the register
106 class and assign registers accordingly. */
107
108 /* Return the union class of CLASS1 and CLASS2.
109 See the x86-64 PS ABI for details. */
110
111 static enum x86_64_reg_class
merge_classes(enum x86_64_reg_class class1,enum x86_64_reg_class class2)112 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
113 {
114 /* Rule #1: If both classes are equal, this is the resulting class. */
115 if (class1 == class2)
116 return class1;
117
118 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
119 the other class. */
120 if (class1 == X86_64_NO_CLASS)
121 return class2;
122 if (class2 == X86_64_NO_CLASS)
123 return class1;
124
125 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
126 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
127 return X86_64_MEMORY_CLASS;
128
129 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
130 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
131 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
132 return X86_64_INTEGERSI_CLASS;
133 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
134 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
135 return X86_64_INTEGER_CLASS;
136
137 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
138 MEMORY is used. */
139 if (class1 == X86_64_X87_CLASS
140 || class1 == X86_64_X87UP_CLASS
141 || class1 == X86_64_COMPLEX_X87_CLASS
142 || class2 == X86_64_X87_CLASS
143 || class2 == X86_64_X87UP_CLASS
144 || class2 == X86_64_COMPLEX_X87_CLASS)
145 return X86_64_MEMORY_CLASS;
146
147 /* Rule #6: Otherwise class SSE is used. */
148 return X86_64_SSE_CLASS;
149 }
150
151 /* Classify the argument of type TYPE and mode MODE.
152 CLASSES will be filled by the register class used to pass each word
153 of the operand. The number of words is returned. In case the parameter
154 should be passed in memory, 0 is returned. As a special case for zero
155 sized containers, classes[0] will be NO_CLASS and 1 is returned.
156
157 See the x86-64 PS ABI for details.
158 */
159 static size_t
classify_argument(ffi_type * type,enum x86_64_reg_class classes[],size_t byte_offset)160 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
161 size_t byte_offset)
162 {
163 switch (type->type)
164 {
165 case FFI_TYPE_UINT8:
166 case FFI_TYPE_SINT8:
167 case FFI_TYPE_UINT16:
168 case FFI_TYPE_SINT16:
169 case FFI_TYPE_UINT32:
170 case FFI_TYPE_SINT32:
171 case FFI_TYPE_UINT64:
172 case FFI_TYPE_SINT64:
173 case FFI_TYPE_POINTER:
174 do_integer:
175 {
176 size_t size = byte_offset + type->size;
177
178 if (size <= 4)
179 {
180 classes[0] = X86_64_INTEGERSI_CLASS;
181 return 1;
182 }
183 else if (size <= 8)
184 {
185 classes[0] = X86_64_INTEGER_CLASS;
186 return 1;
187 }
188 else if (size <= 12)
189 {
190 classes[0] = X86_64_INTEGER_CLASS;
191 classes[1] = X86_64_INTEGERSI_CLASS;
192 return 2;
193 }
194 else if (size <= 16)
195 {
196 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
197 return 2;
198 }
199 else
200 FFI_ASSERT (0);
201 }
202 case FFI_TYPE_FLOAT:
203 if (!(byte_offset % 8))
204 classes[0] = X86_64_SSESF_CLASS;
205 else
206 classes[0] = X86_64_SSE_CLASS;
207 return 1;
208 case FFI_TYPE_DOUBLE:
209 classes[0] = X86_64_SSEDF_CLASS;
210 return 1;
211 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
212 case FFI_TYPE_LONGDOUBLE:
213 classes[0] = X86_64_X87_CLASS;
214 classes[1] = X86_64_X87UP_CLASS;
215 return 2;
216 #endif
217 case FFI_TYPE_STRUCT:
218 {
219 const size_t UNITS_PER_WORD = 8;
220 size_t words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
221 ffi_type **ptr;
222 unsigned int i;
223 enum x86_64_reg_class subclasses[MAX_CLASSES];
224
225 /* If the struct is larger than 32 bytes, pass it on the stack. */
226 if (type->size > 32)
227 return 0;
228
229 for (i = 0; i < words; i++)
230 classes[i] = X86_64_NO_CLASS;
231
232 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
233 signalize memory class, so handle it as special case. */
234 if (!words)
235 {
236 case FFI_TYPE_VOID:
237 classes[0] = X86_64_NO_CLASS;
238 return 1;
239 }
240
241 /* Merge the fields of structure. */
242 for (ptr = type->elements; *ptr != NULL; ptr++)
243 {
244 size_t num;
245
246 byte_offset = FFI_ALIGN (byte_offset, (*ptr)->alignment);
247
248 num = classify_argument (*ptr, subclasses, byte_offset % 8);
249 if (num == 0)
250 return 0;
251 for (i = 0; i < num; i++)
252 {
253 size_t pos = byte_offset / 8;
254 classes[i + pos] =
255 merge_classes (subclasses[i], classes[i + pos]);
256 }
257
258 byte_offset += (*ptr)->size;
259 }
260
261 if (words > 2)
262 {
263 /* When size > 16 bytes, if the first one isn't
264 X86_64_SSE_CLASS or any other ones aren't
265 X86_64_SSEUP_CLASS, everything should be passed in
266 memory. */
267 if (classes[0] != X86_64_SSE_CLASS)
268 return 0;
269
270 for (i = 1; i < words; i++)
271 if (classes[i] != X86_64_SSEUP_CLASS)
272 return 0;
273 }
274
275 /* Final merger cleanup. */
276 for (i = 0; i < words; i++)
277 {
278 /* If one class is MEMORY, everything should be passed in
279 memory. */
280 if (classes[i] == X86_64_MEMORY_CLASS)
281 return 0;
282
283 /* The X86_64_SSEUP_CLASS should be always preceded by
284 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
285 if (i > 1 && classes[i] == X86_64_SSEUP_CLASS
286 && classes[i - 1] != X86_64_SSE_CLASS
287 && classes[i - 1] != X86_64_SSEUP_CLASS)
288 {
289 /* The first one should never be X86_64_SSEUP_CLASS. */
290 FFI_ASSERT (i != 0);
291 classes[i] = X86_64_SSE_CLASS;
292 }
293
294 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
295 everything should be passed in memory. */
296 if (i > 1 && classes[i] == X86_64_X87UP_CLASS
297 && (classes[i - 1] != X86_64_X87_CLASS))
298 {
299 /* The first one should never be X86_64_X87UP_CLASS. */
300 FFI_ASSERT (i != 0);
301 return 0;
302 }
303 }
304 return words;
305 }
306 case FFI_TYPE_COMPLEX:
307 {
308 ffi_type *inner = type->elements[0];
309 switch (inner->type)
310 {
311 case FFI_TYPE_INT:
312 case FFI_TYPE_UINT8:
313 case FFI_TYPE_SINT8:
314 case FFI_TYPE_UINT16:
315 case FFI_TYPE_SINT16:
316 case FFI_TYPE_UINT32:
317 case FFI_TYPE_SINT32:
318 case FFI_TYPE_UINT64:
319 case FFI_TYPE_SINT64:
320 goto do_integer;
321
322 case FFI_TYPE_FLOAT:
323 classes[0] = X86_64_SSE_CLASS;
324 if (byte_offset % 8)
325 {
326 classes[1] = X86_64_SSESF_CLASS;
327 return 2;
328 }
329 return 1;
330 case FFI_TYPE_DOUBLE:
331 classes[0] = classes[1] = X86_64_SSEDF_CLASS;
332 return 2;
333 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
334 case FFI_TYPE_LONGDOUBLE:
335 classes[0] = X86_64_COMPLEX_X87_CLASS;
336 return 1;
337 #endif
338 }
339 }
340 }
341 abort();
342 }
343
344 /* Examine the argument and return set number of register required in each
345 class. Return zero iff parameter should be passed in memory, otherwise
346 the number of registers. */
347
348 static size_t
examine_argument(ffi_type * type,enum x86_64_reg_class classes[MAX_CLASSES],_Bool in_return,int * pngpr,int * pnsse)349 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
350 _Bool in_return, int *pngpr, int *pnsse)
351 {
352 size_t n;
353 unsigned int i;
354 int ngpr, nsse;
355
356 n = classify_argument (type, classes, 0);
357 if (n == 0)
358 return 0;
359
360 ngpr = nsse = 0;
361 for (i = 0; i < n; ++i)
362 switch (classes[i])
363 {
364 case X86_64_INTEGER_CLASS:
365 case X86_64_INTEGERSI_CLASS:
366 ngpr++;
367 break;
368 case X86_64_SSE_CLASS:
369 case X86_64_SSESF_CLASS:
370 case X86_64_SSEDF_CLASS:
371 nsse++;
372 break;
373 case X86_64_NO_CLASS:
374 case X86_64_SSEUP_CLASS:
375 break;
376 case X86_64_X87_CLASS:
377 case X86_64_X87UP_CLASS:
378 case X86_64_COMPLEX_X87_CLASS:
379 return in_return != 0;
380 default:
381 abort ();
382 }
383
384 *pngpr = ngpr;
385 *pnsse = nsse;
386
387 return n;
388 }
389
390 /* Perform machine dependent cif processing. */
391
392 #ifndef __ILP32__
393 extern ffi_status
394 ffi_prep_cif_machdep_efi64(ffi_cif *cif);
395 #endif
396
397 ffi_status FFI_HIDDEN
ffi_prep_cif_machdep(ffi_cif * cif)398 ffi_prep_cif_machdep (ffi_cif *cif)
399 {
400 int gprcount, ssecount, i, avn, ngpr, nsse;
401 unsigned flags;
402 enum x86_64_reg_class classes[MAX_CLASSES];
403 size_t bytes, n, rtype_size;
404 ffi_type *rtype;
405
406 #ifndef __ILP32__
407 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
408 return ffi_prep_cif_machdep_efi64(cif);
409 #endif
410 if (cif->abi != FFI_UNIX64)
411 return FFI_BAD_ABI;
412
413 gprcount = ssecount = 0;
414
415 rtype = cif->rtype;
416 rtype_size = rtype->size;
417 switch (rtype->type)
418 {
419 case FFI_TYPE_VOID:
420 flags = UNIX64_RET_VOID;
421 break;
422 case FFI_TYPE_UINT8:
423 flags = UNIX64_RET_UINT8;
424 break;
425 case FFI_TYPE_SINT8:
426 flags = UNIX64_RET_SINT8;
427 break;
428 case FFI_TYPE_UINT16:
429 flags = UNIX64_RET_UINT16;
430 break;
431 case FFI_TYPE_SINT16:
432 flags = UNIX64_RET_SINT16;
433 break;
434 case FFI_TYPE_UINT32:
435 flags = UNIX64_RET_UINT32;
436 break;
437 case FFI_TYPE_INT:
438 case FFI_TYPE_SINT32:
439 flags = UNIX64_RET_SINT32;
440 break;
441 case FFI_TYPE_UINT64:
442 case FFI_TYPE_SINT64:
443 flags = UNIX64_RET_INT64;
444 break;
445 case FFI_TYPE_POINTER:
446 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
447 break;
448 case FFI_TYPE_FLOAT:
449 flags = UNIX64_RET_XMM32;
450 break;
451 case FFI_TYPE_DOUBLE:
452 flags = UNIX64_RET_XMM64;
453 break;
454 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
455 case FFI_TYPE_LONGDOUBLE:
456 flags = UNIX64_RET_X87;
457 break;
458 #endif
459 case FFI_TYPE_STRUCT:
460 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
461 if (n == 0)
462 {
463 /* The return value is passed in memory. A pointer to that
464 memory is the first argument. Allocate a register for it. */
465 gprcount++;
466 /* We don't have to do anything in asm for the return. */
467 flags = UNIX64_RET_VOID | UNIX64_FLAG_RET_IN_MEM;
468 }
469 else
470 {
471 _Bool sse0 = SSE_CLASS_P (classes[0]);
472
473 if (rtype_size == 4 && sse0)
474 flags = UNIX64_RET_XMM32;
475 else if (rtype_size == 8)
476 flags = sse0 ? UNIX64_RET_XMM64 : UNIX64_RET_INT64;
477 else
478 {
479 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
480 if (sse0 && sse1)
481 flags = UNIX64_RET_ST_XMM0_XMM1;
482 else if (sse0)
483 flags = UNIX64_RET_ST_XMM0_RAX;
484 else if (sse1)
485 flags = UNIX64_RET_ST_RAX_XMM0;
486 else
487 flags = UNIX64_RET_ST_RAX_RDX;
488 flags |= rtype_size << UNIX64_SIZE_SHIFT;
489 }
490 }
491 break;
492 case FFI_TYPE_COMPLEX:
493 switch (rtype->elements[0]->type)
494 {
495 case FFI_TYPE_UINT8:
496 case FFI_TYPE_SINT8:
497 case FFI_TYPE_UINT16:
498 case FFI_TYPE_SINT16:
499 case FFI_TYPE_INT:
500 case FFI_TYPE_UINT32:
501 case FFI_TYPE_SINT32:
502 case FFI_TYPE_UINT64:
503 case FFI_TYPE_SINT64:
504 flags = UNIX64_RET_ST_RAX_RDX | ((unsigned) rtype_size << UNIX64_SIZE_SHIFT);
505 break;
506 case FFI_TYPE_FLOAT:
507 flags = UNIX64_RET_XMM64;
508 break;
509 case FFI_TYPE_DOUBLE:
510 flags = UNIX64_RET_ST_XMM0_XMM1 | (16 << UNIX64_SIZE_SHIFT);
511 break;
512 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
513 case FFI_TYPE_LONGDOUBLE:
514 flags = UNIX64_RET_X87_2;
515 break;
516 #endif
517 default:
518 return FFI_BAD_TYPEDEF;
519 }
520 break;
521 default:
522 return FFI_BAD_TYPEDEF;
523 }
524
525 /* Go over all arguments and determine the way they should be passed.
526 If it's in a register and there is space for it, let that be so. If
527 not, add it's size to the stack byte count. */
528 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
529 {
530 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
531 || gprcount + ngpr > MAX_GPR_REGS
532 || ssecount + nsse > MAX_SSE_REGS)
533 {
534 long align = cif->arg_types[i]->alignment;
535
536 if (align < 8)
537 align = 8;
538
539 bytes = FFI_ALIGN (bytes, align);
540 bytes += cif->arg_types[i]->size;
541 }
542 else
543 {
544 gprcount += ngpr;
545 ssecount += nsse;
546 }
547 }
548 if (ssecount)
549 flags |= UNIX64_FLAG_XMM_ARGS;
550
551 cif->flags = flags;
552 cif->bytes = (unsigned) FFI_ALIGN (bytes, 8);
553
554 return FFI_OK;
555 }
556
557 static void
ffi_call_int(ffi_cif * cif,void (* fn)(void),void * rvalue,void ** avalue,void * closure)558 ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue,
559 void **avalue, void *closure)
560 {
561 enum x86_64_reg_class classes[MAX_CLASSES];
562 char *stack, *argp;
563 ffi_type **arg_types;
564 int gprcount, ssecount, ngpr, nsse, i, avn, flags;
565 struct register_args *reg_args;
566
567 /* Can't call 32-bit mode from 64-bit mode. */
568 FFI_ASSERT (cif->abi == FFI_UNIX64);
569
570 /* If the return value is a struct and we don't have a return value
571 address then we need to make one. Otherwise we can ignore it. */
572 flags = cif->flags;
573 if (rvalue == NULL)
574 {
575 if (flags & UNIX64_FLAG_RET_IN_MEM)
576 rvalue = alloca (cif->rtype->size);
577 else
578 flags = UNIX64_RET_VOID;
579 }
580
581 /* Allocate the space for the arguments, plus 4 words of temp space. */
582 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
583 reg_args = (struct register_args *) stack;
584 argp = stack + sizeof (struct register_args);
585
586 reg_args->r10 = (uintptr_t) closure;
587
588 gprcount = ssecount = 0;
589
590 /* If the return value is passed in memory, add the pointer as the
591 first integer argument. */
592 if (flags & UNIX64_FLAG_RET_IN_MEM)
593 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
594
595 avn = cif->nargs;
596 arg_types = cif->arg_types;
597
598 for (i = 0; i < avn; ++i)
599 {
600 size_t n, size = arg_types[i]->size;
601
602 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
603 if (n == 0
604 || gprcount + ngpr > MAX_GPR_REGS
605 || ssecount + nsse > MAX_SSE_REGS)
606 {
607 long align = arg_types[i]->alignment;
608
609 /* Stack arguments are *always* at least 8 byte aligned. */
610 if (align < 8)
611 align = 8;
612
613 /* Pass this argument in memory. */
614 argp = (void *) FFI_ALIGN (argp, align);
615 memcpy (argp, avalue[i], size);
616 argp += size;
617 }
618 else
619 {
620 /* The argument is passed entirely in registers. */
621 char *a = (char *) avalue[i];
622 unsigned int j;
623
624 for (j = 0; j < n; j++, a += 8, size -= 8)
625 {
626 switch (classes[j])
627 {
628 case X86_64_NO_CLASS:
629 case X86_64_SSEUP_CLASS:
630 break;
631 case X86_64_INTEGER_CLASS:
632 case X86_64_INTEGERSI_CLASS:
633 /* Sign-extend integer arguments passed in general
634 purpose registers, to cope with the fact that
635 LLVM incorrectly assumes that this will be done
636 (the x86-64 PS ABI does not specify this). */
637 switch (arg_types[i]->type)
638 {
639 case FFI_TYPE_SINT8:
640 reg_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
641 break;
642 case FFI_TYPE_SINT16:
643 reg_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
644 break;
645 case FFI_TYPE_SINT32:
646 reg_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
647 break;
648 default:
649 reg_args->gpr[gprcount] = 0;
650 memcpy (®_args->gpr[gprcount], a, size);
651 }
652 gprcount++;
653 break;
654 case X86_64_SSE_CLASS:
655 case X86_64_SSEDF_CLASS:
656 memcpy (®_args->sse[ssecount++].i64, a, sizeof(UINT64));
657 break;
658 case X86_64_SSESF_CLASS:
659 memcpy (®_args->sse[ssecount++].i32, a, sizeof(UINT32));
660 break;
661 default:
662 abort();
663 }
664 }
665 }
666 }
667 reg_args->rax = ssecount;
668
669 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
670 flags, rvalue, fn);
671 }
672
673 #ifndef __ILP32__
674 extern void
675 ffi_call_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue);
676 #endif
677
678 void
ffi_call(ffi_cif * cif,void (* fn)(void),void * rvalue,void ** avalue)679 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
680 {
681 #ifndef __ILP32__
682 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
683 {
684 ffi_call_efi64(cif, fn, rvalue, avalue);
685 return;
686 }
687 #endif
688 ffi_call_int (cif, fn, rvalue, avalue, NULL);
689 }
690
691 #ifndef __ILP32__
692 extern void
693 ffi_call_go_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue,
694 void **avalue, void *closure);
695 #endif
696
697 void
ffi_call_go(ffi_cif * cif,void (* fn)(void),void * rvalue,void ** avalue,void * closure)698 ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
699 void **avalue, void *closure)
700 {
701 #ifndef __ILP32__
702 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
703 {
704 ffi_call_go_efi64(cif, fn, rvalue, avalue, closure);
705 return;
706 }
707 #endif
708 ffi_call_int (cif, fn, rvalue, avalue, closure);
709 }
710
711
712 extern void ffi_closure_unix64(void) FFI_HIDDEN;
713 extern void ffi_closure_unix64_sse(void) FFI_HIDDEN;
714
715 #ifndef __ILP32__
716 extern ffi_status
717 ffi_prep_closure_loc_efi64(ffi_closure* closure,
718 ffi_cif* cif,
719 void (*fun)(ffi_cif*, void*, void**, void*),
720 void *user_data,
721 void *codeloc);
722 #endif
723
724 ffi_status
ffi_prep_closure_loc(ffi_closure * closure,ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data,void * codeloc)725 ffi_prep_closure_loc (ffi_closure* closure,
726 ffi_cif* cif,
727 void (*fun)(ffi_cif*, void*, void**, void*),
728 void *user_data,
729 void *codeloc)
730 {
731 static const unsigned char trampoline[16] = {
732 /* leaq -0x7(%rip),%r10 # 0x0 */
733 0x4c, 0x8d, 0x15, 0xf9, 0xff, 0xff, 0xff,
734 /* jmpq *0x3(%rip) # 0x10 */
735 0xff, 0x25, 0x03, 0x00, 0x00, 0x00,
736 /* nopl (%rax) */
737 0x0f, 0x1f, 0x00
738 };
739 void (*dest)(void);
740 char *tramp = closure->tramp;
741
742 #ifndef __ILP32__
743 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
744 return ffi_prep_closure_loc_efi64(closure, cif, fun, user_data, codeloc);
745 #endif
746 if (cif->abi != FFI_UNIX64)
747 return FFI_BAD_ABI;
748
749 if (cif->flags & UNIX64_FLAG_XMM_ARGS)
750 dest = ffi_closure_unix64_sse;
751 else
752 dest = ffi_closure_unix64;
753
754 memcpy (tramp, trampoline, sizeof(trampoline));
755 *(UINT64 *)(tramp + 16) = (uintptr_t)dest;
756
757 closure->cif = cif;
758 closure->fun = fun;
759 closure->user_data = user_data;
760
761 return FFI_OK;
762 }
763
764 int FFI_HIDDEN
ffi_closure_unix64_inner(ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data,void * rvalue,struct register_args * reg_args,char * argp)765 ffi_closure_unix64_inner(ffi_cif *cif,
766 void (*fun)(ffi_cif*, void*, void**, void*),
767 void *user_data,
768 void *rvalue,
769 struct register_args *reg_args,
770 char *argp)
771 {
772 void **avalue;
773 ffi_type **arg_types;
774 long i, avn;
775 int gprcount, ssecount, ngpr, nsse;
776 int flags;
777
778 avn = cif->nargs;
779 flags = cif->flags;
780 avalue = alloca(avn * sizeof(void *));
781 gprcount = ssecount = 0;
782
783 if (flags & UNIX64_FLAG_RET_IN_MEM)
784 {
785 /* On return, %rax will contain the address that was passed
786 by the caller in %rdi. */
787 void *r = (void *)(uintptr_t)reg_args->gpr[gprcount++];
788 *(void **)rvalue = r;
789 rvalue = r;
790 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
791 }
792
793 arg_types = cif->arg_types;
794 for (i = 0; i < avn; ++i)
795 {
796 enum x86_64_reg_class classes[MAX_CLASSES];
797 size_t n;
798
799 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
800 if (n == 0
801 || gprcount + ngpr > MAX_GPR_REGS
802 || ssecount + nsse > MAX_SSE_REGS)
803 {
804 long align = arg_types[i]->alignment;
805
806 /* Stack arguments are *always* at least 8 byte aligned. */
807 if (align < 8)
808 align = 8;
809
810 /* Pass this argument in memory. */
811 argp = (void *) FFI_ALIGN (argp, align);
812 avalue[i] = argp;
813 argp += arg_types[i]->size;
814 }
815 /* If the argument is in a single register, or two consecutive
816 integer registers, then we can use that address directly. */
817 else if (n == 1
818 || (n == 2 && !(SSE_CLASS_P (classes[0])
819 || SSE_CLASS_P (classes[1]))))
820 {
821 /* The argument is in a single register. */
822 if (SSE_CLASS_P (classes[0]))
823 {
824 avalue[i] = ®_args->sse[ssecount];
825 ssecount += n;
826 }
827 else
828 {
829 avalue[i] = ®_args->gpr[gprcount];
830 gprcount += n;
831 }
832 }
833 /* Otherwise, allocate space to make them consecutive. */
834 else
835 {
836 char *a = alloca (16);
837 unsigned int j;
838
839 avalue[i] = a;
840 for (j = 0; j < n; j++, a += 8)
841 {
842 if (SSE_CLASS_P (classes[j]))
843 memcpy (a, ®_args->sse[ssecount++], 8);
844 else
845 memcpy (a, ®_args->gpr[gprcount++], 8);
846 }
847 }
848 }
849
850 /* Invoke the closure. */
851 fun (cif, rvalue, avalue, user_data);
852
853 /* Tell assembly how to perform return type promotions. */
854 return flags;
855 }
856
857 extern void ffi_go_closure_unix64(void) FFI_HIDDEN;
858 extern void ffi_go_closure_unix64_sse(void) FFI_HIDDEN;
859
860 #ifndef __ILP32__
861 extern ffi_status
862 ffi_prep_go_closure_efi64(ffi_go_closure* closure, ffi_cif* cif,
863 void (*fun)(ffi_cif*, void*, void**, void*));
864 #endif
865
866 ffi_status
ffi_prep_go_closure(ffi_go_closure * closure,ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *))867 ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif,
868 void (*fun)(ffi_cif*, void*, void**, void*))
869 {
870 #ifndef __ILP32__
871 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
872 return ffi_prep_go_closure_efi64(closure, cif, fun);
873 #endif
874 if (cif->abi != FFI_UNIX64)
875 return FFI_BAD_ABI;
876
877 closure->tramp = (cif->flags & UNIX64_FLAG_XMM_ARGS
878 ? ffi_go_closure_unix64_sse
879 : ffi_go_closure_unix64);
880 closure->cif = cif;
881 closure->fun = fun;
882
883 return FFI_OK;
884 }
885
886 #endif /* __x86_64__ */
887