1 /* -----------------------------------------------------------------------
2 ffi64.c - Copyright (c) 2013 The Written Word, Inc.
3 Copyright (c) 2011 Anthony Green
4 Copyright (c) 2008, 2010 Red Hat, Inc.
5 Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
6
7 x86-64 Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdarg.h>
35
36 #ifdef __x86_64__
37
38 #define MAX_GPR_REGS 6
39 #define MAX_SSE_REGS 8
40
41 #if defined(__INTEL_COMPILER)
42 #include "xmmintrin.h"
43 #define UINT128 __m128
44 #else
45 #if defined(__SUNPRO_C)
46 #include <sunmedia_types.h>
47 #define UINT128 __m128i
48 #else
49 #define UINT128 __int128_t
50 #endif
51 #endif
52
53 union big_int_union
54 {
55 UINT32 i32;
56 UINT64 i64;
57 UINT128 i128;
58 };
59
60 struct register_args
61 {
62 /* Registers for argument passing. */
63 UINT64 gpr[MAX_GPR_REGS];
64 union big_int_union sse[MAX_SSE_REGS];
65 };
66
67 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
68 void *raddr, void (*fnaddr)(void), unsigned ssecount);
69
70 /* All reference to register classes here is identical to the code in
71 gcc/config/i386/i386.c. Do *not* change one without the other. */
72
73 /* Register class used for passing given 64bit part of the argument.
74 These represent classes as documented by the PS ABI, with the
75 exception of SSESF, SSEDF classes, that are basically SSE class,
76 just gcc will use SF or DFmode move instead of DImode to avoid
77 reformatting penalties.
78
79 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
80 whenever possible (upper half does contain padding). */
81 enum x86_64_reg_class
82 {
83 X86_64_NO_CLASS,
84 X86_64_INTEGER_CLASS,
85 X86_64_INTEGERSI_CLASS,
86 X86_64_SSE_CLASS,
87 X86_64_SSESF_CLASS,
88 X86_64_SSEDF_CLASS,
89 X86_64_SSEUP_CLASS,
90 X86_64_X87_CLASS,
91 X86_64_X87UP_CLASS,
92 X86_64_COMPLEX_X87_CLASS,
93 X86_64_MEMORY_CLASS
94 };
95
96 #define MAX_CLASSES 4
97
98 #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
99
100 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
101 of this code is to classify each 8bytes of incoming argument by the register
102 class and assign registers accordingly. */
103
104 /* Return the union class of CLASS1 and CLASS2.
105 See the x86-64 PS ABI for details. */
106
107 static enum x86_64_reg_class
merge_classes(enum x86_64_reg_class class1,enum x86_64_reg_class class2)108 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
109 {
110 /* Rule #1: If both classes are equal, this is the resulting class. */
111 if (class1 == class2)
112 return class1;
113
114 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
115 the other class. */
116 if (class1 == X86_64_NO_CLASS)
117 return class2;
118 if (class2 == X86_64_NO_CLASS)
119 return class1;
120
121 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
122 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
123 return X86_64_MEMORY_CLASS;
124
125 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
126 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
127 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
128 return X86_64_INTEGERSI_CLASS;
129 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
130 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
131 return X86_64_INTEGER_CLASS;
132
133 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
134 MEMORY is used. */
135 if (class1 == X86_64_X87_CLASS
136 || class1 == X86_64_X87UP_CLASS
137 || class1 == X86_64_COMPLEX_X87_CLASS
138 || class2 == X86_64_X87_CLASS
139 || class2 == X86_64_X87UP_CLASS
140 || class2 == X86_64_COMPLEX_X87_CLASS)
141 return X86_64_MEMORY_CLASS;
142
143 /* Rule #6: Otherwise class SSE is used. */
144 return X86_64_SSE_CLASS;
145 }
146
147 /* Classify the argument of type TYPE and mode MODE.
148 CLASSES will be filled by the register class used to pass each word
149 of the operand. The number of words is returned. In case the parameter
150 should be passed in memory, 0 is returned. As a special case for zero
151 sized containers, classes[0] will be NO_CLASS and 1 is returned.
152
153 See the x86-64 PS ABI for details.
154 */
155 static size_t
classify_argument(ffi_type * type,enum x86_64_reg_class classes[],size_t byte_offset)156 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
157 size_t byte_offset)
158 {
159 switch (type->type)
160 {
161 case FFI_TYPE_UINT8:
162 case FFI_TYPE_SINT8:
163 case FFI_TYPE_UINT16:
164 case FFI_TYPE_SINT16:
165 case FFI_TYPE_UINT32:
166 case FFI_TYPE_SINT32:
167 case FFI_TYPE_UINT64:
168 case FFI_TYPE_SINT64:
169 case FFI_TYPE_POINTER:
170 {
171 size_t size = byte_offset + type->size;
172
173 if (size <= 4)
174 {
175 classes[0] = X86_64_INTEGERSI_CLASS;
176 return 1;
177 }
178 else if (size <= 8)
179 {
180 classes[0] = X86_64_INTEGER_CLASS;
181 return 1;
182 }
183 else if (size <= 12)
184 {
185 classes[0] = X86_64_INTEGER_CLASS;
186 classes[1] = X86_64_INTEGERSI_CLASS;
187 return 2;
188 }
189 else if (size <= 16)
190 {
191 classes[0] = classes[1] = X86_64_INTEGERSI_CLASS;
192 return 2;
193 }
194 else
195 FFI_ASSERT (0);
196 }
197 case FFI_TYPE_FLOAT:
198 if (!(byte_offset % 8))
199 classes[0] = X86_64_SSESF_CLASS;
200 else
201 classes[0] = X86_64_SSE_CLASS;
202 return 1;
203 case FFI_TYPE_DOUBLE:
204 classes[0] = X86_64_SSEDF_CLASS;
205 return 1;
206 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
207 case FFI_TYPE_LONGDOUBLE:
208 classes[0] = X86_64_X87_CLASS;
209 classes[1] = X86_64_X87UP_CLASS;
210 return 2;
211 #endif
212 case FFI_TYPE_STRUCT:
213 {
214 const size_t UNITS_PER_WORD = 8;
215 size_t words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
216 ffi_type **ptr;
217 int i;
218 enum x86_64_reg_class subclasses[MAX_CLASSES];
219
220 /* If the struct is larger than 32 bytes, pass it on the stack. */
221 if (type->size > 32)
222 return 0;
223
224 for (i = 0; i < words; i++)
225 classes[i] = X86_64_NO_CLASS;
226
227 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
228 signalize memory class, so handle it as special case. */
229 if (!words)
230 {
231 classes[0] = X86_64_NO_CLASS;
232 return 1;
233 }
234
235 /* Merge the fields of structure. */
236 for (ptr = type->elements; *ptr != NULL; ptr++)
237 {
238 size_t num;
239
240 byte_offset = ALIGN (byte_offset, (*ptr)->alignment);
241
242 num = classify_argument (*ptr, subclasses, byte_offset % 8);
243 if (num == 0)
244 return 0;
245 for (i = 0; i < num; i++)
246 {
247 size_t pos = byte_offset / 8;
248 classes[i + pos] =
249 merge_classes (subclasses[i], classes[i + pos]);
250 }
251
252 byte_offset += (*ptr)->size;
253 }
254
255 if (words > 2)
256 {
257 /* When size > 16 bytes, if the first one isn't
258 X86_64_SSE_CLASS or any other ones aren't
259 X86_64_SSEUP_CLASS, everything should be passed in
260 memory. */
261 if (classes[0] != X86_64_SSE_CLASS)
262 return 0;
263
264 for (i = 1; i < words; i++)
265 if (classes[i] != X86_64_SSEUP_CLASS)
266 return 0;
267 }
268
269 /* Final merger cleanup. */
270 for (i = 0; i < words; i++)
271 {
272 /* If one class is MEMORY, everything should be passed in
273 memory. */
274 if (classes[i] == X86_64_MEMORY_CLASS)
275 return 0;
276
277 /* The X86_64_SSEUP_CLASS should be always preceded by
278 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
279 if (classes[i] == X86_64_SSEUP_CLASS
280 && classes[i - 1] != X86_64_SSE_CLASS
281 && classes[i - 1] != X86_64_SSEUP_CLASS)
282 {
283 /* The first one should never be X86_64_SSEUP_CLASS. */
284 FFI_ASSERT (i != 0);
285 classes[i] = X86_64_SSE_CLASS;
286 }
287
288 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
289 everything should be passed in memory. */
290 if (classes[i] == X86_64_X87UP_CLASS
291 && (classes[i - 1] != X86_64_X87_CLASS))
292 {
293 /* The first one should never be X86_64_X87UP_CLASS. */
294 FFI_ASSERT (i != 0);
295 return 0;
296 }
297 }
298 return words;
299 }
300
301 default:
302 FFI_ASSERT(0);
303 }
304 return 0; /* Never reached. */
305 }
306
307 /* Examine the argument and return set number of register required in each
308 class. Return zero iff parameter should be passed in memory, otherwise
309 the number of registers. */
310
311 static size_t
examine_argument(ffi_type * type,enum x86_64_reg_class classes[MAX_CLASSES],_Bool in_return,int * pngpr,int * pnsse)312 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
313 _Bool in_return, int *pngpr, int *pnsse)
314 {
315 size_t n;
316 int i, ngpr, nsse;
317
318 n = classify_argument (type, classes, 0);
319 if (n == 0)
320 return 0;
321
322 ngpr = nsse = 0;
323 for (i = 0; i < n; ++i)
324 switch (classes[i])
325 {
326 case X86_64_INTEGER_CLASS:
327 case X86_64_INTEGERSI_CLASS:
328 ngpr++;
329 break;
330 case X86_64_SSE_CLASS:
331 case X86_64_SSESF_CLASS:
332 case X86_64_SSEDF_CLASS:
333 nsse++;
334 break;
335 case X86_64_NO_CLASS:
336 case X86_64_SSEUP_CLASS:
337 break;
338 case X86_64_X87_CLASS:
339 case X86_64_X87UP_CLASS:
340 case X86_64_COMPLEX_X87_CLASS:
341 return in_return != 0;
342 default:
343 abort ();
344 }
345
346 *pngpr = ngpr;
347 *pnsse = nsse;
348
349 return n;
350 }
351
352 /* Perform machine dependent cif processing. */
353
354 ffi_status
ffi_prep_cif_machdep(ffi_cif * cif)355 ffi_prep_cif_machdep (ffi_cif *cif)
356 {
357 int gprcount, ssecount, i, avn, ngpr, nsse, flags;
358 enum x86_64_reg_class classes[MAX_CLASSES];
359 size_t bytes, n;
360
361 gprcount = ssecount = 0;
362
363 flags = cif->rtype->type;
364 if (flags != FFI_TYPE_VOID)
365 {
366 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
367 if (n == 0)
368 {
369 /* The return value is passed in memory. A pointer to that
370 memory is the first argument. Allocate a register for it. */
371 gprcount++;
372 /* We don't have to do anything in asm for the return. */
373 flags = FFI_TYPE_VOID;
374 }
375 else if (flags == FFI_TYPE_STRUCT)
376 {
377 /* Mark which registers the result appears in. */
378 _Bool sse0 = SSE_CLASS_P (classes[0]);
379 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
380 if (sse0 && !sse1)
381 flags |= 1 << 8;
382 else if (!sse0 && sse1)
383 flags |= 1 << 9;
384 else if (sse0 && sse1)
385 flags |= 1 << 10;
386 /* Mark the true size of the structure. */
387 flags |= cif->rtype->size << 12;
388 }
389 }
390
391 /* Go over all arguments and determine the way they should be passed.
392 If it's in a register and there is space for it, let that be so. If
393 not, add it's size to the stack byte count. */
394 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
395 {
396 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
397 || gprcount + ngpr > MAX_GPR_REGS
398 || ssecount + nsse > MAX_SSE_REGS)
399 {
400 long align = cif->arg_types[i]->alignment;
401
402 if (align < 8)
403 align = 8;
404
405 bytes = ALIGN (bytes, align);
406 bytes += cif->arg_types[i]->size;
407 }
408 else
409 {
410 gprcount += ngpr;
411 ssecount += nsse;
412 }
413 }
414 if (ssecount)
415 flags |= 1 << 11;
416 cif->flags = flags;
417 cif->bytes = (unsigned)ALIGN (bytes, 8);
418
419 return FFI_OK;
420 }
421
422 void
ffi_call(ffi_cif * cif,void (* fn)(void),void * rvalue,void ** avalue)423 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
424 {
425 enum x86_64_reg_class classes[MAX_CLASSES];
426 char *stack, *argp;
427 ffi_type **arg_types;
428 int gprcount, ssecount, ngpr, nsse, i, avn;
429 _Bool ret_in_memory;
430 struct register_args *reg_args;
431
432 /* Can't call 32-bit mode from 64-bit mode. */
433 FFI_ASSERT (cif->abi == FFI_UNIX64);
434
435 /* If the return value is a struct and we don't have a return value
436 address then we need to make one. Note the setting of flags to
437 VOID above in ffi_prep_cif_machdep. */
438 ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT
439 && (cif->flags & 0xff) == FFI_TYPE_VOID);
440 if (rvalue == NULL && ret_in_memory)
441 rvalue = alloca (cif->rtype->size);
442
443 /* Allocate the space for the arguments, plus 4 words of temp space. */
444 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
445 reg_args = (struct register_args *) stack;
446 argp = stack + sizeof (struct register_args);
447
448 gprcount = ssecount = 0;
449
450 /* If the return value is passed in memory, add the pointer as the
451 first integer argument. */
452 if (ret_in_memory)
453 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
454
455 avn = cif->nargs;
456 arg_types = cif->arg_types;
457
458 for (i = 0; i < avn; ++i)
459 {
460 size_t n, size = arg_types[i]->size;
461
462 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
463 if (n == 0
464 || gprcount + ngpr > MAX_GPR_REGS
465 || ssecount + nsse > MAX_SSE_REGS)
466 {
467 long align = arg_types[i]->alignment;
468
469 /* Stack arguments are *always* at least 8 byte aligned. */
470 if (align < 8)
471 align = 8;
472
473 /* Pass this argument in memory. */
474 argp = (void *) ALIGN (argp, align);
475 memcpy (argp, avalue[i], size);
476 argp += size;
477 }
478 else
479 {
480 /* The argument is passed entirely in registers. */
481 char *a = (char *) avalue[i];
482 int j;
483
484 for (j = 0; j < n; j++, a += 8, size -= 8)
485 {
486 switch (classes[j])
487 {
488 case X86_64_INTEGER_CLASS:
489 case X86_64_INTEGERSI_CLASS:
490 /* Sign-extend integer arguments passed in general
491 purpose registers, to cope with the fact that
492 LLVM incorrectly assumes that this will be done
493 (the x86-64 PS ABI does not specify this). */
494 switch (arg_types[i]->type)
495 {
496 case FFI_TYPE_SINT8:
497 *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
498 break;
499 case FFI_TYPE_SINT16:
500 *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
501 break;
502 case FFI_TYPE_SINT32:
503 *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
504 break;
505 default:
506 reg_args->gpr[gprcount] = 0;
507 memcpy (®_args->gpr[gprcount], a, size < 8 ? size : 8);
508 }
509 gprcount++;
510 break;
511 case X86_64_SSE_CLASS:
512 case X86_64_SSEDF_CLASS:
513 reg_args->sse[ssecount++].i64 = *(UINT64 *) a;
514 break;
515 case X86_64_SSESF_CLASS:
516 reg_args->sse[ssecount++].i32 = *(UINT32 *) a;
517 break;
518 default:
519 abort();
520 }
521 }
522 }
523 }
524
525 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
526 cif->flags, rvalue, fn, ssecount);
527 }
528
529
530 extern void ffi_closure_unix64(void);
531
532 ffi_status
ffi_prep_closure_loc(ffi_closure * closure,ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data,void * codeloc)533 ffi_prep_closure_loc (ffi_closure* closure,
534 ffi_cif* cif,
535 void (*fun)(ffi_cif*, void*, void**, void*),
536 void *user_data,
537 void *codeloc)
538 {
539 volatile unsigned short *tramp;
540
541 /* Sanity check on the cif ABI. */
542 {
543 int abi = cif->abi;
544 if (UNLIKELY (! (abi > FFI_FIRST_ABI && abi < FFI_LAST_ABI)))
545 return FFI_BAD_ABI;
546 }
547
548 tramp = (volatile unsigned short *) &closure->tramp[0];
549
550 tramp[0] = 0xbb49; /* mov <code>, %r11 */
551 *((unsigned long long * volatile) &tramp[1])
552 = (unsigned long) ffi_closure_unix64;
553 tramp[5] = 0xba49; /* mov <data>, %r10 */
554 *((unsigned long long * volatile) &tramp[6])
555 = (unsigned long) codeloc;
556
557 /* Set the carry bit iff the function uses any sse registers.
558 This is clc or stc, together with the first byte of the jmp. */
559 tramp[10] = cif->flags & (1 << 11) ? 0x49f9 : 0x49f8;
560
561 tramp[11] = 0xe3ff; /* jmp *%r11 */
562
563 closure->cif = cif;
564 closure->fun = fun;
565 closure->user_data = user_data;
566
567 return FFI_OK;
568 }
569
570 int
ffi_closure_unix64_inner(ffi_closure * closure,void * rvalue,struct register_args * reg_args,char * argp)571 ffi_closure_unix64_inner(ffi_closure *closure, void *rvalue,
572 struct register_args *reg_args, char *argp)
573 {
574 ffi_cif *cif;
575 void **avalue;
576 ffi_type **arg_types;
577 long i, avn;
578 int gprcount, ssecount, ngpr, nsse;
579 int ret;
580
581 cif = closure->cif;
582 avalue = alloca(cif->nargs * sizeof(void *));
583 gprcount = ssecount = 0;
584
585 ret = cif->rtype->type;
586 if (ret != FFI_TYPE_VOID)
587 {
588 enum x86_64_reg_class classes[MAX_CLASSES];
589 size_t n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
590 if (n == 0)
591 {
592 /* The return value goes in memory. Arrange for the closure
593 return value to go directly back to the original caller. */
594 rvalue = (void *) (unsigned long) reg_args->gpr[gprcount++];
595 /* We don't have to do anything in asm for the return. */
596 ret = FFI_TYPE_VOID;
597 }
598 else if (ret == FFI_TYPE_STRUCT && n == 2)
599 {
600 /* Mark which register the second word of the structure goes in. */
601 _Bool sse0 = SSE_CLASS_P (classes[0]);
602 _Bool sse1 = SSE_CLASS_P (classes[1]);
603 if (!sse0 && sse1)
604 ret |= 1 << 8;
605 else if (sse0 && !sse1)
606 ret |= 1 << 9;
607 }
608 }
609
610 avn = cif->nargs;
611 arg_types = cif->arg_types;
612
613 for (i = 0; i < avn; ++i)
614 {
615 enum x86_64_reg_class classes[MAX_CLASSES];
616 size_t n;
617
618 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
619 if (n == 0
620 || gprcount + ngpr > MAX_GPR_REGS
621 || ssecount + nsse > MAX_SSE_REGS)
622 {
623 long align = arg_types[i]->alignment;
624
625 /* Stack arguments are *always* at least 8 byte aligned. */
626 if (align < 8)
627 align = 8;
628
629 /* Pass this argument in memory. */
630 argp = (void *) ALIGN (argp, align);
631 avalue[i] = argp;
632 argp += arg_types[i]->size;
633 }
634 /* If the argument is in a single register, or two consecutive
635 integer registers, then we can use that address directly. */
636 else if (n == 1
637 || (n == 2 && !(SSE_CLASS_P (classes[0])
638 || SSE_CLASS_P (classes[1]))))
639 {
640 /* The argument is in a single register. */
641 if (SSE_CLASS_P (classes[0]))
642 {
643 avalue[i] = ®_args->sse[ssecount];
644 ssecount += n;
645 }
646 else
647 {
648 avalue[i] = ®_args->gpr[gprcount];
649 gprcount += n;
650 }
651 }
652 /* Otherwise, allocate space to make them consecutive. */
653 else
654 {
655 char *a = alloca (16);
656 int j;
657
658 avalue[i] = a;
659 for (j = 0; j < n; j++, a += 8)
660 {
661 if (SSE_CLASS_P (classes[j]))
662 memcpy (a, ®_args->sse[ssecount++], 8);
663 else
664 memcpy (a, ®_args->gpr[gprcount++], 8);
665 }
666 }
667 }
668
669 /* Invoke the closure. */
670 closure->fun (cif, rvalue, avalue, closure->user_data);
671
672 /* Tell assembly how to perform return type promotions. */
673 return ret;
674 }
675
676 #endif /* __x86_64__ */
677