1 #ifdef __x86_64__
2
3 /* -----------------------------------------------------------------------
4 x86-ffi64.c - Copyright (c) 2002 Bo Thorsen <bo@suse.de>
5
6 x86-64 Foreign Function Interface
7
8 Permission is hereby granted, free of charge, to any person obtaining
9 a copy of this software and associated documentation files (the
10 ``Software''), to deal in the Software without restriction, including
11 without limitation the rights to use, copy, modify, merge, publish,
12 distribute, sublicense, and/or sell copies of the Software, and to
13 permit persons to whom the Software is furnished to do so, subject to
14 the following conditions:
15
16 The above copyright notice and this permission notice shall be included
17 in all copies or substantial portions of the Software.
18
19 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
22 IN NO EVENT SHALL CYGNUS SOLUTIONS BE LIABLE FOR ANY CLAIM, DAMAGES OR
23 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
24 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
25 OTHER DEALINGS IN THE SOFTWARE.
26 ----------------------------------------------------------------------- */
27
28 #include <ffi.h>
29 #include <ffi_common.h>
30
31 #include <stdlib.h>
32 #include <stdarg.h>
33
34 #define MAX_GPR_REGS 6
35 #define MAX_SSE_REGS 8
36
37 typedef struct RegisterArgs {
38 /* Registers for argument passing. */
39 UINT64 gpr[MAX_GPR_REGS];
40 __int128_t sse[MAX_SSE_REGS];
41 } RegisterArgs;
42
43 extern void
44 ffi_call_unix64(
45 void* args,
46 unsigned long bytes,
47 unsigned flags,
48 void* raddr,
49 void (*fnaddr)(void),
50 unsigned ssecount);
51
52 /* All reference to register classes here is identical to the code in
53 gcc/config/i386/i386.c. Do *not* change one without the other. */
54
55 /* Register class used for passing given 64bit part of the argument.
56 These represent classes as documented by the PS ABI, with the exception
57 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
58 use SF or DFmode move instead of DImode to avoid reformating penalties.
59
60 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
61 whenever possible (upper half does contain padding). */
62 enum x86_64_reg_class
63 {
64 X86_64_NO_CLASS,
65 X86_64_INTEGER_CLASS,
66 X86_64_INTEGERSI_CLASS,
67 X86_64_SSE_CLASS,
68 X86_64_SSESF_CLASS,
69 X86_64_SSEDF_CLASS,
70 X86_64_SSEUP_CLASS,
71 X86_64_X87_CLASS,
72 X86_64_X87UP_CLASS,
73 X86_64_COMPLEX_X87_CLASS,
74 X86_64_MEMORY_CLASS
75 };
76
77 #define MAX_CLASSES 4
78 #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
79
80 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
81 of this code is to classify each 8bytes of incoming argument by the register
82 class and assign registers accordingly. */
83
84 /* Return the union class of CLASS1 and CLASS2.
85 See the x86-64 PS ABI for details. */
86 static enum x86_64_reg_class
merge_classes(enum x86_64_reg_class class1,enum x86_64_reg_class class2)87 merge_classes(
88 enum x86_64_reg_class class1,
89 enum x86_64_reg_class class2)
90 {
91 /* Rule #1: If both classes are equal, this is the resulting class. */
92 if (class1 == class2)
93 return class1;
94
95 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
96 the other class. */
97 if (class1 == X86_64_NO_CLASS)
98 return class2;
99
100 if (class2 == X86_64_NO_CLASS)
101 return class1;
102
103 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
104 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
105 return X86_64_MEMORY_CLASS;
106
107 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
108 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
109 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
110 return X86_64_INTEGERSI_CLASS;
111
112 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
113 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
114 return X86_64_INTEGER_CLASS;
115
116 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
117 MEMORY is used. */
118 if (class1 == X86_64_X87_CLASS
119 || class1 == X86_64_X87UP_CLASS
120 || class1 == X86_64_COMPLEX_X87_CLASS
121 || class2 == X86_64_X87_CLASS
122 || class2 == X86_64_X87UP_CLASS
123 || class2 == X86_64_COMPLEX_X87_CLASS)
124 return X86_64_MEMORY_CLASS;
125
126 /* Rule #6: Otherwise class SSE is used. */
127 return X86_64_SSE_CLASS;
128 }
129
130 /* Classify the argument of type TYPE and mode MODE.
131 CLASSES will be filled by the register class used to pass each word
132 of the operand. The number of words is returned. In case the parameter
133 should be passed in memory, 0 is returned. As a special case for zero
134 sized containers, classes[0] will be NO_CLASS and 1 is returned.
135
136 See the x86-64 PS ABI for details. */
137
138 static int
classify_argument(ffi_type * type,enum x86_64_reg_class classes[],size_t byte_offset)139 classify_argument(
140 ffi_type* type,
141 enum x86_64_reg_class classes[],
142 size_t byte_offset)
143 {
144 switch (type->type)
145 {
146 case FFI_TYPE_UINT8:
147 case FFI_TYPE_SINT8:
148 case FFI_TYPE_UINT16:
149 case FFI_TYPE_SINT16:
150 case FFI_TYPE_UINT32:
151 case FFI_TYPE_SINT32:
152 case FFI_TYPE_UINT64:
153 case FFI_TYPE_SINT64:
154 case FFI_TYPE_POINTER:
155 #if 0
156 if (byte_offset + type->size <= 4)
157 classes[0] = X86_64_INTEGERSI_CLASS;
158 else
159 classes[0] = X86_64_INTEGER_CLASS;
160
161 return 1;
162 #else
163 {
164 int size = byte_offset + type->size;
165
166 if (size <= 4)
167 {
168 classes[0] = X86_64_INTEGERSI_CLASS;
169 return 1;
170 }
171 else if (size <= 8)
172 {
173 classes[0] = X86_64_INTEGER_CLASS;
174 return 1;
175 }
176 else if (size <= 12)
177 {
178 classes[0] = X86_64_INTEGER_CLASS;
179 classes[1] = X86_64_INTEGERSI_CLASS;
180 return 2;
181 }
182 else if (size <= 16)
183 {
184 classes[0] = classes[1] = X86_64_INTEGERSI_CLASS;
185 return 2;
186 }
187 else
188 FFI_ASSERT (0);
189 }
190 #endif
191
192 case FFI_TYPE_FLOAT:
193 if (byte_offset == 0)
194 classes[0] = X86_64_SSESF_CLASS;
195 else
196 classes[0] = X86_64_SSE_CLASS;
197
198 return 1;
199
200 case FFI_TYPE_DOUBLE:
201 classes[0] = X86_64_SSEDF_CLASS;
202 return 1;
203
204 case FFI_TYPE_LONGDOUBLE:
205 classes[0] = X86_64_X87_CLASS;
206 classes[1] = X86_64_X87UP_CLASS;
207 return 2;
208
209 case FFI_TYPE_STRUCT:
210 {
211 ffi_type** ptr;
212 int i;
213 enum x86_64_reg_class subclasses[MAX_CLASSES];
214 const int UNITS_PER_WORD = 8;
215 int words =
216 (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
217
218 /* If the struct is larger than 16 bytes, pass it on the stack. */
219 if (type->size > 16)
220 return 0;
221
222 for (i = 0; i < words; i++)
223 classes[i] = X86_64_NO_CLASS;
224
225 /* Merge the fields of structure. */
226 for (ptr = type->elements; *ptr != NULL; ptr++)
227 {
228 int num, pos;
229
230 byte_offset = ALIGN(byte_offset, (*ptr)->alignment);
231
232 num = classify_argument(*ptr, subclasses, byte_offset % 8);
233
234 if (num == 0)
235 return 0;
236
237 pos = byte_offset / 8;
238
239 for (i = 0; i < num; i++)
240 {
241 classes[i + pos] =
242 merge_classes(subclasses[i], classes[i + pos]);
243 }
244
245 byte_offset += (*ptr)->size;
246 }
247
248 if (words > 2)
249 {
250 /* When size > 16 bytes, if the first one isn't
251 X86_64_SSE_CLASS or any other ones aren't
252 X86_64_SSEUP_CLASS, everything should be passed in
253 memory. */
254 if (classes[0] != X86_64_SSE_CLASS)
255 return 0;
256
257 for (i = 1; i < words; i++)
258 if (classes[i] != X86_64_SSEUP_CLASS)
259 return 0;
260 }
261
262
263 /* Final merger cleanup. */
264 for (i = 0; i < words; i++)
265 {
266 /* If one class is MEMORY, everything should be passed in
267 memory. */
268 if (classes[i] == X86_64_MEMORY_CLASS)
269 return 0;
270
271 /* The X86_64_SSEUP_CLASS should be always preceded by
272 X86_64_SSE_CLASS. */
273 if (classes[i] == X86_64_SSEUP_CLASS
274 && classes[i - 1] != X86_64_SSE_CLASS
275 && classes[i - 1] != X86_64_SSEUP_CLASS)
276 {
277 FFI_ASSERT(i != 0);
278 classes[i] = X86_64_SSE_CLASS;
279 }
280
281 /* X86_64_X87UP_CLASS should be preceded by X86_64_X87_CLASS. */
282 if (classes[i] == X86_64_X87UP_CLASS
283 && classes[i - 1] != X86_64_X87_CLASS)
284 {
285 FFI_ASSERT(i != 0);
286 classes[i] = X86_64_SSE_CLASS;
287 }
288 }
289
290 return words;
291 }
292
293 default:
294 FFI_ASSERT(0);
295 }
296
297 return 0; /* Never reached. */
298 }
299
300 /* Examine the argument and return set number of register required in each
301 class. Return zero if parameter should be passed in memory, otherwise
302 the number of registers. */
303 static int
examine_argument(ffi_type * type,enum x86_64_reg_class classes[MAX_CLASSES],_Bool in_return,int * pngpr,int * pnsse)304 examine_argument(
305 ffi_type* type,
306 enum x86_64_reg_class classes[MAX_CLASSES],
307 _Bool in_return,
308 int* pngpr,
309 int* pnsse)
310 {
311 int n = classify_argument(type, classes, 0);
312 int ngpr = 0;
313 int nsse = 0;
314 int i;
315
316 if (n == 0)
317 return 0;
318
319 for (i = 0; i < n; ++i)
320 {
321 switch (classes[i])
322 {
323 case X86_64_INTEGER_CLASS:
324 case X86_64_INTEGERSI_CLASS:
325 ngpr++;
326 break;
327
328 case X86_64_SSE_CLASS:
329 case X86_64_SSESF_CLASS:
330 case X86_64_SSEDF_CLASS:
331 nsse++;
332 break;
333
334 case X86_64_NO_CLASS:
335 case X86_64_SSEUP_CLASS:
336 break;
337
338 case X86_64_X87_CLASS:
339 case X86_64_X87UP_CLASS:
340 case X86_64_COMPLEX_X87_CLASS:
341 return in_return != 0;
342
343 default:
344 abort();
345 }
346 }
347
348 *pngpr = ngpr;
349 *pnsse = nsse;
350
351 return n;
352 }
353
354 /* Perform machine dependent cif processing. */
355 ffi_status
ffi_prep_cif_machdep(ffi_cif * cif)356 ffi_prep_cif_machdep(
357 ffi_cif* cif)
358 {
359 int gprcount = 0;
360 int ssecount = 0;
361 int flags = cif->rtype->type;
362 int i, avn, n, ngpr, nsse;
363 enum x86_64_reg_class classes[MAX_CLASSES];
364 size_t bytes;
365
366 if (flags != FFI_TYPE_VOID)
367 {
368 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
369
370 if (n == 0)
371 {
372 /* The return value is passed in memory. A pointer to that
373 memory is the first argument. Allocate a register for it. */
374 gprcount++;
375
376 /* We don't have to do anything in asm for the return. */
377 flags = FFI_TYPE_VOID;
378 }
379 else if (flags == FFI_TYPE_STRUCT)
380 {
381 /* Mark which registers the result appears in. */
382 _Bool sse0 = SSE_CLASS_P(classes[0]);
383 _Bool sse1 = n == 2 && SSE_CLASS_P(classes[1]);
384
385 if (sse0 && !sse1)
386 flags |= 1 << 8;
387 else if (!sse0 && sse1)
388 flags |= 1 << 9;
389 else if (sse0 && sse1)
390 flags |= 1 << 10;
391
392 /* Mark the true size of the structure. */
393 flags |= cif->rtype->size << 12;
394 }
395 }
396
397 /* Go over all arguments and determine the way they should be passed.
398 If it's in a register and there is space for it, let that be so. If
399 not, add it's size to the stack byte count. */
400 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
401 {
402 if (examine_argument(cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
403 || gprcount + ngpr > MAX_GPR_REGS
404 || ssecount + nsse > MAX_SSE_REGS)
405 {
406 long align = cif->arg_types[i]->alignment;
407
408 if (align < 8)
409 align = 8;
410
411 bytes = ALIGN(bytes, align);
412 bytes += cif->arg_types[i]->size;
413 }
414 else
415 {
416 gprcount += ngpr;
417 ssecount += nsse;
418 }
419 }
420
421 if (ssecount)
422 flags |= 1 << 11;
423
424 cif->flags = flags;
425 cif->bytes = bytes;
426 cif->bytes = ALIGN(bytes,8);
427
428 return FFI_OK;
429 }
430
431 void
ffi_call(ffi_cif * cif,void (* fn)(void),void * rvalue,void ** avalue)432 ffi_call(
433 ffi_cif* cif,
434 void (*fn)(void),
435 void* rvalue,
436 void** avalue)
437 {
438 enum x86_64_reg_class classes[MAX_CLASSES];
439 char* stack;
440 char* argp;
441 ffi_type** arg_types;
442 int gprcount, ssecount, ngpr, nsse, i, avn;
443 _Bool ret_in_memory;
444 RegisterArgs* reg_args;
445
446 /* Can't call 32-bit mode from 64-bit mode. */
447 FFI_ASSERT(cif->abi == FFI_UNIX64);
448
449 /* If the return value is a struct and we don't have a return value
450 address then we need to make one. Note the setting of flags to
451 VOID above in ffi_prep_cif_machdep. */
452 ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT
453 && (cif->flags & 0xff) == FFI_TYPE_VOID);
454
455 if (rvalue == NULL && ret_in_memory)
456 rvalue = alloca (cif->rtype->size);
457
458 /* Allocate the space for the arguments, plus 4 words of temp space. */
459 stack = alloca(sizeof(RegisterArgs) + cif->bytes + 4 * 8);
460 reg_args = (RegisterArgs*)stack;
461 argp = stack + sizeof(RegisterArgs);
462
463 gprcount = ssecount = 0;
464
465 /* If the return value is passed in memory, add the pointer as the
466 first integer argument. */
467 if (ret_in_memory)
468 reg_args->gpr[gprcount++] = (long) rvalue;
469
470 avn = cif->nargs;
471 arg_types = cif->arg_types;
472
473 for (i = 0; i < avn; ++i)
474 {
475 size_t size = arg_types[i]->size;
476 int n;
477
478 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
479
480 if (n == 0
481 || gprcount + ngpr > MAX_GPR_REGS
482 || ssecount + nsse > MAX_SSE_REGS)
483 {
484 long align = arg_types[i]->alignment;
485
486 /* Stack arguments are *always* at least 8 byte aligned. */
487 if (align < 8)
488 align = 8;
489
490 /* Pass this argument in memory. */
491 argp = (void *) ALIGN (argp, align);
492 memcpy (argp, avalue[i], size);
493 argp += size;
494 }
495 else
496 { /* The argument is passed entirely in registers. */
497 char *a = (char *) avalue[i];
498 int j;
499
500 for (j = 0; j < n; j++, a += 8, size -= 8)
501 {
502 switch (classes[j])
503 {
504 case X86_64_INTEGER_CLASS:
505 case X86_64_INTEGERSI_CLASS:
506 reg_args->gpr[gprcount] = 0;
507 switch (arg_types[i]->type) {
508 case FFI_TYPE_SINT8:
509 {
510 int8_t shortval = *(int8_t*)a;
511 int64_t actval = (int64_t)shortval;
512 reg_args->gpr[gprcount] = actval;
513 /*memcpy (®_args->gpr[gprcount], &actval, 8);*/
514 break;
515 }
516
517 case FFI_TYPE_SINT16:
518 {
519 int16_t shortval = *(int16_t*)a;
520 int64_t actval = (int64_t)shortval;
521 memcpy (®_args->gpr[gprcount], &actval, 8);
522 break;
523 }
524
525 case FFI_TYPE_SINT32:
526 {
527 int32_t shortval = *(int32_t*)a;
528 int64_t actval = (int64_t)shortval;
529 memcpy (®_args->gpr[gprcount], &actval, 8);
530 break;
531 }
532
533 case FFI_TYPE_UINT8:
534 {
535 u_int8_t shortval = *(u_int8_t*)a;
536 u_int64_t actval = (u_int64_t)shortval;
537 /*memcpy (®_args->gpr[gprcount], &actval, 8);*/
538 reg_args->gpr[gprcount] = actval;
539 break;
540 }
541
542 case FFI_TYPE_UINT16:
543 {
544 u_int16_t shortval = *(u_int16_t*)a;
545 u_int64_t actval = (u_int64_t)shortval;
546 memcpy (®_args->gpr[gprcount], &actval, 8);
547 break;
548 }
549
550 case FFI_TYPE_UINT32:
551 {
552 u_int32_t shortval = *(u_int32_t*)a;
553 u_int64_t actval = (u_int64_t)shortval;
554 memcpy (®_args->gpr[gprcount], &actval, 8);
555 break;
556 }
557
558 default:
559 //memcpy (®_args->gpr[gprcount], a, size < 8 ? size : 8);
560 reg_args->gpr[gprcount] = *(int64_t*)a;
561 }
562 gprcount++;
563 break;
564
565 case X86_64_SSE_CLASS:
566 case X86_64_SSEDF_CLASS:
567 reg_args->sse[ssecount++] = *(UINT64 *) a;
568 break;
569
570 case X86_64_SSESF_CLASS:
571 reg_args->sse[ssecount++] = *(UINT32 *) a;
572 break;
573
574 default:
575 abort();
576 }
577 }
578 }
579 }
580
581 ffi_call_unix64 (stack, cif->bytes + sizeof(RegisterArgs),
582 cif->flags, rvalue, fn, ssecount);
583 }
584
585 extern void ffi_closure_unix64(void);
586
587 ffi_status
ffi_prep_closure(ffi_closure * closure,ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data)588 ffi_prep_closure(
589 ffi_closure* closure,
590 ffi_cif* cif,
591 void (*fun)(ffi_cif*, void*, void**, void*),
592 void* user_data)
593 {
594 volatile unsigned short* tramp;
595
596 if (cif->abi != FFI_UNIX64)
597 return FFI_BAD_ABI;
598
599 tramp = (volatile unsigned short*)&closure->tramp[0];
600
601 tramp[0] = 0xbb49; /* mov <code>, %r11 */
602 *(void* volatile*)&tramp[1] = ffi_closure_unix64;
603 tramp[5] = 0xba49; /* mov <data>, %r10 */
604 *(void* volatile*)&tramp[6] = closure;
605
606 /* Set the carry bit if the function uses any sse registers.
607 This is clc or stc, together with the first byte of the jmp. */
608 tramp[10] = cif->flags & (1 << 11) ? 0x49f9 : 0x49f8;
609 tramp[11] = 0xe3ff; /* jmp *%r11 */
610
611 closure->cif = cif;
612 closure->fun = fun;
613 closure->user_data = user_data;
614
615 return FFI_OK;
616 }
617
618 #pragma clang diagnostic push
619 #pragma clang diagnostic ignored "-Wmissing-prototypes"
620 int
ffi_closure_unix64_inner(ffi_closure * closure,void * rvalue,RegisterArgs * reg_args,char * argp)621 ffi_closure_unix64_inner(
622 ffi_closure* closure,
623 void* rvalue,
624 RegisterArgs* reg_args,
625 char* argp)
626 #pragma clang diagnostic pop
627 {
628 ffi_cif* cif = closure->cif;
629 void** avalue = alloca(cif->nargs * sizeof(void *));
630 ffi_type** arg_types;
631 long i, avn;
632 int gprcount = 0;
633 int ssecount = 0;
634 int ngpr, nsse;
635 int ret;
636
637 ret = cif->rtype->type;
638
639 if (ret != FFI_TYPE_VOID)
640 {
641 enum x86_64_reg_class classes[MAX_CLASSES];
642 int n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
643
644 if (n == 0)
645 {
646 /* The return value goes in memory. Arrange for the closure
647 return value to go directly back to the original caller. */
648 rvalue = (void *) reg_args->gpr[gprcount++];
649
650 /* We don't have to do anything in asm for the return. */
651 ret = FFI_TYPE_VOID;
652 }
653 else if (ret == FFI_TYPE_STRUCT && n == 2)
654 {
655 /* Mark which register the second word of the structure goes in. */
656 _Bool sse0 = SSE_CLASS_P (classes[0]);
657 _Bool sse1 = SSE_CLASS_P (classes[1]);
658
659 if (!sse0 && sse1)
660 ret |= 1 << 8;
661 else if (sse0 && !sse1)
662 ret |= 1 << 9;
663 }
664 }
665
666 avn = cif->nargs;
667 arg_types = cif->arg_types;
668
669 for (i = 0; i < avn; ++i)
670 {
671 enum x86_64_reg_class classes[MAX_CLASSES];
672 int n;
673
674 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
675
676 if (n == 0
677 || gprcount + ngpr > MAX_GPR_REGS
678 || ssecount + nsse > MAX_SSE_REGS)
679 {
680 long align = arg_types[i]->alignment;
681
682 /* Stack arguments are *always* at least 8 byte aligned. */
683 if (align < 8)
684 align = 8;
685
686 /* Pass this argument in memory. */
687 argp = (void *) ALIGN (argp, align);
688 avalue[i] = argp;
689 argp += arg_types[i]->size;
690 }
691
692 #if !defined(X86_DARWIN)
693 /* If the argument is in a single register, or two consecutive
694 registers, then we can use that address directly. */
695 else if (n == 1 || (n == 2 &&
696 SSE_CLASS_P (classes[0]) == SSE_CLASS_P (classes[1])))
697 {
698 // The argument is in a single register.
699 if (SSE_CLASS_P (classes[0]))
700 {
701 avalue[i] = ®_args->sse[ssecount];
702 ssecount += n;
703 }
704 else
705 {
706 avalue[i] = ®_args->gpr[gprcount];
707 gprcount += n;
708 }
709 }
710 #endif
711
712 /* Otherwise, allocate space to make them consecutive. */
713 else
714 {
715 char *a = alloca (16);
716 int j;
717
718 avalue[i] = a;
719
720 for (j = 0; j < n; j++, a += 8)
721 {
722 if (SSE_CLASS_P (classes[j]))
723 memcpy (a, ®_args->sse[ssecount++], 8);
724 else
725 memcpy (a, ®_args->gpr[gprcount++], 8);
726 }
727 }
728 }
729
730 /* Invoke the closure. */
731 closure->fun (cif, rvalue, avalue, closure->user_data);
732
733 /* Tell assembly how to perform return type promotions. */
734 return ret;
735 }
736
737 #endif /* __x86_64__ */
738