1 /* -----------------------------------------------------------------------
2 ffi64.c - Copyright (c) 2011, 2018 Anthony Green
3 Copyright (c) 2013 The Written Word, Inc.
4 Copyright (c) 2008, 2010 Red Hat, Inc.
5 Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
6
7 x86-64 Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdarg.h>
35 #include <stdint.h>
36 #include <tramp.h>
37 #include "internal64.h"
38
39 #ifdef __x86_64__
40
41 #define MAX_GPR_REGS 6
42 #define MAX_SSE_REGS 8
43
44 #if defined(__INTEL_COMPILER)
45 #include "xmmintrin.h"
46 #define UINT128 __m128
47 #else
48 #if defined(__SUNPRO_C)
49 #include <sunmedia_types.h>
50 #define UINT128 __m128i
51 #else
52 #define UINT128 __int128_t
53 #endif
54 #endif
55
56 union big_int_union
57 {
58 UINT32 i32;
59 UINT64 i64;
60 UINT128 i128;
61 };
62
63 struct register_args
64 {
65 /* Registers for argument passing. */
66 UINT64 gpr[MAX_GPR_REGS];
67 union big_int_union sse[MAX_SSE_REGS];
68 UINT64 rax; /* ssecount */
69 UINT64 r10; /* static chain */
70 };
71
72 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
73 void *raddr, void (*fnaddr)(void)) FFI_HIDDEN;
74
75 /* All reference to register classes here is identical to the code in
76 gcc/config/i386/i386.c. Do *not* change one without the other. */
77
78 /* Register class used for passing given 64bit part of the argument.
79 These represent classes as documented by the PS ABI, with the
80 exception of SSESF, SSEDF classes, that are basically SSE class,
81 just gcc will use SF or DFmode move instead of DImode to avoid
82 reformatting penalties.
83
84 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
85 whenever possible (upper half does contain padding). */
86 enum x86_64_reg_class
87 {
88 X86_64_NO_CLASS,
89 X86_64_INTEGER_CLASS,
90 X86_64_INTEGERSI_CLASS,
91 X86_64_SSE_CLASS,
92 X86_64_SSESF_CLASS,
93 X86_64_SSEDF_CLASS,
94 X86_64_SSEUP_CLASS,
95 X86_64_X87_CLASS,
96 X86_64_X87UP_CLASS,
97 X86_64_COMPLEX_X87_CLASS,
98 X86_64_MEMORY_CLASS
99 };
100
101 #define MAX_CLASSES 4
102
103 #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
104
105 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
106 of this code is to classify each 8bytes of incoming argument by the register
107 class and assign registers accordingly. */
108
109 /* Return the union class of CLASS1 and CLASS2.
110 See the x86-64 PS ABI for details. */
111
112 static enum x86_64_reg_class
113 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
114 {
115 /* Rule #1: If both classes are equal, this is the resulting class. */
116 if (class1 == class2)
117 return class1;
118
119 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
120 the other class. */
121 if (class1 == X86_64_NO_CLASS)
122 return class2;
123 if (class2 == X86_64_NO_CLASS)
124 return class1;
125
126 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
127 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
128 return X86_64_MEMORY_CLASS;
129
130 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
131 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
132 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
133 return X86_64_INTEGERSI_CLASS;
134 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
135 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
136 return X86_64_INTEGER_CLASS;
137
138 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
139 MEMORY is used. */
140 if (class1 == X86_64_X87_CLASS
141 || class1 == X86_64_X87UP_CLASS
142 || class1 == X86_64_COMPLEX_X87_CLASS
143 || class2 == X86_64_X87_CLASS
144 || class2 == X86_64_X87UP_CLASS
145 || class2 == X86_64_COMPLEX_X87_CLASS)
146 return X86_64_MEMORY_CLASS;
147
148 /* Rule #6: Otherwise class SSE is used. */
149 return X86_64_SSE_CLASS;
150 }
151
152 /* Classify the argument of type TYPE and mode MODE.
153 CLASSES will be filled by the register class used to pass each word
154 of the operand. The number of words is returned. In case the parameter
155 should be passed in memory, 0 is returned. As a special case for zero
156 sized containers, classes[0] will be NO_CLASS and 1 is returned.
157
158 See the x86-64 PS ABI for details.
159 */
160 static size_t
161 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
162 size_t byte_offset)
163 {
164 switch (type->type)
165 {
166 case FFI_TYPE_UINT8:
167 case FFI_TYPE_SINT8:
168 case FFI_TYPE_UINT16:
169 case FFI_TYPE_SINT16:
170 case FFI_TYPE_UINT32:
171 case FFI_TYPE_SINT32:
172 case FFI_TYPE_UINT64:
173 case FFI_TYPE_SINT64:
174 case FFI_TYPE_POINTER:
175 do_integer:
176 {
177 size_t size = byte_offset + type->size;
178
179 if (size <= 4)
180 {
181 classes[0] = X86_64_INTEGERSI_CLASS;
182 return 1;
183 }
184 else if (size <= 8)
185 {
186 classes[0] = X86_64_INTEGER_CLASS;
187 return 1;
188 }
189 else if (size <= 12)
190 {
191 classes[0] = X86_64_INTEGER_CLASS;
192 classes[1] = X86_64_INTEGERSI_CLASS;
193 return 2;
194 }
195 else if (size <= 16)
196 {
197 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
198 return 2;
199 }
200 else
201 FFI_ASSERT (0);
202 }
203 case FFI_TYPE_FLOAT:
204 if (!(byte_offset % 8))
205 classes[0] = X86_64_SSESF_CLASS;
206 else
207 classes[0] = X86_64_SSE_CLASS;
208 return 1;
209 case FFI_TYPE_DOUBLE:
210 classes[0] = X86_64_SSEDF_CLASS;
211 return 1;
212 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
213 case FFI_TYPE_LONGDOUBLE:
214 classes[0] = X86_64_X87_CLASS;
215 classes[1] = X86_64_X87UP_CLASS;
216 return 2;
217 #endif
218 case FFI_TYPE_STRUCT:
219 {
220 const size_t UNITS_PER_WORD = 8;
221 size_t words = (type->size + byte_offset + UNITS_PER_WORD - 1)
222 / UNITS_PER_WORD;
223 ffi_type **ptr;
224 unsigned int i;
225 enum x86_64_reg_class subclasses[MAX_CLASSES];
226
227 /* If the struct is larger than 32 bytes, pass it on the stack. */
228 if (type->size > 32)
229 return 0;
230
231 for (i = 0; i < words; i++)
232 classes[i] = X86_64_NO_CLASS;
233
234 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
235 signalize memory class, so handle it as special case. */
236 if (!words)
237 {
238 case FFI_TYPE_VOID:
239 classes[0] = X86_64_NO_CLASS;
240 return 1;
241 }
242
243 /* Merge the fields of structure. */
244 for (ptr = type->elements; *ptr != NULL; ptr++)
245 {
246 size_t num, pos;
247
248 byte_offset = FFI_ALIGN (byte_offset, (*ptr)->alignment);
249
250 num = classify_argument (*ptr, subclasses, byte_offset % 8);
251 if (num == 0)
252 return 0;
253 pos = byte_offset / 8;
254 for (i = 0; i < num && (i + pos) < words; i++)
255 {
256 size_t pos = byte_offset / 8;
257 classes[i + pos] =
258 merge_classes (subclasses[i], classes[i + pos]);
259 }
260
261 byte_offset += (*ptr)->size;
262 }
263
264 if (words > 2)
265 {
266 /* When size > 16 bytes, if the first one isn't
267 X86_64_SSE_CLASS or any other ones aren't
268 X86_64_SSEUP_CLASS, everything should be passed in
269 memory. */
270 if (classes[0] != X86_64_SSE_CLASS)
271 return 0;
272
273 for (i = 1; i < words; i++)
274 if (classes[i] != X86_64_SSEUP_CLASS)
275 return 0;
276 }
277
278 /* Final merger cleanup. */
279 for (i = 0; i < words; i++)
280 {
281 /* If one class is MEMORY, everything should be passed in
282 memory. */
283 if (classes[i] == X86_64_MEMORY_CLASS)
284 return 0;
285
286 /* The X86_64_SSEUP_CLASS should be always preceded by
287 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
288 if (i > 1 && classes[i] == X86_64_SSEUP_CLASS
289 && classes[i - 1] != X86_64_SSE_CLASS
290 && classes[i - 1] != X86_64_SSEUP_CLASS)
291 {
292 /* The first one should never be X86_64_SSEUP_CLASS. */
293 FFI_ASSERT (i != 0);
294 classes[i] = X86_64_SSE_CLASS;
295 }
296
297 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
298 everything should be passed in memory. */
299 if (i > 1 && classes[i] == X86_64_X87UP_CLASS
300 && (classes[i - 1] != X86_64_X87_CLASS))
301 {
302 /* The first one should never be X86_64_X87UP_CLASS. */
303 FFI_ASSERT (i != 0);
304 return 0;
305 }
306 }
307 return words;
308 }
309 case FFI_TYPE_COMPLEX:
310 {
311 ffi_type *inner = type->elements[0];
312 switch (inner->type)
313 {
314 case FFI_TYPE_INT:
315 case FFI_TYPE_UINT8:
316 case FFI_TYPE_SINT8:
317 case FFI_TYPE_UINT16:
318 case FFI_TYPE_SINT16:
319 case FFI_TYPE_UINT32:
320 case FFI_TYPE_SINT32:
321 case FFI_TYPE_UINT64:
322 case FFI_TYPE_SINT64:
323 goto do_integer;
324
325 case FFI_TYPE_FLOAT:
326 classes[0] = X86_64_SSE_CLASS;
327 if (byte_offset % 8)
328 {
329 classes[1] = X86_64_SSESF_CLASS;
330 return 2;
331 }
332 return 1;
333 case FFI_TYPE_DOUBLE:
334 classes[0] = classes[1] = X86_64_SSEDF_CLASS;
335 return 2;
336 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
337 case FFI_TYPE_LONGDOUBLE:
338 classes[0] = X86_64_COMPLEX_X87_CLASS;
339 return 1;
340 #endif
341 }
342 }
343 }
344 abort();
345 }
346
347 /* Examine the argument and return set number of register required in each
348 class. Return zero iff parameter should be passed in memory, otherwise
349 the number of registers. */
350
351 static size_t
352 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
353 _Bool in_return, int *pngpr, int *pnsse)
354 {
355 size_t n;
356 unsigned int i;
357 int ngpr, nsse;
358
359 n = classify_argument (type, classes, 0);
360 if (n == 0)
361 return 0;
362
363 ngpr = nsse = 0;
364 for (i = 0; i < n; ++i)
365 switch (classes[i])
366 {
367 case X86_64_INTEGER_CLASS:
368 case X86_64_INTEGERSI_CLASS:
369 ngpr++;
370 break;
371 case X86_64_SSE_CLASS:
372 case X86_64_SSESF_CLASS:
373 case X86_64_SSEDF_CLASS:
374 nsse++;
375 break;
376 case X86_64_NO_CLASS:
377 case X86_64_SSEUP_CLASS:
378 break;
379 case X86_64_X87_CLASS:
380 case X86_64_X87UP_CLASS:
381 case X86_64_COMPLEX_X87_CLASS:
382 return in_return != 0;
383 default:
384 abort ();
385 }
386
387 *pngpr = ngpr;
388 *pnsse = nsse;
389
390 return n;
391 }
392
393 /* Perform machine dependent cif processing. */
394
395 #ifndef __ILP32__
396 extern ffi_status
397 ffi_prep_cif_machdep_efi64(ffi_cif *cif);
398 #endif
399
400 ffi_status FFI_HIDDEN
401 ffi_prep_cif_machdep (ffi_cif *cif)
402 {
403 int gprcount, ssecount, i, avn, ngpr, nsse;
404 unsigned flags;
405 enum x86_64_reg_class classes[MAX_CLASSES];
406 size_t bytes, n, rtype_size;
407 ffi_type *rtype;
408
409 #ifndef __ILP32__
410 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
411 return ffi_prep_cif_machdep_efi64(cif);
412 #endif
413 if (cif->abi != FFI_UNIX64)
414 return FFI_BAD_ABI;
415
416 gprcount = ssecount = 0;
417
418 rtype = cif->rtype;
419 rtype_size = rtype->size;
420 switch (rtype->type)
421 {
422 case FFI_TYPE_VOID:
423 flags = UNIX64_RET_VOID;
424 break;
425 case FFI_TYPE_UINT8:
426 flags = UNIX64_RET_UINT8;
427 break;
428 case FFI_TYPE_SINT8:
429 flags = UNIX64_RET_SINT8;
430 break;
431 case FFI_TYPE_UINT16:
432 flags = UNIX64_RET_UINT16;
433 break;
434 case FFI_TYPE_SINT16:
435 flags = UNIX64_RET_SINT16;
436 break;
437 case FFI_TYPE_UINT32:
438 flags = UNIX64_RET_UINT32;
439 break;
440 case FFI_TYPE_INT:
441 case FFI_TYPE_SINT32:
442 flags = UNIX64_RET_SINT32;
443 break;
444 case FFI_TYPE_UINT64:
445 case FFI_TYPE_SINT64:
446 flags = UNIX64_RET_INT64;
447 break;
448 case FFI_TYPE_POINTER:
449 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
450 break;
451 case FFI_TYPE_FLOAT:
452 flags = UNIX64_RET_XMM32;
453 break;
454 case FFI_TYPE_DOUBLE:
455 flags = UNIX64_RET_XMM64;
456 break;
457 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
458 case FFI_TYPE_LONGDOUBLE:
459 flags = UNIX64_RET_X87;
460 break;
461 #endif
462 case FFI_TYPE_STRUCT:
463 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
464 if (n == 0)
465 {
466 /* The return value is passed in memory. A pointer to that
467 memory is the first argument. Allocate a register for it. */
468 gprcount++;
469 /* We don't have to do anything in asm for the return. */
470 flags = UNIX64_RET_VOID | UNIX64_FLAG_RET_IN_MEM;
471 }
472 else
473 {
474 _Bool sse0 = SSE_CLASS_P (classes[0]);
475
476 if (rtype_size == 4 && sse0)
477 flags = UNIX64_RET_XMM32;
478 else if (rtype_size == 8)
479 flags = sse0 ? UNIX64_RET_XMM64 : UNIX64_RET_INT64;
480 else
481 {
482 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
483 if (sse0 && sse1)
484 flags = UNIX64_RET_ST_XMM0_XMM1;
485 else if (sse0)
486 flags = UNIX64_RET_ST_XMM0_RAX;
487 else if (sse1)
488 flags = UNIX64_RET_ST_RAX_XMM0;
489 else
490 flags = UNIX64_RET_ST_RAX_RDX;
491 flags |= rtype_size << UNIX64_SIZE_SHIFT;
492 }
493 }
494 break;
495 case FFI_TYPE_COMPLEX:
496 switch (rtype->elements[0]->type)
497 {
498 case FFI_TYPE_UINT8:
499 case FFI_TYPE_SINT8:
500 case FFI_TYPE_UINT16:
501 case FFI_TYPE_SINT16:
502 case FFI_TYPE_INT:
503 case FFI_TYPE_UINT32:
504 case FFI_TYPE_SINT32:
505 case FFI_TYPE_UINT64:
506 case FFI_TYPE_SINT64:
507 flags = UNIX64_RET_ST_RAX_RDX | ((unsigned) rtype_size << UNIX64_SIZE_SHIFT);
508 break;
509 case FFI_TYPE_FLOAT:
510 flags = UNIX64_RET_XMM64;
511 break;
512 case FFI_TYPE_DOUBLE:
513 flags = UNIX64_RET_ST_XMM0_XMM1 | (16 << UNIX64_SIZE_SHIFT);
514 break;
515 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
516 case FFI_TYPE_LONGDOUBLE:
517 flags = UNIX64_RET_X87_2;
518 break;
519 #endif
520 default:
521 return FFI_BAD_TYPEDEF;
522 }
523 break;
524 default:
525 return FFI_BAD_TYPEDEF;
526 }
527
528 /* Go over all arguments and determine the way they should be passed.
529 If it's in a register and there is space for it, let that be so. If
530 not, add it's size to the stack byte count. */
531 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
532 {
533 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
534 || gprcount + ngpr > MAX_GPR_REGS
535 || ssecount + nsse > MAX_SSE_REGS)
536 {
537 long align = cif->arg_types[i]->alignment;
538
539 if (align < 8)
540 align = 8;
541
542 bytes = FFI_ALIGN (bytes, align);
543 bytes += cif->arg_types[i]->size;
544 }
545 else
546 {
547 gprcount += ngpr;
548 ssecount += nsse;
549 }
550 }
551 if (ssecount)
552 flags |= UNIX64_FLAG_XMM_ARGS;
553
554 cif->flags = flags;
555 cif->bytes = (unsigned) FFI_ALIGN (bytes, 8);
556
557 return FFI_OK;
558 }
559
560 static void
561 ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue,
562 void **avalue, void *closure)
563 {
564 enum x86_64_reg_class classes[MAX_CLASSES];
565 char *stack, *argp;
566 ffi_type **arg_types;
567 int gprcount, ssecount, ngpr, nsse, i, avn, flags;
568 struct register_args *reg_args;
569
570 /* Can't call 32-bit mode from 64-bit mode. */
571 FFI_ASSERT (cif->abi == FFI_UNIX64);
572
573 /* If the return value is a struct and we don't have a return value
574 address then we need to make one. Otherwise we can ignore it. */
575 flags = cif->flags;
576 if (rvalue == NULL)
577 {
578 if (flags & UNIX64_FLAG_RET_IN_MEM)
579 rvalue = alloca (cif->rtype->size);
580 else
581 flags = UNIX64_RET_VOID;
582 }
583
584 /* Allocate the space for the arguments, plus 4 words of temp space. */
585 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
586 reg_args = (struct register_args *) stack;
587 argp = stack + sizeof (struct register_args);
588
589 reg_args->r10 = (uintptr_t) closure;
590
591 gprcount = ssecount = 0;
592
593 /* If the return value is passed in memory, add the pointer as the
594 first integer argument. */
595 if (flags & UNIX64_FLAG_RET_IN_MEM)
596 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
597
598 avn = cif->nargs;
599 arg_types = cif->arg_types;
600
601 for (i = 0; i < avn; ++i)
602 {
603 size_t n, size = arg_types[i]->size;
604
605 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
606 if (n == 0
607 || gprcount + ngpr > MAX_GPR_REGS
608 || ssecount + nsse > MAX_SSE_REGS)
609 {
610 long align = arg_types[i]->alignment;
611
612 /* Stack arguments are *always* at least 8 byte aligned. */
613 if (align < 8)
614 align = 8;
615
616 /* Pass this argument in memory. */
617 argp = (void *) FFI_ALIGN (argp, align);
618 memcpy (argp, avalue[i], size);
619 argp += size;
620 }
621 else
622 {
623 /* The argument is passed entirely in registers. */
624 char *a = (char *) avalue[i];
625 unsigned int j;
626
627 for (j = 0; j < n; j++, a += 8, size -= 8)
628 {
629 switch (classes[j])
630 {
631 case X86_64_NO_CLASS:
632 case X86_64_SSEUP_CLASS:
633 break;
634 case X86_64_INTEGER_CLASS:
635 case X86_64_INTEGERSI_CLASS:
636 /* Sign-extend integer arguments passed in general
637 purpose registers, to cope with the fact that
638 LLVM incorrectly assumes that this will be done
639 (the x86-64 PS ABI does not specify this). */
640 switch (arg_types[i]->type)
641 {
642 case FFI_TYPE_SINT8:
643 reg_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
644 break;
645 case FFI_TYPE_SINT16:
646 reg_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
647 break;
648 case FFI_TYPE_SINT32:
649 reg_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
650 break;
651 default:
652 reg_args->gpr[gprcount] = 0;
653 memcpy (®_args->gpr[gprcount], a, size);
654 }
655 gprcount++;
656 break;
657 case X86_64_SSE_CLASS:
658 case X86_64_SSEDF_CLASS:
659 memcpy (®_args->sse[ssecount++].i64, a, sizeof(UINT64));
660 break;
661 case X86_64_SSESF_CLASS:
662 memcpy (®_args->sse[ssecount++].i32, a, sizeof(UINT32));
663 break;
664 default:
665 abort();
666 }
667 }
668 }
669 }
670 reg_args->rax = ssecount;
671
672 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
673 flags, rvalue, fn);
674 }
675
676 #ifndef __ILP32__
677 extern void
678 ffi_call_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue);
679 #endif
680
681 void
682 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
683 {
684 #ifndef __ILP32__
685 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
686 {
687 ffi_call_efi64(cif, fn, rvalue, avalue);
688 return;
689 }
690 #endif
691 ffi_call_int (cif, fn, rvalue, avalue, NULL);
692 }
693
694 #ifdef FFI_GO_CLOSURES
695
696 #ifndef __ILP32__
697 extern void
698 ffi_call_go_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue,
699 void **avalue, void *closure);
700 #endif
701
702 void
703 ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
704 void **avalue, void *closure)
705 {
706 #ifndef __ILP32__
707 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
708 {
709 ffi_call_go_efi64(cif, fn, rvalue, avalue, closure);
710 return;
711 }
712 #endif
713 ffi_call_int (cif, fn, rvalue, avalue, closure);
714 }
715
716 #endif /* FFI_GO_CLOSURES */
717
718 extern void ffi_closure_unix64(void) FFI_HIDDEN;
719 extern void ffi_closure_unix64_sse(void) FFI_HIDDEN;
720 #if defined(FFI_EXEC_STATIC_TRAMP)
721 extern void ffi_closure_unix64_alt(void) FFI_HIDDEN;
722 extern void ffi_closure_unix64_sse_alt(void) FFI_HIDDEN;
723 #endif
724
725 #ifndef __ILP32__
726 extern ffi_status
727 ffi_prep_closure_loc_efi64(ffi_closure* closure,
728 ffi_cif* cif,
729 void (*fun)(ffi_cif*, void*, void**, void*),
730 void *user_data,
731 void *codeloc);
732 #endif
733
734 ffi_status
735 ffi_prep_closure_loc (ffi_closure* closure,
736 ffi_cif* cif,
737 void (*fun)(ffi_cif*, void*, void**, void*),
738 void *user_data,
739 void *codeloc)
740 {
741 static const unsigned char trampoline[24] = {
742 /* endbr64 */
743 0xf3, 0x0f, 0x1e, 0xfa,
744 /* leaq -0xb(%rip),%r10 # 0x0 */
745 0x4c, 0x8d, 0x15, 0xf5, 0xff, 0xff, 0xff,
746 /* jmpq *0x7(%rip) # 0x18 */
747 0xff, 0x25, 0x07, 0x00, 0x00, 0x00,
748 /* nopl 0(%rax) */
749 0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00
750 };
751 void (*dest)(void);
752 char *tramp = closure->tramp;
753
754 #ifndef __ILP32__
755 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
756 return ffi_prep_closure_loc_efi64(closure, cif, fun, user_data, codeloc);
757 #endif
758 if (cif->abi != FFI_UNIX64)
759 return FFI_BAD_ABI;
760
761 if (cif->flags & UNIX64_FLAG_XMM_ARGS)
762 dest = ffi_closure_unix64_sse;
763 else
764 dest = ffi_closure_unix64;
765
766 #if defined(FFI_EXEC_STATIC_TRAMP)
767 if (ffi_tramp_is_present(closure))
768 {
769 /* Initialize the static trampoline's parameters. */
770 if (dest == ffi_closure_unix64_sse)
771 dest = ffi_closure_unix64_sse_alt;
772 else
773 dest = ffi_closure_unix64_alt;
774 ffi_tramp_set_parms (closure->ftramp, dest, closure);
775 goto out;
776 }
777 #endif
778
779 /* Initialize the dynamic trampoline. */
780 memcpy (tramp, trampoline, sizeof(trampoline));
781 *(UINT64 *)(tramp + sizeof (trampoline)) = (uintptr_t)dest;
782
783 out:
784 closure->cif = cif;
785 closure->fun = fun;
786 closure->user_data = user_data;
787
788 return FFI_OK;
789 }
790
791 int FFI_HIDDEN
792 ffi_closure_unix64_inner(ffi_cif *cif,
793 void (*fun)(ffi_cif*, void*, void**, void*),
794 void *user_data,
795 void *rvalue,
796 struct register_args *reg_args,
797 char *argp)
798 {
799 void **avalue;
800 ffi_type **arg_types;
801 long i, avn;
802 int gprcount, ssecount, ngpr, nsse;
803 int flags;
804
805 avn = cif->nargs;
806 flags = cif->flags;
807 avalue = alloca(avn * sizeof(void *));
808 gprcount = ssecount = 0;
809
810 if (flags & UNIX64_FLAG_RET_IN_MEM)
811 {
812 /* On return, %rax will contain the address that was passed
813 by the caller in %rdi. */
814 void *r = (void *)(uintptr_t)reg_args->gpr[gprcount++];
815 *(void **)rvalue = r;
816 rvalue = r;
817 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
818 }
819
820 arg_types = cif->arg_types;
821 for (i = 0; i < avn; ++i)
822 {
823 enum x86_64_reg_class classes[MAX_CLASSES];
824 size_t n;
825
826 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
827 if (n == 0
828 || gprcount + ngpr > MAX_GPR_REGS
829 || ssecount + nsse > MAX_SSE_REGS)
830 {
831 long align = arg_types[i]->alignment;
832
833 /* Stack arguments are *always* at least 8 byte aligned. */
834 if (align < 8)
835 align = 8;
836
837 /* Pass this argument in memory. */
838 argp = (void *) FFI_ALIGN (argp, align);
839 avalue[i] = argp;
840 argp += arg_types[i]->size;
841 }
842 /* If the argument is in a single register, or two consecutive
843 integer registers, then we can use that address directly. */
844 else if (n == 1
845 || (n == 2 && !(SSE_CLASS_P (classes[0])
846 || SSE_CLASS_P (classes[1]))))
847 {
848 /* The argument is in a single register. */
849 if (SSE_CLASS_P (classes[0]))
850 {
851 avalue[i] = ®_args->sse[ssecount];
852 ssecount += n;
853 }
854 else
855 {
856 avalue[i] = ®_args->gpr[gprcount];
857 gprcount += n;
858 }
859 }
860 /* Otherwise, allocate space to make them consecutive. */
861 else
862 {
863 char *a = alloca (16);
864 unsigned int j;
865
866 avalue[i] = a;
867 for (j = 0; j < n; j++, a += 8)
868 {
869 if (SSE_CLASS_P (classes[j]))
870 memcpy (a, ®_args->sse[ssecount++], 8);
871 else
872 memcpy (a, ®_args->gpr[gprcount++], 8);
873 }
874 }
875 }
876
877 /* Invoke the closure. */
878 fun (cif, rvalue, avalue, user_data);
879
880 /* Tell assembly how to perform return type promotions. */
881 return flags;
882 }
883
884 #ifdef FFI_GO_CLOSURES
885
886 extern void ffi_go_closure_unix64(void) FFI_HIDDEN;
887 extern void ffi_go_closure_unix64_sse(void) FFI_HIDDEN;
888
889 #ifndef __ILP32__
890 extern ffi_status
891 ffi_prep_go_closure_efi64(ffi_go_closure* closure, ffi_cif* cif,
892 void (*fun)(ffi_cif*, void*, void**, void*));
893 #endif
894
895 ffi_status
896 ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif,
897 void (*fun)(ffi_cif*, void*, void**, void*))
898 {
899 #ifndef __ILP32__
900 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
901 return ffi_prep_go_closure_efi64(closure, cif, fun);
902 #endif
903 if (cif->abi != FFI_UNIX64)
904 return FFI_BAD_ABI;
905
906 closure->tramp = (cif->flags & UNIX64_FLAG_XMM_ARGS
907 ? ffi_go_closure_unix64_sse
908 : ffi_go_closure_unix64);
909 closure->cif = cif;
910 closure->fun = fun;
911
912 return FFI_OK;
913 }
914
915 #endif /* FFI_GO_CLOSURES */
916
917 #if defined(FFI_EXEC_STATIC_TRAMP)
918 void *
919 ffi_tramp_arch (size_t *tramp_size, size_t *map_size)
920 {
921 extern void *trampoline_code_table;
922
923 *map_size = UNIX64_TRAMP_MAP_SIZE;
924 *tramp_size = UNIX64_TRAMP_SIZE;
925 return &trampoline_code_table;
926 }
927 #endif
928
929 #endif /* __x86_64__ */