1 /* -----------------------------------------------------------------------
2 ffi.c - Copyright (C) 2011 Anthony Green
3 Copyright (C) 2011 Kyle Moffett
4 Copyright (C) 2008 Red Hat, Inc
5 Copyright (C) 2007, 2008 Free Software Foundation, Inc
6 Copyright (c) 1998 Geoffrey Keating
8 PowerPC Foreign Function Interface
10 Permission is hereby granted, free of charge, to any person obtaining
11 a copy of this software and associated documentation files (the
12 ``Software''), to deal in the Software without restriction, including
13 without limitation the rights to use, copy, modify, merge, publish,
14 distribute, sublicense, and/or sell copies of the Software, and to
15 permit persons to whom the Software is furnished to do so, subject to
16 the following conditions:
18 The above copyright notice and this permission notice shall be included
19 in all copies or substantial portions of the Software.
21 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
23 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
24 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR
25 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
26 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
27 OTHER DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
31 #include <ffi_common.h>
37 extern void ffi_closure_SYSV (void);
38 extern void FFI_HIDDEN ffi_closure_LINUX64 (void);
41 /* The assembly depends on these exact flags. */
42 FLAG_RETURNS_SMST = 1 << (31-31), /* Used for FFI_SYSV small structs. */
43 FLAG_RETURNS_NOTHING = 1 << (31-30), /* These go in cr7 */
45 FLAG_RETURNS_FP = 1 << (31-29),
47 FLAG_RETURNS_64BITS = 1 << (31-28),
49 FLAG_RETURNS_128BITS = 1 << (31-27), /* cr6 */
51 FLAG_SYSV_SMST_R4 = 1 << (31-26), /* use r4 for FFI_SYSV 8 byte
53 FLAG_SYSV_SMST_R3 = 1 << (31-25), /* use r3 for FFI_SYSV 4 byte
56 FLAG_ARG_NEEDS_COPY = 1 << (31- 7),
58 FLAG_FP_ARGUMENTS = 1 << (31- 6), /* cr1.eq; specified by ABI */
60 FLAG_4_GPR_ARGUMENTS = 1 << (31- 5),
61 FLAG_RETVAL_REFERENCE = 1 << (31- 4)
64 /* About the SYSV ABI. */
65 #define ASM_NEEDS_REGISTERS 4
66 #define NUM_GPR_ARG_REGISTERS 8
68 # define NUM_FPR_ARG_REGISTERS 8
71 /* ffi_prep_args_SYSV is called by the assembly routine once stack space
72 has been allocated for the function's arguments.
74 The stack layout we want looks like this:
76 | Return address from ffi_call_SYSV 4bytes | higher addresses
77 |--------------------------------------------|
78 | Previous backchain pointer 4 | stack pointer here
79 |--------------------------------------------|<+ <<< on entry to
80 | Saved r28-r31 4*4 | | ffi_call_SYSV
81 |--------------------------------------------| |
82 | GPR registers r3-r10 8*4 | | ffi_call_SYSV
83 |--------------------------------------------| |
84 | FPR registers f1-f8 (optional) 8*8 | |
85 |--------------------------------------------| | stack |
86 | Space for copied structures | | grows |
87 |--------------------------------------------| | down V
88 | Parameters that didn't fit in registers | |
89 |--------------------------------------------| | lower addresses
90 | Space for callee's LR 4 | |
91 |--------------------------------------------| | stack pointer here
92 | Current backchain pointer 4 |-/ during
93 |--------------------------------------------| <<< ffi_call_SYSV
98 ffi_prep_args_SYSV (extended_cif *ecif, unsigned *const stack)
100 const unsigned bytes = ecif->cif->bytes;
101 const unsigned flags = ecif->cif->flags;
111 /* 'stacktop' points at the previous backchain pointer. */
114 /* 'gpr_base' points at the space for gpr3, and grows upwards as
115 we use GPR registers. */
120 /* 'fpr_base' points at the space for fpr1, and grows upwards as
121 we use FPR registers. */
126 /* 'copy_space' grows down as we put structures in it. It should
127 stay 16-byte aligned. */
130 /* 'next_arg' grows up as we put parameters in it. */
150 size_t struct_copy_size;
153 stacktop.c = (char *) stack + bytes;
154 gpr_base.u = stacktop.u - ASM_NEEDS_REGISTERS - NUM_GPR_ARG_REGISTERS;
157 fpr_base.d = gpr_base.d - NUM_FPR_ARG_REGISTERS;
159 copy_space.c = ((flags & FLAG_FP_ARGUMENTS) ? fpr_base.c : gpr_base.c);
161 copy_space.c = gpr_base.c;
163 next_arg.u = stack + 2;
165 /* Check that everything starts aligned properly. */
166 FFI_ASSERT (((unsigned long) (char *) stack & 0xF) == 0);
167 FFI_ASSERT (((unsigned long) copy_space.c & 0xF) == 0);
168 FFI_ASSERT (((unsigned long) stacktop.c & 0xF) == 0);
169 FFI_ASSERT ((bytes & 0xF) == 0);
170 FFI_ASSERT (copy_space.c >= next_arg.c);
172 /* Deal with return values that are actually pass-by-reference. */
173 if (flags & FLAG_RETVAL_REFERENCE)
175 *gpr_base.u++ = (unsigned long) (char *) ecif->rvalue;
179 /* Now for the arguments. */
180 p_argv.v = ecif->avalue;
181 for (ptr = ecif->cif->arg_types, i = ecif->cif->nargs;
183 i--, ptr++, p_argv.v++)
185 unsigned short typenum = (*ptr)->type;
187 /* We may need to handle some values depending on ABI */
188 if (ecif->cif->abi == FFI_LINUX_SOFT_FLOAT) {
189 if (typenum == FFI_TYPE_FLOAT)
190 typenum = FFI_TYPE_UINT32;
191 if (typenum == FFI_TYPE_DOUBLE)
192 typenum = FFI_TYPE_UINT64;
193 if (typenum == FFI_TYPE_LONGDOUBLE)
194 typenum = FFI_TYPE_UINT128;
195 } else if (ecif->cif->abi != FFI_LINUX) {
196 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
197 if (typenum == FFI_TYPE_LONGDOUBLE)
198 typenum = FFI_TYPE_STRUCT;
202 /* Now test the translated value */
206 /* With FFI_LINUX_SOFT_FLOAT floats are handled like UINT32. */
207 double_tmp = **p_argv.f;
208 if (fparg_count >= NUM_FPR_ARG_REGISTERS)
210 *next_arg.f = (float) double_tmp;
215 *fpr_base.d++ = double_tmp;
217 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
220 case FFI_TYPE_DOUBLE:
221 /* With FFI_LINUX_SOFT_FLOAT doubles are handled like UINT64. */
222 double_tmp = **p_argv.d;
224 if (fparg_count >= NUM_FPR_ARG_REGISTERS)
226 if (intarg_count >= NUM_GPR_ARG_REGISTERS
227 && intarg_count % 2 != 0)
232 *next_arg.d = double_tmp;
236 *fpr_base.d++ = double_tmp;
238 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
241 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
242 case FFI_TYPE_LONGDOUBLE:
243 double_tmp = (*p_argv.d)[0];
245 if (fparg_count >= NUM_FPR_ARG_REGISTERS - 1)
247 if (intarg_count >= NUM_GPR_ARG_REGISTERS
248 && intarg_count % 2 != 0)
253 *next_arg.d = double_tmp;
255 double_tmp = (*p_argv.d)[1];
256 *next_arg.d = double_tmp;
261 *fpr_base.d++ = double_tmp;
262 double_tmp = (*p_argv.d)[1];
263 *fpr_base.d++ = double_tmp;
267 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
270 #endif /* have FPRs */
273 * The soft float ABI for long doubles works like this, a long double
274 * is passed in four consecutive GPRs if available. A maximum of 2
275 * long doubles can be passed in gprs. If we do not have 4 GPRs
276 * left, the long double is passed on the stack, 4-byte aligned.
278 case FFI_TYPE_UINT128: {
279 unsigned int int_tmp = (*p_argv.ui)[0];
281 if (intarg_count >= NUM_GPR_ARG_REGISTERS - 3) {
282 if (intarg_count < NUM_GPR_ARG_REGISTERS)
283 intarg_count += NUM_GPR_ARG_REGISTERS - intarg_count;
284 *(next_arg.u++) = int_tmp;
285 for (ii = 1; ii < 4; ii++) {
286 int_tmp = (*p_argv.ui)[ii];
287 *(next_arg.u++) = int_tmp;
290 *(gpr_base.u++) = int_tmp;
291 for (ii = 1; ii < 4; ii++) {
292 int_tmp = (*p_argv.ui)[ii];
293 *(gpr_base.u++) = int_tmp;
300 case FFI_TYPE_UINT64:
301 case FFI_TYPE_SINT64:
302 if (intarg_count == NUM_GPR_ARG_REGISTERS-1)
304 if (intarg_count >= NUM_GPR_ARG_REGISTERS)
306 if (intarg_count % 2 != 0)
311 *next_arg.ll = **p_argv.ll;
316 /* whoops: abi states only certain register pairs
317 * can be used for passing long long int
318 * specifically (r3,r4), (r5,r6), (r7,r8),
319 * (r9,r10) and if next arg is long long but
320 * not correct starting register of pair then skip
321 * until the proper starting register
323 if (intarg_count % 2 != 0)
328 *gpr_base.ll++ = **p_argv.ll;
333 case FFI_TYPE_STRUCT:
334 struct_copy_size = ((*ptr)->size + 15) & ~0xF;
335 copy_space.c -= struct_copy_size;
336 memcpy (copy_space.c, *p_argv.c, (*ptr)->size);
338 gprvalue = (unsigned long) copy_space.c;
340 FFI_ASSERT (copy_space.c > next_arg.c);
341 FFI_ASSERT (flags & FLAG_ARG_NEEDS_COPY);
345 gprvalue = **p_argv.uc;
348 gprvalue = **p_argv.sc;
350 case FFI_TYPE_UINT16:
351 gprvalue = **p_argv.us;
353 case FFI_TYPE_SINT16:
354 gprvalue = **p_argv.ss;
358 case FFI_TYPE_UINT32:
359 case FFI_TYPE_SINT32:
360 case FFI_TYPE_POINTER:
362 gprvalue = **p_argv.ui;
365 if (intarg_count >= NUM_GPR_ARG_REGISTERS)
366 *next_arg.u++ = gprvalue;
368 *gpr_base.u++ = gprvalue;
374 /* Check that we didn't overrun the stack... */
375 FFI_ASSERT (copy_space.c >= next_arg.c);
376 FFI_ASSERT (gpr_base.u <= stacktop.u - ASM_NEEDS_REGISTERS);
377 /* The assert below is testing that the number of integer arguments agrees
378 with the number found in ffi_prep_cif_machdep(). However, intarg_count
379 is incremented whenever we place an FP arg on the stack, so account for
380 that before our assert test. */
382 if (fparg_count > NUM_FPR_ARG_REGISTERS)
383 intarg_count -= fparg_count - NUM_FPR_ARG_REGISTERS;
384 FFI_ASSERT (fpr_base.u
385 <= stacktop.u - ASM_NEEDS_REGISTERS - NUM_GPR_ARG_REGISTERS);
387 FFI_ASSERT (flags & FLAG_4_GPR_ARGUMENTS || intarg_count <= 4);
390 /* About the LINUX64 ABI. */
392 NUM_GPR_ARG_REGISTERS64 = 8,
393 NUM_FPR_ARG_REGISTERS64 = 13
395 enum { ASM_NEEDS_REGISTERS64 = 4 };
397 /* ffi_prep_args64 is called by the assembly routine once stack space
398 has been allocated for the function's arguments.
400 The stack layout we want looks like this:
402 | Ret addr from ffi_call_LINUX64 8bytes | higher addresses
403 |--------------------------------------------|
404 | CR save area 8bytes |
405 |--------------------------------------------|
406 | Previous backchain pointer 8 | stack pointer here
407 |--------------------------------------------|<+ <<< on entry to
408 | Saved r28-r31 4*8 | | ffi_call_LINUX64
409 |--------------------------------------------| |
410 | GPR registers r3-r10 8*8 | |
411 |--------------------------------------------| |
412 | FPR registers f1-f13 (optional) 13*8 | |
413 |--------------------------------------------| |
414 | Parameter save area | |
415 |--------------------------------------------| |
416 | TOC save area 8 | |
417 |--------------------------------------------| | stack |
418 | Linker doubleword 8 | | grows |
419 |--------------------------------------------| | down V
420 | Compiler doubleword 8 | |
421 |--------------------------------------------| | lower addresses
422 | Space for callee's LR 8 | |
423 |--------------------------------------------| |
425 |--------------------------------------------| | stack pointer here
426 | Current backchain pointer 8 |-/ during
427 |--------------------------------------------| <<< ffi_call_LINUX64
432 ffi_prep_args64 (extended_cif *ecif, unsigned long *const stack)
434 const unsigned long bytes = ecif->cif->bytes;
435 const unsigned long flags = ecif->cif->flags;
444 /* 'stacktop' points at the previous backchain pointer. */
447 /* 'next_arg' points at the space for gpr3, and grows upwards as
448 we use GPR registers, then continues at rest. */
454 /* 'fpr_base' points at the space for fpr3, and grows upwards as
455 we use FPR registers. */
457 unsigned int fparg_count;
459 unsigned int i, words, nargs, nfixedargs;
475 unsigned long gprvalue;
477 stacktop.c = (char *) stack + bytes;
478 gpr_base.ul = stacktop.ul - ASM_NEEDS_REGISTERS64 - NUM_GPR_ARG_REGISTERS64;
479 gpr_end.ul = gpr_base.ul + NUM_GPR_ARG_REGISTERS64;
480 rest.ul = stack + 6 + NUM_GPR_ARG_REGISTERS64;
481 fpr_base.d = gpr_base.d - NUM_FPR_ARG_REGISTERS64;
483 next_arg.ul = gpr_base.ul;
485 /* Check that everything starts aligned properly. */
486 FFI_ASSERT (((unsigned long) (char *) stack & 0xF) == 0);
487 FFI_ASSERT (((unsigned long) stacktop.c & 0xF) == 0);
488 FFI_ASSERT ((bytes & 0xF) == 0);
490 /* Deal with return values that are actually pass-by-reference. */
491 if (flags & FLAG_RETVAL_REFERENCE)
492 *next_arg.ul++ = (unsigned long) (char *) ecif->rvalue;
494 /* Now for the arguments. */
495 p_argv.v = ecif->avalue;
496 nargs = ecif->cif->nargs;
497 nfixedargs = ecif->cif->nfixedargs;
498 for (ptr = ecif->cif->arg_types, i = 0;
500 i++, ptr++, p_argv.v++)
502 switch ((*ptr)->type)
505 double_tmp = **p_argv.f;
506 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
507 *fpr_base.d++ = double_tmp;
509 *next_arg.f = (float) double_tmp;
510 if (++next_arg.ul == gpr_end.ul)
511 next_arg.ul = rest.ul;
513 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
516 case FFI_TYPE_DOUBLE:
517 double_tmp = **p_argv.d;
518 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
519 *fpr_base.d++ = double_tmp;
521 *next_arg.d = double_tmp;
522 if (++next_arg.ul == gpr_end.ul)
523 next_arg.ul = rest.ul;
525 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
528 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
529 case FFI_TYPE_LONGDOUBLE:
530 double_tmp = (*p_argv.d)[0];
531 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
532 *fpr_base.d++ = double_tmp;
534 *next_arg.d = double_tmp;
535 if (++next_arg.ul == gpr_end.ul)
536 next_arg.ul = rest.ul;
538 double_tmp = (*p_argv.d)[1];
539 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
540 *fpr_base.d++ = double_tmp;
542 *next_arg.d = double_tmp;
543 if (++next_arg.ul == gpr_end.ul)
544 next_arg.ul = rest.ul;
546 FFI_ASSERT (__LDBL_MANT_DIG__ == 106);
547 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
551 case FFI_TYPE_STRUCT:
552 words = ((*ptr)->size + 7) / 8;
553 if (next_arg.ul >= gpr_base.ul && next_arg.ul + words > gpr_end.ul)
555 size_t first = gpr_end.c - next_arg.c;
556 memcpy (next_arg.c, *p_argv.c, first);
557 memcpy (rest.c, *p_argv.c + first, (*ptr)->size - first);
558 next_arg.c = rest.c + words * 8 - first;
562 char *where = next_arg.c;
564 #ifndef __LITTLE_ENDIAN__
565 /* Structures with size less than eight bytes are passed
567 if ((*ptr)->size < 8)
568 where += 8 - (*ptr)->size;
570 memcpy (where, *p_argv.c, (*ptr)->size);
571 next_arg.ul += words;
572 if (next_arg.ul == gpr_end.ul)
573 next_arg.ul = rest.ul;
578 gprvalue = **p_argv.uc;
581 gprvalue = **p_argv.sc;
583 case FFI_TYPE_UINT16:
584 gprvalue = **p_argv.us;
586 case FFI_TYPE_SINT16:
587 gprvalue = **p_argv.ss;
589 case FFI_TYPE_UINT32:
590 gprvalue = **p_argv.ui;
593 case FFI_TYPE_SINT32:
594 gprvalue = **p_argv.si;
597 case FFI_TYPE_UINT64:
598 case FFI_TYPE_SINT64:
599 case FFI_TYPE_POINTER:
600 gprvalue = **p_argv.ul;
602 *next_arg.ul++ = gprvalue;
603 if (next_arg.ul == gpr_end.ul)
604 next_arg.ul = rest.ul;
609 FFI_ASSERT (flags & FLAG_4_GPR_ARGUMENTS
610 || (next_arg.ul >= gpr_base.ul
611 && next_arg.ul <= gpr_base.ul + 4));
616 /* Perform machine dependent cif processing */
618 ffi_prep_cif_machdep_core (ffi_cif *cif)
620 /* All this is for the SYSV and LINUX64 ABI. */
623 unsigned i, fparg_count = 0, intarg_count = 0;
624 unsigned flags = cif->flags;
625 unsigned struct_copy_size = 0;
626 unsigned type = cif->rtype->type;
627 unsigned size = cif->rtype->size;
629 if (cif->abi != FFI_LINUX64)
631 /* All the machine-independent calculation of cif->bytes will be wrong.
632 Redo the calculation for SYSV. */
634 /* Space for the frame pointer, callee's LR, and the asm's temp regs. */
635 bytes = (2 + ASM_NEEDS_REGISTERS) * sizeof (int);
637 /* Space for the GPR registers. */
638 bytes += NUM_GPR_ARG_REGISTERS * sizeof (int);
644 /* Space for backchain, CR, LR, cc/ld doubleword, TOC and the asm's temp
646 bytes = (6 + ASM_NEEDS_REGISTERS64) * sizeof (long);
648 /* Space for the mandatory parm save area and general registers. */
649 bytes += 2 * NUM_GPR_ARG_REGISTERS64 * sizeof (long);
652 /* Return value handling. The rules for SYSV are as follows:
653 - 32-bit (or less) integer values are returned in gpr3;
654 - Structures of size <= 4 bytes also returned in gpr3;
655 - 64-bit integer values and structures between 5 and 8 bytes are returned
657 - Single/double FP values are returned in fpr1;
658 - Larger structures are allocated space and a pointer is passed as
660 - long doubles (if not equivalent to double) are returned in
661 fpr1,fpr2 for Linux and as for large structs for SysV.
663 - integer values in gpr3;
664 - Structures/Unions by reference;
665 - Single/double FP values in fpr1, long double in fpr1,fpr2.
666 - soft-float float/doubles are treated as UINT32/UINT64 respectivley.
667 - soft-float long doubles are returned in gpr3-gpr6. */
668 /* First translate for softfloat/nonlinux */
669 if (cif->abi == FFI_LINUX_SOFT_FLOAT)
671 if (type == FFI_TYPE_FLOAT)
672 type = FFI_TYPE_UINT32;
673 if (type == FFI_TYPE_DOUBLE)
674 type = FFI_TYPE_UINT64;
675 if (type == FFI_TYPE_LONGDOUBLE)
676 type = FFI_TYPE_UINT128;
678 else if (cif->abi != FFI_LINUX
679 && cif->abi != FFI_LINUX64)
681 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
682 if (type == FFI_TYPE_LONGDOUBLE)
683 type = FFI_TYPE_STRUCT;
690 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
691 case FFI_TYPE_LONGDOUBLE:
692 flags |= FLAG_RETURNS_128BITS;
695 case FFI_TYPE_DOUBLE:
696 flags |= FLAG_RETURNS_64BITS;
699 flags |= FLAG_RETURNS_FP;
703 case FFI_TYPE_UINT128:
704 flags |= FLAG_RETURNS_128BITS;
706 case FFI_TYPE_UINT64:
707 case FFI_TYPE_SINT64:
708 flags |= FLAG_RETURNS_64BITS;
711 case FFI_TYPE_STRUCT:
712 if (cif->abi == FFI_SYSV)
714 /* The final SYSV ABI says that structures smaller or equal 8 bytes
715 are returned in r3/r4. The FFI_GCC_SYSV ABI instead returns them
718 /* Treat structs with size <= 8 bytes. */
721 flags |= FLAG_RETURNS_SMST;
722 /* These structs are returned in r3. We pack the type and the
723 precalculated shift value (needed in the sysv.S) into flags.
724 The same applies for the structs returned in r3/r4. */
727 flags |= FLAG_SYSV_SMST_R3;
728 flags |= 8 * (4 - size) << 8;
731 /* These structs are returned in r3 and r4. See above. */
734 flags |= FLAG_SYSV_SMST_R3 | FLAG_SYSV_SMST_R4;
735 flags |= 8 * (8 - size) << 8;
742 flags |= FLAG_RETVAL_REFERENCE;
745 flags |= FLAG_RETURNS_NOTHING;
749 /* Returns 32-bit integer, or similar. Nothing to do here. */
753 if (cif->abi != FFI_LINUX64)
754 /* The first NUM_GPR_ARG_REGISTERS words of integer arguments, and the
755 first NUM_FPR_ARG_REGISTERS fp arguments, go in registers; the rest
756 goes on the stack. Structures and long doubles (if not equivalent
757 to double) are passed as a pointer to a copy of the structure.
758 Stuff on the stack needs to keep proper alignment. */
759 for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++)
761 unsigned short typenum = (*ptr)->type;
763 /* We may need to handle some values depending on ABI */
764 if (cif->abi == FFI_LINUX_SOFT_FLOAT) {
765 if (typenum == FFI_TYPE_FLOAT)
766 typenum = FFI_TYPE_UINT32;
767 if (typenum == FFI_TYPE_DOUBLE)
768 typenum = FFI_TYPE_UINT64;
769 if (typenum == FFI_TYPE_LONGDOUBLE)
770 typenum = FFI_TYPE_UINT128;
771 } else if (cif->abi != FFI_LINUX && cif->abi != FFI_LINUX64) {
772 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
773 if (typenum == FFI_TYPE_LONGDOUBLE)
774 typenum = FFI_TYPE_STRUCT;
782 /* floating singles are not 8-aligned on stack */
785 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
786 case FFI_TYPE_LONGDOUBLE:
790 case FFI_TYPE_DOUBLE:
792 /* If this FP arg is going on the stack, it must be
794 if (fparg_count > NUM_FPR_ARG_REGISTERS
795 && intarg_count >= NUM_GPR_ARG_REGISTERS
796 && intarg_count % 2 != 0)
800 case FFI_TYPE_UINT128:
802 * A long double in FFI_LINUX_SOFT_FLOAT can use only a set
803 * of four consecutive gprs. If we do not have enough, we
804 * have to adjust the intarg_count value.
806 if (intarg_count >= NUM_GPR_ARG_REGISTERS - 3
807 && intarg_count < NUM_GPR_ARG_REGISTERS)
808 intarg_count = NUM_GPR_ARG_REGISTERS;
812 case FFI_TYPE_UINT64:
813 case FFI_TYPE_SINT64:
814 /* 'long long' arguments are passed as two words, but
815 either both words must fit in registers or both go
816 on the stack. If they go on the stack, they must
819 Also, only certain register pairs can be used for
820 passing long long int -- specifically (r3,r4), (r5,r6),
823 if (intarg_count == NUM_GPR_ARG_REGISTERS-1
824 || intarg_count % 2 != 0)
829 case FFI_TYPE_STRUCT:
830 /* We must allocate space for a copy of these to enforce
831 pass-by-value. Pad the space up to a multiple of 16
832 bytes (the maximum alignment required for anything under
834 struct_copy_size += ((*ptr)->size + 15) & ~0xF;
835 /* Fall through (allocate space for the pointer). */
837 case FFI_TYPE_POINTER:
839 case FFI_TYPE_UINT32:
840 case FFI_TYPE_SINT32:
841 case FFI_TYPE_UINT16:
842 case FFI_TYPE_SINT16:
845 /* Everything else is passed as a 4-byte word in a GPR, either
846 the object itself or a pointer to it. */
854 for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++)
856 switch ((*ptr)->type)
858 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
859 case FFI_TYPE_LONGDOUBLE:
865 case FFI_TYPE_DOUBLE:
870 case FFI_TYPE_STRUCT:
871 intarg_count += ((*ptr)->size + 7) / 8;
874 case FFI_TYPE_POINTER:
875 case FFI_TYPE_UINT64:
876 case FFI_TYPE_SINT64:
878 case FFI_TYPE_UINT32:
879 case FFI_TYPE_SINT32:
880 case FFI_TYPE_UINT16:
881 case FFI_TYPE_SINT16:
884 /* Everything else is passed as a 8-byte word in a GPR, either
885 the object itself or a pointer to it. */
894 if (fparg_count != 0)
895 flags |= FLAG_FP_ARGUMENTS;
897 if (intarg_count > 4)
898 flags |= FLAG_4_GPR_ARGUMENTS;
899 if (struct_copy_size != 0)
900 flags |= FLAG_ARG_NEEDS_COPY;
902 if (cif->abi != FFI_LINUX64)
905 /* Space for the FPR registers, if needed. */
906 if (fparg_count != 0)
907 bytes += NUM_FPR_ARG_REGISTERS * sizeof (double);
911 if (intarg_count > NUM_GPR_ARG_REGISTERS)
912 bytes += (intarg_count - NUM_GPR_ARG_REGISTERS) * sizeof (int);
914 if (fparg_count > NUM_FPR_ARG_REGISTERS)
915 bytes += (fparg_count - NUM_FPR_ARG_REGISTERS) * sizeof (double);
921 /* Space for the FPR registers, if needed. */
922 if (fparg_count != 0)
923 bytes += NUM_FPR_ARG_REGISTERS64 * sizeof (double);
927 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
928 bytes += (intarg_count - NUM_GPR_ARG_REGISTERS64) * sizeof (long);
931 /* The stack space allocated needs to be a multiple of 16 bytes. */
932 bytes = (bytes + 15) & ~0xF;
934 /* Add in the space for the copied structures. */
935 bytes += struct_copy_size;
944 ffi_prep_cif_machdep (ffi_cif *cif)
946 cif->nfixedargs = cif->nargs;
947 return ffi_prep_cif_machdep_core (cif);
951 ffi_prep_cif_machdep_var (ffi_cif *cif,
952 unsigned int nfixedargs,
953 unsigned int ntotalargs MAYBE_UNUSED)
955 cif->nfixedargs = nfixedargs;
956 return ffi_prep_cif_machdep_core (cif);
959 extern void ffi_call_SYSV(extended_cif *, unsigned, unsigned, unsigned *,
961 extern void FFI_HIDDEN ffi_call_LINUX64(extended_cif *, unsigned long,
962 unsigned long, unsigned long *,
966 ffi_call(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
969 * The final SYSV ABI says that structures smaller or equal 8 bytes
970 * are returned in r3/r4. The FFI_GCC_SYSV ABI instead returns them
973 * Just to keep things simple for the assembly code, we will always
974 * bounce-buffer struct return values less than or equal to 8 bytes.
975 * This allows the ASM to handle SYSV small structures by directly
976 * writing r3 and r4 to memory without worrying about struct size.
978 unsigned int smst_buffer[2];
980 unsigned int rsize = 0;
983 ecif.avalue = avalue;
985 /* Ensure that we have a valid struct return value */
986 ecif.rvalue = rvalue;
987 if (cif->rtype->type == FFI_TYPE_STRUCT) {
988 rsize = cif->rtype->size;
990 ecif.rvalue = smst_buffer;
992 ecif.rvalue = alloca(rsize);
1003 case FFI_LINUX_SOFT_FLOAT:
1004 ffi_call_SYSV (&ecif, -cif->bytes, cif->flags, ecif.rvalue, fn);
1008 ffi_call_LINUX64 (&ecif, -(long) cif->bytes, cif->flags, ecif.rvalue, fn);
1016 /* Check for a bounce-buffered return value */
1017 if (rvalue && ecif.rvalue == smst_buffer)
1018 memcpy(rvalue, smst_buffer, rsize);
1023 #define MIN_CACHE_LINE_SIZE 8
1026 flush_icache (char *wraddr, char *xaddr, int size)
1029 for (i = 0; i < size; i += MIN_CACHE_LINE_SIZE)
1030 __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;"
1031 : : "r" (xaddr + i), "r" (wraddr + i) : "memory");
1032 __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;" "sync;" "isync;"
1033 : : "r"(xaddr + size - 1), "r"(wraddr + size - 1)
1039 ffi_prep_closure_loc (ffi_closure *closure,
1041 void (*fun) (ffi_cif *, void *, void **, void *),
1046 void **tramp = (void **) &closure->tramp[0];
1048 if (cif->abi != FFI_LINUX64)
1050 /* Copy function address and TOC from ffi_closure_LINUX64. */
1051 memcpy (tramp, (char *) ffi_closure_LINUX64, 16);
1054 unsigned int *tramp;
1056 if (! (cif->abi == FFI_GCC_SYSV
1057 || cif->abi == FFI_SYSV
1058 || cif->abi == FFI_LINUX
1059 || cif->abi == FFI_LINUX_SOFT_FLOAT))
1062 tramp = (unsigned int *) &closure->tramp[0];
1063 tramp[0] = 0x7c0802a6; /* mflr r0 */
1064 tramp[1] = 0x4800000d; /* bl 10 <trampoline_initial+0x10> */
1065 tramp[4] = 0x7d6802a6; /* mflr r11 */
1066 tramp[5] = 0x7c0803a6; /* mtlr r0 */
1067 tramp[6] = 0x800b0000; /* lwz r0,0(r11) */
1068 tramp[7] = 0x816b0004; /* lwz r11,4(r11) */
1069 tramp[8] = 0x7c0903a6; /* mtctr r0 */
1070 tramp[9] = 0x4e800420; /* bctr */
1071 *(void **) &tramp[2] = (void *) ffi_closure_SYSV; /* function */
1072 *(void **) &tramp[3] = codeloc; /* context */
1074 /* Flush the icache. */
1075 flush_icache ((char *)tramp, (char *)codeloc, FFI_TRAMPOLINE_SIZE);
1080 closure->user_data = user_data;
1091 int ffi_closure_helper_SYSV (ffi_closure *, void *, unsigned long *,
1092 ffi_dblfl *, unsigned long *);
1094 /* Basically the trampoline invokes ffi_closure_SYSV, and on
1095 * entry, r11 holds the address of the closure.
1096 * After storing the registers that could possibly contain
1097 * parameters to be passed into the stack frame and setting
1098 * up space for a return value, ffi_closure_SYSV invokes the
1099 * following helper function to do most of the work
1103 ffi_closure_helper_SYSV (ffi_closure *closure, void *rvalue,
1104 unsigned long *pgr, ffi_dblfl *pfr,
1107 /* rvalue is the pointer to space for return value in closure assembly */
1108 /* pgr is the pointer to where r3-r10 are stored in ffi_closure_SYSV */
1109 /* pfr is the pointer to where f1-f8 are stored in ffi_closure_SYSV */
1110 /* pst is the pointer to outgoing parameter stack in original caller */
1113 ffi_type ** arg_types;
1116 long nf = 0; /* number of floating registers already used */
1118 long ng = 0; /* number of general registers already used */
1120 ffi_cif *cif = closure->cif;
1121 unsigned size = cif->rtype->size;
1122 unsigned short rtypenum = cif->rtype->type;
1124 avalue = alloca (cif->nargs * sizeof (void *));
1126 /* First translate for softfloat/nonlinux */
1127 if (cif->abi == FFI_LINUX_SOFT_FLOAT) {
1128 if (rtypenum == FFI_TYPE_FLOAT)
1129 rtypenum = FFI_TYPE_UINT32;
1130 if (rtypenum == FFI_TYPE_DOUBLE)
1131 rtypenum = FFI_TYPE_UINT64;
1132 if (rtypenum == FFI_TYPE_LONGDOUBLE)
1133 rtypenum = FFI_TYPE_UINT128;
1134 } else if (cif->abi != FFI_LINUX && cif->abi != FFI_LINUX64) {
1135 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
1136 if (rtypenum == FFI_TYPE_LONGDOUBLE)
1137 rtypenum = FFI_TYPE_STRUCT;
1142 /* Copy the caller's structure return value address so that the closure
1143 returns the data directly to the caller.
1144 For FFI_SYSV the result is passed in r3/r4 if the struct size is less
1145 or equal 8 bytes. */
1146 if (rtypenum == FFI_TYPE_STRUCT && ((cif->abi != FFI_SYSV) || (size > 8))) {
1147 rvalue = (void *) *pgr;
1154 arg_types = cif->arg_types;
1156 /* Grab the addresses of the arguments from the stack frame. */
1158 unsigned short typenum = arg_types[i]->type;
1160 /* We may need to handle some values depending on ABI */
1161 if (cif->abi == FFI_LINUX_SOFT_FLOAT) {
1162 if (typenum == FFI_TYPE_FLOAT)
1163 typenum = FFI_TYPE_UINT32;
1164 if (typenum == FFI_TYPE_DOUBLE)
1165 typenum = FFI_TYPE_UINT64;
1166 if (typenum == FFI_TYPE_LONGDOUBLE)
1167 typenum = FFI_TYPE_UINT128;
1168 } else if (cif->abi != FFI_LINUX && cif->abi != FFI_LINUX64) {
1169 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
1170 if (typenum == FFI_TYPE_LONGDOUBLE)
1171 typenum = FFI_TYPE_STRUCT;
1177 case FFI_TYPE_FLOAT:
1178 /* unfortunately float values are stored as doubles
1179 * in the ffi_closure_SYSV code (since we don't check
1180 * the type in that routine).
1183 /* there are 8 64bit floating point registers */
1187 double temp = pfr->d;
1188 pfr->f = (float) temp;
1195 /* FIXME? here we are really changing the values
1196 * stored in the original calling routines outgoing
1197 * parameter stack. This is probably a really
1198 * naughty thing to do but...
1205 case FFI_TYPE_DOUBLE:
1206 /* On the outgoing stack all values are aligned to 8 */
1207 /* there are 8 64bit floating point registers */
1217 if (((long) pst) & 4)
1224 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
1225 case FFI_TYPE_LONGDOUBLE:
1234 if (((long) pst) & 4)
1242 #endif /* have FPRS */
1244 case FFI_TYPE_UINT128:
1246 * Test if for the whole long double, 4 gprs are available.
1247 * otherwise the stuff ends up on the stack.
1260 case FFI_TYPE_SINT8:
1261 case FFI_TYPE_UINT8:
1262 #ifndef __LITTLE_ENDIAN__
1263 /* there are 8 gpr registers used to pass values */
1266 avalue[i] = (char *) pgr + 3;
1272 avalue[i] = (char *) pst + 3;
1278 case FFI_TYPE_SINT16:
1279 case FFI_TYPE_UINT16:
1280 #ifndef __LITTLE_ENDIAN__
1281 /* there are 8 gpr registers used to pass values */
1284 avalue[i] = (char *) pgr + 2;
1290 avalue[i] = (char *) pst + 2;
1296 case FFI_TYPE_SINT32:
1297 case FFI_TYPE_UINT32:
1298 case FFI_TYPE_POINTER:
1299 /* there are 8 gpr registers used to pass values */
1313 case FFI_TYPE_STRUCT:
1314 /* Structs are passed by reference. The address will appear in a
1315 gpr if it is one of the first 8 arguments. */
1318 avalue[i] = (void *) *pgr;
1324 avalue[i] = (void *) *pst;
1329 case FFI_TYPE_SINT64:
1330 case FFI_TYPE_UINT64:
1331 /* passing long long ints are complex, they must
1332 * be passed in suitable register pairs such as
1333 * (r3,r4) or (r5,r6) or (r6,r7), or (r7,r8) or (r9,r10)
1334 * and if the entire pair aren't available then the outgoing
1335 * parameter stack is used for both but an alignment of 8
1336 * must will be kept. So we must either look in pgr
1337 * or pst to find the correct address for this type
1344 /* skip r4, r6, r8 as starting points */
1354 if (((long) pst) & 4)
1370 (closure->fun) (cif, rvalue, avalue, closure->user_data);
1372 /* Tell ffi_closure_SYSV how to perform return type promotions.
1373 Because the FFI_SYSV ABI returns the structures <= 8 bytes in r3/r4
1374 we have to tell ffi_closure_SYSV how to treat them. We combine the base
1375 type FFI_SYSV_TYPE_SMALL_STRUCT - 1 with the size of the struct.
1376 So a one byte struct gets the return type 16. Return type 1 to 15 are
1377 already used and we never have a struct with size zero. That is the reason
1378 for the subtraction of 1. See the comment in ffitarget.h about ordering.
1380 if (cif->abi == FFI_SYSV && rtypenum == FFI_TYPE_STRUCT && size <= 8)
1381 return (FFI_SYSV_TYPE_SMALL_STRUCT - 1) + size;
1385 int FFI_HIDDEN ffi_closure_helper_LINUX64 (ffi_closure *, void *,
1386 unsigned long *, ffi_dblfl *);
1389 ffi_closure_helper_LINUX64 (ffi_closure *closure, void *rvalue,
1390 unsigned long *pst, ffi_dblfl *pfr)
1392 /* rvalue is the pointer to space for return value in closure assembly */
1393 /* pst is the pointer to parameter save area
1394 (r3-r10 are stored into its first 8 slots by ffi_closure_LINUX64) */
1395 /* pfr is the pointer to where f1-f13 are stored in ffi_closure_LINUX64 */
1398 ffi_type **arg_types;
1399 unsigned long i, avn, nfixedargs;
1401 ffi_dblfl *end_pfr = pfr + NUM_FPR_ARG_REGISTERS64;
1404 avalue = alloca (cif->nargs * sizeof (void *));
1406 /* Copy the caller's structure return value address so that the closure
1407 returns the data directly to the caller. */
1408 if (cif->rtype->type == FFI_TYPE_STRUCT)
1410 rvalue = (void *) *pst;
1416 nfixedargs = cif->nfixedargs;
1417 arg_types = cif->arg_types;
1419 /* Grab the addresses of the arguments from the stack frame. */
1422 switch (arg_types[i]->type)
1424 case FFI_TYPE_SINT8:
1425 case FFI_TYPE_UINT8:
1426 #ifndef __LITTLE_ENDIAN__
1427 avalue[i] = (char *) pst + 7;
1432 case FFI_TYPE_SINT16:
1433 case FFI_TYPE_UINT16:
1434 #ifndef __LITTLE_ENDIAN__
1435 avalue[i] = (char *) pst + 6;
1440 case FFI_TYPE_SINT32:
1441 case FFI_TYPE_UINT32:
1442 #ifndef __LITTLE_ENDIAN__
1443 avalue[i] = (char *) pst + 4;
1448 case FFI_TYPE_SINT64:
1449 case FFI_TYPE_UINT64:
1450 case FFI_TYPE_POINTER:
1455 case FFI_TYPE_STRUCT:
1456 #ifndef __LITTLE_ENDIAN__
1457 /* Structures with size less than eight bytes are passed
1459 if (arg_types[i]->size < 8)
1460 avalue[i] = (char *) pst + 8 - arg_types[i]->size;
1464 pst += (arg_types[i]->size + 7) / 8;
1467 case FFI_TYPE_FLOAT:
1468 /* unfortunately float values are stored as doubles
1469 * in the ffi_closure_LINUX64 code (since we don't check
1470 * the type in that routine).
1473 /* there are 13 64bit floating point registers */
1475 if (pfr < end_pfr && i < nfixedargs)
1477 double temp = pfr->d;
1478 pfr->f = (float) temp;
1487 case FFI_TYPE_DOUBLE:
1488 /* On the outgoing stack all values are aligned to 8 */
1489 /* there are 13 64bit floating point registers */
1491 if (pfr < end_pfr && i < nfixedargs)
1501 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
1502 case FFI_TYPE_LONGDOUBLE:
1503 if (pfr + 1 < end_pfr && i + 1 < nfixedargs)
1510 if (pfr < end_pfr && i < nfixedargs)
1512 /* Passed partly in f13 and partly on the stack.
1513 Move it all to the stack. */
1514 *pst = *(unsigned long *) pfr;
1531 (closure->fun) (cif, rvalue, avalue, closure->user_data);
1533 /* Tell ffi_closure_LINUX64 how to perform return type promotions. */
1534 return cif->rtype->type;