1 /* -----------------------------------------------------------------------
2 ffi.c - Copyright (c) 2015 Michael Knyszek <mknyszek@berkeley.edu>
3 2015 Andrew Waterman <waterman@cs.berkeley.edu>
4 2018 Stef O'Rear <sorear2@gmail.com>
5 Based on MIPS N32/64 port
6
7 RISC-V Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdint.h>
35
36 #if __riscv_float_abi_double
37 #define ABI_FLEN 64
38 #define ABI_FLOAT double
39 #elif __riscv_float_abi_single
40 #define ABI_FLEN 32
41 #define ABI_FLOAT float
42 #endif
43
44 #define NARGREG 8
45 #define STKALIGN 16
46 #define MAXCOPYARG (2 * sizeof(double))
47
48 typedef struct call_context
49 {
50 #if ABI_FLEN
51 ABI_FLOAT fa[8];
52 #endif
53 size_t a[8];
54 /* used by the assembly code to in-place construct its own stack frame */
55 char frame[16];
56 } call_context;
57
58 typedef struct call_builder
59 {
60 call_context *aregs;
61 int used_integer;
62 int used_float;
63 size_t *used_stack;
64 } call_builder;
65
66 /* integer (not pointer) less than ABI XLEN */
67 /* FFI_TYPE_INT does not appear to be used */
68 #if __SIZEOF_POINTER__ == 8
69 #define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT64)
70 #else
71 #define IS_INT(type) ((type) >= FFI_TYPE_UINT8 && (type) <= FFI_TYPE_SINT32)
72 #endif
73
74 #if ABI_FLEN
75 typedef struct {
76 char as_elements, type1, offset2, type2;
77 } float_struct_info;
78
79 #if ABI_FLEN >= 64
80 #define IS_FLOAT(type) ((type) >= FFI_TYPE_FLOAT && (type) <= FFI_TYPE_DOUBLE)
81 #else
82 #define IS_FLOAT(type) ((type) == FFI_TYPE_FLOAT)
83 #endif
84
85 static ffi_type **flatten_struct(ffi_type *in, ffi_type **out, ffi_type **out_end) {
86 int i;
87 if (out == out_end) return out;
88 if (in->type != FFI_TYPE_STRUCT) {
89 *(out++) = in;
90 } else {
91 for (i = 0; in->elements[i]; i++)
92 out = flatten_struct(in->elements[i], out, out_end);
93 }
94 return out;
95 }
96
97 /* Structs with at most two fields after flattening, one of which is of
98 floating point type, are passed in multiple registers if sufficient
99 registers are available. */
100 static float_struct_info struct_passed_as_elements(call_builder *cb, ffi_type *top) {
101 float_struct_info ret = {0, 0, 0, 0};
102 ffi_type *fields[3];
103 int num_floats, num_ints;
104 int num_fields = flatten_struct(top, fields, fields + 3) - fields;
105
106 if (num_fields == 1) {
107 if (IS_FLOAT(fields[0]->type)) {
108 ret.as_elements = 1;
109 ret.type1 = fields[0]->type;
110 }
111 } else if (num_fields == 2) {
112 num_floats = IS_FLOAT(fields[0]->type) + IS_FLOAT(fields[1]->type);
113 num_ints = IS_INT(fields[0]->type) + IS_INT(fields[1]->type);
114 if (num_floats == 0 || num_floats + num_ints != 2)
115 return ret;
116 if (cb->used_float + num_floats > NARGREG || cb->used_integer + (2 - num_floats) > NARGREG)
117 return ret;
118 if (!IS_FLOAT(fields[0]->type) && !IS_FLOAT(fields[1]->type))
119 return ret;
120
121 ret.type1 = fields[0]->type;
122 ret.type2 = fields[1]->type;
123 ret.offset2 = FFI_ALIGN(fields[0]->size, fields[1]->alignment);
124 ret.as_elements = 1;
125 }
126
127 return ret;
128 }
129 #endif
130
131 /* allocates a single register, float register, or XLEN-sized stack slot to a datum */
132 static void marshal_atom(call_builder *cb, int type, void *data) {
133 size_t value = 0;
134 switch (type) {
135 case FFI_TYPE_UINT8: value = *(uint8_t *)data; break;
136 case FFI_TYPE_SINT8: value = *(int8_t *)data; break;
137 case FFI_TYPE_UINT16: value = *(uint16_t *)data; break;
138 case FFI_TYPE_SINT16: value = *(int16_t *)data; break;
139 /* 32-bit quantities are always sign-extended in the ABI */
140 case FFI_TYPE_UINT32: value = *(int32_t *)data; break;
141 case FFI_TYPE_SINT32: value = *(int32_t *)data; break;
142 #if __SIZEOF_POINTER__ == 8
143 case FFI_TYPE_UINT64: value = *(uint64_t *)data; break;
144 case FFI_TYPE_SINT64: value = *(int64_t *)data; break;
145 #endif
146 case FFI_TYPE_POINTER: value = *(size_t *)data; break;
147
148 /* float values may be recoded in an implementation-defined way
149 by hardware conforming to 2.1 or earlier, so use asm to
150 reinterpret floats as doubles */
151 #if ABI_FLEN >= 32
152 case FFI_TYPE_FLOAT:
153 asm("" : "=f"(cb->aregs->fa[cb->used_float++]) : "0"(*(float *)data));
154 return;
155 #endif
156 #if ABI_FLEN >= 64
157 case FFI_TYPE_DOUBLE:
158 asm("" : "=f"(cb->aregs->fa[cb->used_float++]) : "0"(*(double *)data));
159 return;
160 #endif
161 default: FFI_ASSERT(0); break;
162 }
163
164 if (cb->used_integer == NARGREG) {
165 *cb->used_stack++ = value;
166 } else {
167 cb->aregs->a[cb->used_integer++] = value;
168 }
169 }
170
171 static void unmarshal_atom(call_builder *cb, int type, void *data) {
172 size_t value;
173 switch (type) {
174 #if ABI_FLEN >= 32
175 case FFI_TYPE_FLOAT:
176 asm("" : "=f"(*(float *)data) : "0"(cb->aregs->fa[cb->used_float++]));
177 return;
178 #endif
179 #if ABI_FLEN >= 64
180 case FFI_TYPE_DOUBLE:
181 asm("" : "=f"(*(double *)data) : "0"(cb->aregs->fa[cb->used_float++]));
182 return;
183 #endif
184 }
185
186 if (cb->used_integer == NARGREG) {
187 value = *cb->used_stack++;
188 } else {
189 value = cb->aregs->a[cb->used_integer++];
190 }
191
192 switch (type) {
193 case FFI_TYPE_UINT8: *(uint8_t *)data = value; break;
194 case FFI_TYPE_SINT8: *(uint8_t *)data = value; break;
195 case FFI_TYPE_UINT16: *(uint16_t *)data = value; break;
196 case FFI_TYPE_SINT16: *(uint16_t *)data = value; break;
197 case FFI_TYPE_UINT32: *(uint32_t *)data = value; break;
198 case FFI_TYPE_SINT32: *(uint32_t *)data = value; break;
199 #if __SIZEOF_POINTER__ == 8
200 case FFI_TYPE_UINT64: *(uint64_t *)data = value; break;
201 case FFI_TYPE_SINT64: *(uint64_t *)data = value; break;
202 #endif
203 case FFI_TYPE_POINTER: *(size_t *)data = value; break;
204 default: FFI_ASSERT(0); break;
205 }
206 }
207
208 /* adds an argument to a call, or a not by reference return value */
209 static void marshal(call_builder *cb, ffi_type *type, int var, void *data) {
210 size_t realign[2];
211
212 #if ABI_FLEN
213 if (!var && type->type == FFI_TYPE_STRUCT) {
214 float_struct_info fsi = struct_passed_as_elements(cb, type);
215 if (fsi.as_elements) {
216 marshal_atom(cb, fsi.type1, data);
217 if (fsi.offset2)
218 marshal_atom(cb, fsi.type2, ((char*)data) + fsi.offset2);
219 return;
220 }
221 }
222
223 if (!var && cb->used_float < NARGREG && IS_FLOAT(type->type)) {
224 marshal_atom(cb, type->type, data);
225 return;
226 }
227 #endif
228
229 if (type->size > 2 * __SIZEOF_POINTER__) {
230 /* pass by reference */
231 marshal_atom(cb, FFI_TYPE_POINTER, &data);
232 } else if (IS_INT(type->type) || type->type == FFI_TYPE_POINTER) {
233 marshal_atom(cb, type->type, data);
234 } else {
235 /* overlong integers, soft-float floats, and structs without special
236 float handling are treated identically from this point on */
237
238 /* variadics are aligned even in registers */
239 if (type->alignment > __SIZEOF_POINTER__) {
240 if (var)
241 cb->used_integer = FFI_ALIGN(cb->used_integer, 2);
242 cb->used_stack = (size_t *)FFI_ALIGN(cb->used_stack, 2*__SIZEOF_POINTER__);
243 }
244
245 memcpy(realign, data, type->size);
246 if (type->size > 0)
247 marshal_atom(cb, FFI_TYPE_POINTER, realign);
248 if (type->size > __SIZEOF_POINTER__)
249 marshal_atom(cb, FFI_TYPE_POINTER, realign + 1);
250 }
251 }
252
253 /* for arguments passed by reference returns the pointer, otherwise the arg is copied (up to MAXCOPYARG bytes) */
254 static void *unmarshal(call_builder *cb, ffi_type *type, int var, void *data) {
255 size_t realign[2];
256 void *pointer;
257
258 #if ABI_FLEN
259 if (!var && type->type == FFI_TYPE_STRUCT) {
260 float_struct_info fsi = struct_passed_as_elements(cb, type);
261 if (fsi.as_elements) {
262 unmarshal_atom(cb, fsi.type1, data);
263 if (fsi.offset2)
264 unmarshal_atom(cb, fsi.type2, ((char*)data) + fsi.offset2);
265 return data;
266 }
267 }
268
269 if (!var && cb->used_float < NARGREG && IS_FLOAT(type->type)) {
270 unmarshal_atom(cb, type->type, data);
271 return data;
272 }
273 #endif
274
275 if (type->size > 2 * __SIZEOF_POINTER__) {
276 /* pass by reference */
277 unmarshal_atom(cb, FFI_TYPE_POINTER, (char*)&pointer);
278 return pointer;
279 } else if (IS_INT(type->type) || type->type == FFI_TYPE_POINTER) {
280 unmarshal_atom(cb, type->type, data);
281 return data;
282 } else {
283 /* overlong integers, soft-float floats, and structs without special
284 float handling are treated identically from this point on */
285
286 /* variadics are aligned even in registers */
287 if (type->alignment > __SIZEOF_POINTER__) {
288 if (var)
289 cb->used_integer = FFI_ALIGN(cb->used_integer, 2);
290 cb->used_stack = (size_t *)FFI_ALIGN(cb->used_stack, 2*__SIZEOF_POINTER__);
291 }
292
293 if (type->size > 0)
294 unmarshal_atom(cb, FFI_TYPE_POINTER, realign);
295 if (type->size > __SIZEOF_POINTER__)
296 unmarshal_atom(cb, FFI_TYPE_POINTER, realign + 1);
297 memcpy(data, realign, type->size);
298 return data;
299 }
300 }
301
302 static int passed_by_ref(call_builder *cb, ffi_type *type, int var) {
303 #if ABI_FLEN
304 if (!var && type->type == FFI_TYPE_STRUCT) {
305 float_struct_info fsi = struct_passed_as_elements(cb, type);
306 if (fsi.as_elements) return 0;
307 }
308 #endif
309
310 return type->size > 2 * __SIZEOF_POINTER__;
311 }
312
313 /* Perform machine dependent cif processing */
314 ffi_status ffi_prep_cif_machdep(ffi_cif *cif) {
315 cif->riscv_nfixedargs = cif->nargs;
316 return FFI_OK;
317 }
318
319 /* Perform machine dependent cif processing when we have a variadic function */
320
321 ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif, unsigned int nfixedargs, unsigned int ntotalargs) {
322 cif->riscv_nfixedargs = nfixedargs;
323 return FFI_OK;
324 }
325
326 /* Low level routine for calling functions */
327 extern void ffi_call_asm (void *stack, struct call_context *regs,
328 void (*fn) (void), void *closure) FFI_HIDDEN;
329
330 static void
331 ffi_call_int (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue,
332 void *closure)
333 {
334 /* this is a conservative estimate, assuming a complex return value and
335 that all remaining arguments are long long / __int128 */
336 size_t arg_bytes = cif->nargs <= 3 ? 0 :
337 FFI_ALIGN(2 * sizeof(size_t) * (cif->nargs - 3), STKALIGN);
338 size_t rval_bytes = 0;
339 if (rvalue == NULL && cif->rtype->size > 2*__SIZEOF_POINTER__)
340 rval_bytes = FFI_ALIGN(cif->rtype->size, STKALIGN);
341 size_t alloc_size = arg_bytes + rval_bytes + sizeof(call_context);
342
343 /* the assembly code will deallocate all stack data at lower addresses
344 than the argument region, so we need to allocate the frame and the
345 return value after the arguments in a single allocation */
346 size_t alloc_base;
347 /* Argument region must be 16-byte aligned */
348 if (_Alignof(max_align_t) >= STKALIGN) {
349 /* since sizeof long double is normally 16, the compiler will
350 guarantee alloca alignment to at least that much */
351 alloc_base = (size_t)alloca(alloc_size);
352 } else {
353 alloc_base = FFI_ALIGN(alloca(alloc_size + STKALIGN - 1), STKALIGN);
354 }
355
356 if (rval_bytes)
357 rvalue = (void*)(alloc_base + arg_bytes);
358
359 call_builder cb;
360 cb.used_float = cb.used_integer = 0;
361 cb.aregs = (call_context*)(alloc_base + arg_bytes + rval_bytes);
362 cb.used_stack = (void*)alloc_base;
363
364 int return_by_ref = passed_by_ref(&cb, cif->rtype, 0);
365 if (return_by_ref)
366 marshal(&cb, &ffi_type_pointer, 0, &rvalue);
367
368 int i;
369 for (i = 0; i < cif->nargs; i++)
370 marshal(&cb, cif->arg_types[i], i >= cif->riscv_nfixedargs, avalue[i]);
371
372 ffi_call_asm ((void *) alloc_base, cb.aregs, fn, closure);
373
374 cb.used_float = cb.used_integer = 0;
375 if (!return_by_ref && rvalue)
376 unmarshal(&cb, cif->rtype, 0, rvalue);
377 }
378
379 void
380 ffi_call (ffi_cif *cif, void (*fn) (void), void *rvalue, void **avalue)
381 {
382 ffi_call_int(cif, fn, rvalue, avalue, NULL);
383 }
384
385 void
386 ffi_call_go (ffi_cif *cif, void (*fn) (void), void *rvalue,
387 void **avalue, void *closure)
388 {
389 ffi_call_int(cif, fn, rvalue, avalue, closure);
390 }
391
392 extern void ffi_closure_asm(void) FFI_HIDDEN;
393
394 ffi_status ffi_prep_closure_loc(ffi_closure *closure, ffi_cif *cif, void (*fun)(ffi_cif*,void*,void**,void*), void *user_data, void *codeloc)
395 {
396 uint32_t *tramp = (uint32_t *) &closure->tramp[0];
397 uint64_t fn = (uint64_t) (uintptr_t) ffi_closure_asm;
398
399 if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
400 return FFI_BAD_ABI;
401
402 /* we will call ffi_closure_inner with codeloc, not closure, but as long
403 as the memory is readable it should work */
404
405 tramp[0] = 0x00000317; /* auipc t1, 0 (i.e. t0 <- codeloc) */
406 #if __SIZEOF_POINTER__ == 8
407 tramp[1] = 0x01033383; /* ld t2, 16(t1) */
408 #else
409 tramp[1] = 0x01032383; /* lw t2, 16(t1) */
410 #endif
411 tramp[2] = 0x00038067; /* jr t2 */
412 tramp[3] = 0x00000013; /* nop */
413 tramp[4] = fn;
414 tramp[5] = fn >> 32;
415
416 closure->cif = cif;
417 closure->fun = fun;
418 closure->user_data = user_data;
419
420 __builtin___clear_cache(codeloc, codeloc + FFI_TRAMPOLINE_SIZE);
421
422 return FFI_OK;
423 }
424
425 extern void ffi_go_closure_asm (void) FFI_HIDDEN;
426
427 ffi_status
428 ffi_prep_go_closure (ffi_go_closure *closure, ffi_cif *cif,
429 void (*fun) (ffi_cif *, void *, void **, void *))
430 {
431 if (cif->abi <= FFI_FIRST_ABI || cif->abi >= FFI_LAST_ABI)
432 return FFI_BAD_ABI;
433
434 closure->tramp = (void *) ffi_go_closure_asm;
435 closure->cif = cif;
436 closure->fun = fun;
437
438 return FFI_OK;
439 }
440
441 /* Called by the assembly code with aregs pointing to saved argument registers
442 and stack pointing to the stacked arguments. Return values passed in
443 registers will be reloaded from aregs. */
444 void FFI_HIDDEN
445 ffi_closure_inner (ffi_cif *cif,
446 void (*fun) (ffi_cif *, void *, void **, void *),
447 void *user_data,
448 size_t *stack, call_context *aregs)
449 {
450 void **avalue = alloca(cif->nargs * sizeof(void*));
451 /* storage for arguments which will be copied by unmarshal(). We could
452 theoretically avoid the copies in many cases and use at most 128 bytes
453 of memory, but allocating disjoint storage for each argument is
454 simpler. */
455 char *astorage = alloca(cif->nargs * MAXCOPYARG);
456 void *rvalue;
457 call_builder cb;
458 int return_by_ref;
459 int i;
460
461 cb.aregs = aregs;
462 cb.used_integer = cb.used_float = 0;
463 cb.used_stack = stack;
464
465 return_by_ref = passed_by_ref(&cb, cif->rtype, 0);
466 if (return_by_ref)
467 unmarshal(&cb, &ffi_type_pointer, 0, &rvalue);
468 else
469 rvalue = alloca(cif->rtype->size);
470
471 for (i = 0; i < cif->nargs; i++)
472 avalue[i] = unmarshal(&cb, cif->arg_types[i],
473 i >= cif->riscv_nfixedargs, astorage + i*MAXCOPYARG);
474
475 fun (cif, rvalue, avalue, user_data);
476
477 if (!return_by_ref && cif->rtype->type != FFI_TYPE_VOID) {
478 cb.used_integer = cb.used_float = 0;
479 marshal(&cb, cif->rtype, 0, rvalue);
480 }
481 }