1 /* Assembly macros for 64-bit PowerPC.
2 Copyright (C) 2002-2023 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19 #include <sysdeps/powerpc/sysdep.h>
20 #include <tls.h>
21
22 #ifdef __ASSEMBLER__
23
24 /* Stack frame offsets. */
25 #define FRAME_BACKCHAIN 0
26 #define FRAME_CR_SAVE 8
27 #define FRAME_LR_SAVE 16
28 #if _CALL_ELF != 2
29 #define FRAME_MIN_SIZE 112
30 #define FRAME_MIN_SIZE_PARM 112
31 #define FRAME_TOC_SAVE 40
32 #define FRAME_PARM_SAVE 48
33 #else
34 #define FRAME_MIN_SIZE 32
35 #define FRAME_MIN_SIZE_PARM 96
36 #define FRAME_TOC_SAVE 24
37 #define FRAME_PARM_SAVE 32
38 #endif
39
40 /* Support macros for CALL_MCOUNT. */
41 .macro SAVE_ARG NARG
42 .if \NARG
43 SAVE_ARG \NARG-1
44 std 2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG)(1)
45 .endif
46 .endm
47
48 .macro REST_ARG NARG
49 .if \NARG
50 REST_ARG \NARG-1
51 ld 2+\NARG,FRAME_PARM_SAVE-8+8*(\NARG)(1)
52 .endif
53 .endm
54
55 .macro CFI_SAVE_ARG NARG
56 .if \NARG
57 CFI_SAVE_ARG \NARG-1
58 cfi_offset(2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG))
59 .endif
60 .endm
61
62 .macro CFI_REST_ARG NARG
63 .if \NARG
64 CFI_REST_ARG \NARG-1
65 cfi_restore(2+\NARG)
66 .endif
67 .endm
68
69 /* If compiled for profiling, call `_mcount' at the start of each function.
70 see ppc-mcount.S for more details. */
71 .macro CALL_MCOUNT NARG
72 #ifdef PROF
73 mflr r0
74 SAVE_ARG \NARG
75 std r0,FRAME_LR_SAVE(r1)
76 stdu r1,-FRAME_MIN_SIZE_PARM(r1)
77 cfi_adjust_cfa_offset(FRAME_MIN_SIZE_PARM)
78 cfi_offset(lr,FRAME_LR_SAVE)
79 CFI_SAVE_ARG \NARG
80 bl JUMPTARGET (_mcount)
81 #ifndef SHARED
82 nop
83 #endif
84 ld r0,FRAME_MIN_SIZE_PARM+FRAME_LR_SAVE(r1)
85 REST_ARG \NARG
86 mtlr r0
87 addi r1,r1,FRAME_MIN_SIZE_PARM
88 cfi_adjust_cfa_offset(-FRAME_MIN_SIZE_PARM)
89 cfi_restore(lr)
90 CFI_REST_ARG \NARG
91 #endif
92 .endm
93
94 #if _CALL_ELF != 2
95
96 /* Macro to prepare for calling via a function pointer. */
97 .macro PPC64_LOAD_FUNCPTR PTR
98 ld r12,0(\PTR)
99 ld r2,8(\PTR)
100 mtctr r12
101 ld r11,16(\PTR)
102 .endm
103
104 #ifdef USE_PPC64_OVERLAPPING_OPD
105 # define OPD_ENT(name) .quad BODY_LABEL (name), .TOC.@tocbase
106 #else
107 # define OPD_ENT(name) .quad BODY_LABEL (name), .TOC.@tocbase, 0
108 #endif
109
110 #define ENTRY_1(name) \
111 .type BODY_LABEL(name),@function; \
112 .globl name; \
113 .section ".opd","aw"; \
114 .p2align 3;FUNC_LABEL(name): \
115 OPD_ENT (name); \
116 .previous
117
118 #define FUNC_LABEL(X) X
119 #define BODY_LABEL(X) .LY##X
120 #define ENTRY_2(name) \
121 .type name,@function; \
122 ENTRY_1(name)
123 #define END_2(name) \
124 .size name,.-BODY_LABEL(name); \
125 .size BODY_LABEL(name),.-BODY_LABEL(name)
126 #define LOCALENTRY(name)
127
128 #else /* _CALL_ELF == 2 */
129
130 /* Macro to prepare for calling via a function pointer. */
131 .macro PPC64_LOAD_FUNCPTR PTR
132 mr r12,\PTR
133 mtctr r12
134 .endm
135
136 #define FUNC_LABEL(X) X
137 #define BODY_LABEL(X) X
138 #define ENTRY_2(name) \
139 .globl name; \
140 .type name,@function
141 #define END_2(name) \
142 .size name,.-name
143 #define LOCALENTRY(name) \
144 1: addis r2,r12,.TOC.-1b@ha; \
145 addi r2,r2,.TOC.-1b@l; \
146 .localentry name,.-name
147
148 #endif /* _CALL_ELF */
149
150 .macro NOPS NARG
151 .if \NARG
152 NOPS \NARG-1
153 nop
154 .endif
155 .endm
156
157 .macro ENTRY_3 name, alignp2=2, nopwords=0
158 .text
159 ENTRY_2(\name)
160 .p2align \alignp2
161 NOPS \nopwords
162 BODY_LABEL(\name):
163 .endm
164
165 /* Use ENTRY_TOCLESS for functions that make no use of r2 and
166 guarantee r2 is unchanged on exit. Any function that has @toc or
167 @got relocs uses r2. Functions that call other functions via the
168 PLT use r2. Use ENTRY for functions that may use or change r2.
169 The first argument is the function name.
170 The optional second argument specifies alignment of the function's
171 code, as the logarithm base two of the byte alignment. For
172 example, a value of four aligns to a sixteen byte boundary.
173 The optional third argument specifies the number of NOPs to emit
174 before the start of the function's code. */
175 #ifndef PROF
176 #define ENTRY_TOCLESS(name, ...) \
177 ENTRY_3 name, ## __VA_ARGS__; \
178 cfi_startproc
179
180 #define ENTRY(name, ...) \
181 ENTRY_TOCLESS(name, ## __VA_ARGS__); \
182 LOCALENTRY(name)
183 #else
184 /* The call to _mcount is potentially via the plt, so profiling code
185 is never free of an r2 use. */
186 #define ENTRY_TOCLESS(name, ...) \
187 ENTRY_3 name, ## __VA_ARGS__; \
188 cfi_startproc; \
189 LOCALENTRY(name)
190
191 #define ENTRY(name, ...) \
192 ENTRY_TOCLESS(name, ## __VA_ARGS__)
193 #endif
194
195 /* Local labels stripped out by the linker. */
196 #undef L
197 #define L(x) .L##x
198
199 #define tostring(s) #s
200 #define stringify(s) tostring(s)
201 #define XGLUE(a,b) a##b
202 #define GLUE(a,b) XGLUE(a,b)
203 #define LT_LABEL(name) GLUE(.LT,name)
204 #define LT_LABELSUFFIX(name,suffix) GLUE(GLUE(.LT,name),suffix)
205
206 /* Support Traceback tables */
207 #define TB_ASM 0x000c000000000000
208 #define TB_GLOBALLINK 0x0000800000000000
209 #define TB_IS_EPROL 0x0000400000000000
210 #define TB_HAS_TBOFF 0x0000200000000000
211 #define TB_INT_PROC 0x0000100000000000
212 #define TB_HAS_CTL 0x0000080000000000
213 #define TB_TOCLESS 0x0000040000000000
214 #define TB_FP_PRESENT 0x0000020000000000
215 #define TB_LOG_ABORT 0x0000010000000000
216 #define TB_INT_HANDL 0x0000008000000000
217 #define TB_NAME_PRESENT 0x0000004000000000
218 #define TB_USES_ALLOCA 0x0000002000000000
219 #define TB_SAVES_CR 0x0000000200000000
220 #define TB_SAVES_LR 0x0000000100000000
221 #define TB_STORES_BC 0x0000000080000000
222 #define TB_FIXUP 0x0000000040000000
223 #define TB_FP_SAVED(fprs) (((fprs) & 0x3f) << 24)
224 #define TB_GPR_SAVED(gprs) (((fprs) & 0x3f) << 16)
225 #define TB_FIXEDPARMS(parms) (((parms) & 0xff) << 8)
226 #define TB_FLOATPARMS(parms) (((parms) & 0x7f) << 1)
227 #define TB_PARMSONSTK 0x0000000000000001
228
229 #define PPC_HIGHER(v) (((v) >> 32) & 0xffff)
230 #define TB_DEFAULT TB_ASM | TB_HAS_TBOFF | TB_NAME_PRESENT
231
232 #define TRACEBACK(name) \
233 LT_LABEL(name): ; \
234 .long 0 ; \
235 .quad TB_DEFAULT ; \
236 .long LT_LABEL(name)-BODY_LABEL(name) ; \
237 .short LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
238 LT_LABELSUFFIX(name,_name_start): ;\
239 .ascii stringify(name) ; \
240 LT_LABELSUFFIX(name,_name_end): ; \
241 .p2align 2
242
243 #define TRACEBACK_MASK(name,mask) \
244 LT_LABEL(name): ; \
245 .long 0 ; \
246 .quad TB_DEFAULT | mask ; \
247 .long LT_LABEL(name)-BODY_LABEL(name) ; \
248 .short LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
249 LT_LABELSUFFIX(name,_name_start): ;\
250 .ascii stringify(name) ; \
251 LT_LABELSUFFIX(name,_name_end): ; \
252 .p2align 2
253
254 /* END generates Traceback tables */
255 #undef END
256 #define END(name) \
257 cfi_endproc; \
258 TRACEBACK(name); \
259 END_2(name)
260
261 /* This form supports more informative traceback tables */
262 #define END_GEN_TB(name,mask) \
263 cfi_endproc; \
264 TRACEBACK_MASK(name,mask); \
265 END_2(name)
266
267 /* We will allocate a new frame to save LR and the non-volatile register used to
268 read the TCB when checking for scv support on syscall code. We actually just
269 need the minimum frame size plus room for 1 reg (8 bytes). But the ABI
270 mandates stack frames should be aligned at 16 Bytes, so we end up allocating
271 a bit more space then what will actually be used. */
272 #define SCV_FRAME_SIZE (FRAME_MIN_SIZE+16)
273 #define SCV_FRAME_NVOLREG_SAVE FRAME_MIN_SIZE
274
275 /* Allocate frame and save register */
276 #define NVOLREG_SAVE \
277 stdu r1,-SCV_FRAME_SIZE(r1); \
278 cfi_adjust_cfa_offset(SCV_FRAME_SIZE); \
279 std r31,SCV_FRAME_NVOLREG_SAVE(r1); \
280 cfi_rel_offset(r31,SCV_FRAME_NVOLREG_SAVE);
281
282 /* Restore register and destroy frame */
283 #define NVOLREG_RESTORE \
284 ld r31,SCV_FRAME_NVOLREG_SAVE(r1); \
285 cfi_restore(r31); \
286 addi r1,r1,SCV_FRAME_SIZE; \
287 cfi_adjust_cfa_offset(-SCV_FRAME_SIZE);
288
289 /* Check PPC_FEATURE2_SCV bit from hwcap2 in the TCB. If it is not set, scv is
290 not available, then go to JUMPFALSE (label given by the macro's caller). We
291 save the value we read from the TCB in a non-volatile register so we can
292 reuse it later when exiting from the syscall in PSEUDO_RET. Note that for
293 the static case we need an extra check to guarantee the thread pointer has
294 already been initialized, otherwise we may try to access an invalid address
295 if a syscall is called before the TLS has been setup. */
296 .macro CHECK_SCV_SUPPORT REG JUMPFALSE
297
298 #ifndef SHARED
299 /* Check if thread pointer has already been setup. */
300 cmpdi r13,0
301 beq \JUMPFALSE
302 #endif
303
304 /* Read PPC_FEATURE2_SCV from TCB and store it in REG */
305 ld \REG,TCB_HWCAP(PT_THREAD_POINTER)
306 andis. \REG,\REG,PPC_FEATURE2_SCV>>16
307
308 beq \JUMPFALSE
309 .endm
310
311 #if !defined(USE_PPC_SCV) || IS_IN(rtld)
312 # define DO_CALL(syscall) \
313 li r0,syscall; \
314 DO_CALL_SC
315 #else
316 /* Before doing the syscall, check if we can use scv. scv is supported by P9
317 and later with Linux v5.9 and later. If so, use it. Otherwise, fallback to
318 sc. We use a non-volatile register to save hwcap2 from the TCB, so we need
319 to save its content beforehand. */
320 # define DO_CALL(syscall) \
321 li r0,syscall; \
322 NVOLREG_SAVE; \
323 CHECK_SCV_SUPPORT r31 0f; \
324 DO_CALL_SCV; \
325 b 1f; \
326 0: DO_CALL_SC; \
327 1:
328 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
329
330 /* DO_CALL_SC and DO_CALL_SCV expect the syscall number to be in r0. */
331 #define DO_CALL_SC \
332 sc
333
334 #define DO_CALL_SCV \
335 mflr r9; \
336 std r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1); \
337 cfi_rel_offset(lr,SCV_FRAME_SIZE+FRAME_LR_SAVE); \
338 .machine "push"; \
339 .machine "power9"; \
340 scv 0; \
341 .machine "pop"; \
342 ld r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1); \
343 mtlr r9; \
344 cfi_restore(lr);
345
346 /* ppc64 is always PIC */
347 #undef JUMPTARGET
348 #define JUMPTARGET(name) FUNC_LABEL(name)
349
350 #define PSEUDO(name, syscall_name, args) \
351 .section ".text"; \
352 ENTRY (name); \
353 DO_CALL (SYS_ify (syscall_name))
354
355 #ifdef SHARED
356 #define TAIL_CALL_SYSCALL_ERROR \
357 b JUMPTARGET (NOTOC (__syscall_error))
358 #else
359 /* Static version might be linked into a large app with a toc exceeding
360 64k. We can't put a toc adjusting stub on a plain branch, so can't
361 tail call __syscall_error. */
362 #define TAIL_CALL_SYSCALL_ERROR \
363 .ifdef .Local_syscall_error; \
364 b .Local_syscall_error; \
365 .else; \
366 .Local_syscall_error: \
367 mflr 0; \
368 std 0,FRAME_LR_SAVE(1); \
369 stdu 1,-FRAME_MIN_SIZE(1); \
370 cfi_adjust_cfa_offset(FRAME_MIN_SIZE); \
371 cfi_offset(lr,FRAME_LR_SAVE); \
372 bl JUMPTARGET(__syscall_error); \
373 nop; \
374 ld 0,FRAME_MIN_SIZE+FRAME_LR_SAVE(1); \
375 addi 1,1,FRAME_MIN_SIZE; \
376 cfi_adjust_cfa_offset(-FRAME_MIN_SIZE); \
377 mtlr 0; \
378 cfi_restore(lr); \
379 blr; \
380 .endif
381 #endif
382
383 #if !defined(USE_PPC_SCV) || IS_IN(rtld)
384 # define PSEUDO_RET \
385 RET_SC; \
386 TAIL_CALL_SYSCALL_ERROR
387 #else
388 /* This should only be called after a DO_CALL. In such cases, r31 contains the
389 value of PPC_FEATURE2_SCV read from hwcap2 by CHECK_SCV_SUPPORT. If it is
390 set, we know we have entered the kernel using scv, so handle the return code
391 accordingly. */
392 # define PSEUDO_RET \
393 cmpdi cr5,r31,0; \
394 NVOLREG_RESTORE; \
395 beq cr5,0f; \
396 RET_SCV; \
397 b 1f; \
398 0: RET_SC; \
399 1: TAIL_CALL_SYSCALL_ERROR
400 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
401
402 #define RET_SCV \
403 li r9,-4095; \
404 cmpld r3,r9; \
405 bltlr+; \
406 neg r3,r3;
407
408 #define RET_SC \
409 bnslr+;
410
411 #define ret PSEUDO_RET
412
413 #undef PSEUDO_END
414 #define PSEUDO_END(name) \
415 END (name)
416
417 #define PSEUDO_NOERRNO(name, syscall_name, args) \
418 .section ".text"; \
419 ENTRY (name); \
420 DO_CALL (SYS_ify (syscall_name))
421
422 #if !defined(USE_PPC_SCV) || IS_IN(rtld)
423 # define PSEUDO_RET_NOERRNO \
424 blr
425 #else
426 /* This should only be called after a DO_CALL. */
427 # define PSEUDO_RET_NOERRNO \
428 NVOLREG_RESTORE; \
429 blr
430 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
431
432 #define ret_NOERRNO PSEUDO_RET_NOERRNO
433
434 #undef PSEUDO_END_NOERRNO
435 #define PSEUDO_END_NOERRNO(name) \
436 END (name)
437
438 #define PSEUDO_ERRVAL(name, syscall_name, args) \
439 .section ".text"; \
440 ENTRY (name); \
441 DO_CALL (SYS_ify (syscall_name))
442
443 #if !defined(USE_PPC_SCV) || IS_IN(rtld)
444 # define PSEUDO_RET_ERRVAL \
445 blr
446 #else
447 /* This should only be called after a DO_CALL. */
448 # define PSEUDO_RET_ERRVAL \
449 NVOLREG_RESTORE; \
450 blr
451 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
452
453 #define ret_ERRVAL PSEUDO_RET_ERRVAL
454
455 #undef PSEUDO_END_ERRVAL
456 #define PSEUDO_END_ERRVAL(name) \
457 END (name)
458
459 #ifdef SHARED
460 # if IS_IN (rtld)
461 /* Inside ld.so we use the local alias to avoid runtime GOT
462 relocations. */
463 # define __GLRO_DEF(var) \
464 .LC__ ## var: \
465 .tc _rtld_local_ro[TC],_rtld_local_ro
466 # else
467 # define __GLRO_DEF(var) \
468 .LC__ ## var: \
469 .tc _rtld_global_ro[TC],_rtld_global_ro
470 # endif
471 # define __GLRO(rOUT, var, offset) \
472 addis rOUT,r2,.LC__ ## var@toc@ha; \
473 ld rOUT,.LC__ ## var@toc@l(rOUT); \
474 lwz rOUT,offset(rOUT)
475 #else
476 # define __GLRO_DEF(var) \
477 .LC__ ## var: \
478 .tc _ ## var[TC],_ ## var
479 # define __GLRO(rOUT, var, offset) \
480 addis rOUT,r2,.LC__ ## var@toc@ha; \
481 ld rOUT,.LC__ ## var@toc@l(rOUT); \
482 lwz rOUT,0(rOUT)
483 #endif
484
485 #ifdef USE_PPC64_NOTOC
486 # define NOTOC(l) l@notoc
487 #else
488 # define NOTOC(l) l
489 #endif
490
491 #else /* !__ASSEMBLER__ */
492
493 #if _CALL_ELF != 2
494
495 #define PPC64_LOAD_FUNCPTR(ptr) \
496 "ld 12,0(" #ptr ")\n" \
497 "ld 2,8(" #ptr ")\n" \
498 "mtctr 12\n" \
499 "ld 11,16(" #ptr ")"
500
501 #ifdef USE_PPC64_OVERLAPPING_OPD
502 # define OPD_ENT(name) ".quad " BODY_PREFIX #name ", .TOC.@tocbase"
503 #else
504 # define OPD_ENT(name) ".quad " BODY_PREFIX #name ", .TOC.@tocbase, 0"
505 #endif
506
507 #define ENTRY_1(name) \
508 ".type " BODY_PREFIX #name ",@function\n" \
509 ".globl " #name "\n" \
510 ".pushsection \".opd\",\"aw\"\n" \
511 ".p2align 3\n" \
512 #name ":\n" \
513 OPD_ENT (name) "\n" \
514 ".popsection"
515
516 #define DOT_PREFIX ""
517 #define BODY_PREFIX ".LY"
518 #define ENTRY_2(name) \
519 ".type " #name ",@function\n" \
520 ENTRY_1(name)
521 #define END_2(name) \
522 ".size " #name ",.-" BODY_PREFIX #name "\n" \
523 ".size " BODY_PREFIX #name ",.-" BODY_PREFIX #name
524 #define LOCALENTRY(name)
525
526 #else /* _CALL_ELF */
527
528 #define PPC64_LOAD_FUNCPTR(ptr) \
529 "mr 12," #ptr "\n" \
530 "mtctr 12"
531
532 #define DOT_PREFIX ""
533 #define BODY_PREFIX ""
534 #define ENTRY_2(name) \
535 ".type " #name ",@function\n" \
536 ".globl " #name
537 #define END_2(name) \
538 ".size " #name ",.-" #name
539 #define LOCALENTRY(name) \
540 "1: addis 2,12,.TOC.-1b@ha\n" \
541 "addi 2,2,.TOC.-1b@l\n" \
542 ".localentry " #name ",.-" #name
543
544 #endif /* _CALL_ELF */
545
546 #endif /* __ASSEMBLER__ */