1 /* Shared HTM header. Work around false transactional execution facility
2 intrinsics.
3
4 Copyright (C) 2016-2023 Free Software Foundation, Inc.
5 This file is part of the GNU C Library.
6
7 The GNU C Library is free software; you can redistribute it and/or
8 modify it under the terms of the GNU Lesser General Public
9 License as published by the Free Software Foundation; either
10 version 2.1 of the License, or (at your option) any later version.
11
12 The GNU C Library is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 Lesser General Public License for more details.
16
17 You should have received a copy of the GNU Lesser General Public
18 License along with the GNU C Library; if not, see
19 <https://www.gnu.org/licenses/>. */
20
21 #ifndef _HTM_H
22 #define _HTM_H 1
23
24 #include <htmintrin.h>
25
26 #ifdef __s390x__
27 # define TX_FPRS_BYTES 64
28 # define TX_SAVE_FPRS \
29 " std %%f8, 0(%[R_FPRS])\n\t" \
30 " std %%f9, 8(%[R_FPRS])\n\t" \
31 " std %%f10, 16(%[R_FPRS])\n\t" \
32 " std %%f11, 24(%[R_FPRS])\n\t" \
33 " std %%f12, 32(%[R_FPRS])\n\t" \
34 " std %%f13, 40(%[R_FPRS])\n\t" \
35 " std %%f14, 48(%[R_FPRS])\n\t" \
36 " std %%f15, 56(%[R_FPRS])\n\t"
37
38 # define TX_RESTORE_FPRS \
39 " ld %%f8, 0(%[R_FPRS])\n\t" \
40 " ld %%f9, 8(%[R_FPRS])\n\t" \
41 " ld %%f10, 16(%[R_FPRS])\n\t" \
42 " ld %%f11, 24(%[R_FPRS])\n\t" \
43 " ld %%f12, 32(%[R_FPRS])\n\t" \
44 " ld %%f13, 40(%[R_FPRS])\n\t" \
45 " ld %%f14, 48(%[R_FPRS])\n\t" \
46 " ld %%f15, 56(%[R_FPRS])\n\t"
47
48 #else
49
50 # define TX_FPRS_BYTES 16
51 # define TX_SAVE_FPRS \
52 " std %%f4, 0(%[R_FPRS])\n\t" \
53 " std %%f6, 8(%[R_FPRS])\n\t"
54
55 # define TX_RESTORE_FPRS \
56 " ld %%f4, 0(%[R_FPRS])\n\t" \
57 " ld %%f6, 8(%[R_FPRS])\n\t"
58
59 #endif /* ! __s390x__ */
60
61 /* Use own inline assembly instead of __builtin_tbegin, as tbegin
62 has to filter program interruptions which can't be done with the builtin.
63 Now the fprs have to be saved / restored here, too.
64 The fpc is also not saved / restored with the builtin.
65 The used inline assembly does not clobber the volatile fprs / vrs!
66 Clobbering the latter ones would force the compiler to save / restore
67 the call saved fprs as those overlap with the vrs, but they only need to be
68 restored if the transaction fails but not if the transaction is successfully
69 started. Thus the user of the tbegin macros in this header file has to
70 compile the file / function with -msoft-float. It prevents gcc from using
71 fprs / vrs. */
72 #define __libc_tbegin(tdb) __libc_tbegin_base(tdb,,,)
73
74 #define __libc_tbegin_retry_output_regs , [R_TX_CNT] "+&d" (__tx_cnt)
75 #define __libc_tbegin_retry_input_regs(retry_cnt) , [R_RETRY] "d" (retry_cnt)
76 #define __libc_tbegin_retry_abort_path_insn \
77 /* If tbegin returned _HTM_TBEGIN_TRANSIENT, retry immediately so \
78 that max tbegin_cnt transactions are tried. Otherwise return and \
79 let the caller of this macro do the fallback path. */ \
80 " jnh 1f\n\t" /* cc 1/3: jump to fallback path. */ \
81 /* tbegin returned _HTM_TBEGIN_TRANSIENT: retry with transaction. */ \
82 " crje %[R_TX_CNT], %[R_RETRY], 1f\n\t" /* Reached max retries? */ \
83 " ahi %[R_TX_CNT], 1\n\t" \
84 " ppa %[R_TX_CNT], 0, 1\n\t" /* Transaction-Abort Assist. */ \
85 " j 2b\n\t" /* Loop to tbegin. */
86
87 /* Same as __libc_tbegin except if tbegin aborts with _HTM_TBEGIN_TRANSIENT.
88 Then this macros restores the fpc, fprs and automatically retries up to
89 retry_cnt tbegins. Further saving of the state is omitted as it is already
90 saved. This macro calls tbegin at most as retry_cnt + 1 times. */
91 #define __libc_tbegin_retry(tdb, retry_cnt) \
92 ({ int __ret; \
93 int __tx_cnt = 0; \
94 __ret = __libc_tbegin_base(tdb, \
95 __libc_tbegin_retry_abort_path_insn, \
96 __libc_tbegin_retry_output_regs, \
97 __libc_tbegin_retry_input_regs(retry_cnt)); \
98 __ret; \
99 })
100
101 #define __libc_tbegin_base(tdb, abort_path_insn, output_regs, input_regs) \
102 ({ int __ret; \
103 int __fpc; \
104 char __fprs[TX_FPRS_BYTES]; \
105 __asm__ __volatile__ (".machine push\n\t" \
106 ".machinemode \"zarch_nohighgprs\"\n\t" \
107 ".machine \"all\"\n\t" \
108 /* Save state at the outermost transaction. \
109 As extracting nesting depth is expensive \
110 on at least zEC12, save fprs at inner \
111 transactions, too. \
112 The fpc and fprs are saved here as they \
113 are not saved by tbegin. There exist no \
114 call-saved vrs, thus they are not saved \
115 here. */ \
116 " efpc %[R_FPC]\n\t" \
117 TX_SAVE_FPRS \
118 /* Begin transaction: save all gprs, allow \
119 ar modification and fp operations. Some \
120 program-interruptions (e.g. a null \
121 pointer access) are filtered and the \
122 transaction will abort. In this case \
123 the normal lock path will execute it \
124 again and result in a core dump which does \
125 now show at tbegin but the real executed \
126 instruction. \
127 However it is not guaranteed that this \
128 retry operate on the same data and thus \
129 may not end in an program-interruption. \
130 Note: This could also be used to probe \
131 memory for being accessible! */ \
132 "2: tbegin 0, 0xFF0E\n\t" \
133 /* Branch away in abort case (this is the \
134 preferred sequence. See PoP in chapter 5 \
135 Transactional-Execution Facility \
136 Operation). */ \
137 " jnz 0f\n\t" \
138 /* Transaction has successfully started. */ \
139 " lhi %[R_RET], 0\n\t" \
140 " j 1f\n\t" \
141 /* Transaction has aborted. Now we are at \
142 the outermost transaction. Restore fprs \
143 and fpc. */ \
144 "0: ipm %[R_RET]\n\t" \
145 " srl %[R_RET], 28\n\t" \
146 " sfpc %[R_FPC]\n\t" \
147 TX_RESTORE_FPRS \
148 abort_path_insn \
149 "1:\n\t" \
150 ".machine pop\n" \
151 : [R_RET] "=&d" (__ret), \
152 [R_FPC] "=&d" (__fpc) \
153 output_regs \
154 : [R_FPRS] "a" (__fprs) \
155 input_regs \
156 : "cc", "memory"); \
157 __ret; \
158 })
159
160 /* These builtins are usable in context of glibc lock elision code without any
161 changes. Use them. */
162 #define __libc_tend() \
163 ({ __asm__ __volatile__ (".machine push\n\t" \
164 ".machinemode \"zarch_nohighgprs\"\n\t" \
165 ".machine \"all\"\n\t"); \
166 int __ret = __builtin_tend (); \
167 __asm__ __volatile__ (".machine pop"); \
168 __ret; \
169 })
170
171 #define __libc_tabort(abortcode) \
172 __asm__ __volatile__ (".machine push\n\t" \
173 ".machinemode \"zarch_nohighgprs\"\n\t" \
174 ".machine \"all\"\n\t"); \
175 __builtin_tabort (abortcode); \
176 __asm__ __volatile__ (".machine pop")
177
178 #define __libc_tx_nesting_depth() \
179 ({ __asm__ __volatile__ (".machine push\n\t" \
180 ".machinemode \"zarch_nohighgprs\"\n\t" \
181 ".machine \"all\"\n\t"); \
182 int __ret = __builtin_tx_nesting_depth (); \
183 __asm__ __volatile__ (".machine pop"); \
184 __ret; \
185 })
186
187 #endif