1 /* Copyright (C) 2012-2023 Free Software Foundation, Inc.
2 Contributed by Richard Henderson <rth@redhat.com>.
3
4 This file is part of the GNU Atomic Library (libatomic).
5
6 Libatomic is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
9 any later version.
10
11 Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 more details.
15
16 Under Section 7 of GPL version 3, you are granted additional
17 permissions described in the GCC Runtime Library Exception, version
18 3.1, as published by the Free Software Foundation.
19
20 You should have received a copy of the GNU General Public License and
21 a copy of the GCC Runtime Library Exception along with this program;
22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 <http://www.gnu.org/licenses/>. */
24
25 /* This file contains data types and function declarations that are
26 private to the implementation of libatomic. */
27
28 #ifndef LIBATOMIC_H
29 #define LIBATOMIC_H 1
30
31 #include "auto-config.h"
32 #include <stdbool.h>
33 #include <stdint.h>
34 #include <stddef.h>
35 #include <limits.h>
36 #include <string.h>
37
38
39 /* Symbol concatenation macros. */
40 #define C2_(X,Y) X ## Y
41 #define C2(X,Y) C2_(X,Y)
42 #define C3_(X,Y,Z) X ## Y ## Z
43 #define C3(X,Y,Z) C3_(X,Y,Z)
44 #define C4_(W,X,Y,Z) W ## X ## Y ## Z
45 #define C4(W,X,Y,Z) C4_(W,X,Y,Z)
46
47 /* Stringification macros. */
48 #define S2(X) #X
49 #define S(X) S2(X)
50
51 /* All of the primitive types on which we operate. */
52 typedef unsigned U_1 __attribute__((mode(QI)));
53 #if HAVE_INT2
54 typedef unsigned U_2 __attribute__((mode(HI)));
55 #endif
56 #if HAVE_INT4
57 typedef unsigned U_4 __attribute__((mode(SI)));
58 #endif
59 #if HAVE_INT8
60 typedef unsigned U_8 __attribute__((mode(DI)));
61 #endif
62 #if HAVE_INT16
63 typedef unsigned U_16 __attribute__((mode(TI)));
64 #endif
65
66 /* The widest type that we support. */
67 #if HAVE_INT16
68 # define MAX_SIZE 16
69 #elif HAVE_INT8
70 # define MAX_SIZE 8
71 #elif HAVE_INT4
72 # define MAX_SIZE 4
73 #elif HAVE_INT2
74 # define MAX_SIZE 2
75 #else
76 # define MAX_SIZE 1
77 #endif
78 typedef C2(U_,MAX_SIZE) U_MAX;
79
80 /* Provide dummy fallback types so that stuff is syntactically correct
81 without having to overdo the ifdefs. The code using these should
82 always be protected with the HAVE_INT{n} macros. */
83 #if !HAVE_INT2
84 typedef U_MAX U_2;
85 #endif
86 #if !HAVE_INT4
87 typedef U_MAX U_4;
88 #endif
89 #if !HAVE_INT8
90 typedef U_MAX U_8;
91 #endif
92 #if !HAVE_INT16
93 typedef U_MAX U_16;
94 #endif
95
96 union max_size_u
97 {
98 U_1 b[MAX_SIZE];
99 U_2 i2;
100 U_4 i4;
101 U_8 i8;
102 U_16 i16;
103 };
104
105 /* The "word" size of the machine. */
106 typedef unsigned UWORD __attribute__((mode(word)));
107
108 /* Macros for handing sub-word sized quantities. */
109 #define MASK_1 ((UWORD)0xff)
110 #define MASK_2 ((UWORD)0xffff)
111 #define MASK_4 ((UWORD)0xffffffff)
112 #define MASK_8 ((UWORD)0xffffffffffffffff)
113 #define INVERT_MASK_1 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 1) * CHAR_BIT))
114 #define INVERT_MASK_2 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 2) * CHAR_BIT))
115 #define INVERT_MASK_4 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 4) * CHAR_BIT))
116 #define INVERT_MASK_8 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 8) * CHAR_BIT))
117
118 /* Most of the files in this library are compiled multiple times with
119 N defined to be a power of 2 between 1 and 16. The SIZE macro is
120 then used to append _N to the symbol being manipulated. */
121 #define SIZE(X) C3(X,_,N)
122 #define WSIZE(X) C3(X,_,WORDSIZE)
123 #define PTR(N,X) ((C2(U_,N) *)X)
124
125 /* And thus, the type on which this compilation will be operating. */
126 #define ITYPE SIZE(I)
127 #define UTYPE SIZE(U)
128
129 /* Utility macros for GCC attributes. */
130 #define UNUSED __attribute__((unused))
131 #ifdef HAVE_ATTRIBUTE_VISIBILITY
132 # define HIDDEN __attribute__((visibility("hidden")))
133 #else
134 # define HIDDEN
135 #endif
136
137 /* Occasionally we have to play games with internal and external symbol
138 names, in order to work around builtin functions of the same name.
139 This macro sets the external name of the function appropriately. */
140 #define ASMNAME(X) __asm__(S(C2(__USER_LABEL_PREFIX__,X)))
141
142 /* Locking for a "small" operation. In the bare-metal single processor
143 cases this could be implemented by disabling interrupts. Thus the extra
144 word passed between the two functions, saving the interrupt level.
145 It is assumed that the object being locked does not cross the locking
146 granularity.
147
148 Not actually declared here so that they can be defined static inline
149 in a target-specfic <host-config.h>.
150
151 UWORD protect_start (void *ptr);
152 void protect_end (void *ptr, UWORD);
153 */
154
155 /* Locking for a "large' operation. This should always be some sort of
156 test-and-set operation, as we assume that the interrupt latency would
157 be unreasonably large. */
158 void libat_lock_n (void *ptr, size_t n);
159 void libat_unlock_n (void *ptr, size_t n);
160
161 /* We'll need to declare all of the sized functions a few times... */
162 #define DECLARE_ALL_SIZED(N) DECLARE_ALL_SIZED_(N,C2(U_,N))
163 #define DECLARE_ALL_SIZED_(N,T) \
164 DECLARE_1(T, C2(load_,N), (T *mptr, int)); \
165 DECLARE_1(void, C2(store_,N), (T *mptr, T val, int)); \
166 DECLARE_1(T, C2(exchange_,N), (T *mptr, T, int)); \
167 DECLARE_1(bool, C2(compare_exchange_,N), (T *mptr, T *, T, int, int)); \
168 DECLARE_1(bool, C2(test_and_set_,N), (T *mptr, int)); \
169 DECLARE_1(T, C2(fetch_add_,N), (T *mptr, T, int)); \
170 DECLARE_1(T, C2(fetch_sub_,N), (T *mptr, T, int)); \
171 DECLARE_1(T, C2(fetch_and_,N), (T *mptr, T, int)); \
172 DECLARE_1(T, C2(fetch_xor_,N), (T *mptr, T, int)); \
173 DECLARE_1(T, C2(fetch_or_,N), (T *mptr, T, int)); \
174 DECLARE_1(T, C2(fetch_nand_,N), (T *mptr, T, int)); \
175 DECLARE_1(T, C2(add_fetch_,N), (T *mptr, T, int)); \
176 DECLARE_1(T, C2(sub_fetch_,N), (T *mptr, T, int)); \
177 DECLARE_1(T, C2(and_fetch_,N), (T *mptr, T, int)); \
178 DECLARE_1(T, C2(xor_fetch_,N), (T *mptr, T, int)); \
179 DECLARE_1(T, C2(or_fetch_,N), (T *mptr, T, int)); \
180 DECLARE_1(T, C2(nand_fetch_,N), (T *mptr, T, int))
181
182 /* All sized operations are implemented in hidden functions prefixed with
183 "libat_". These are either renamed or aliased to the expected prefix
184 of "__atomic". Some amount of renaming is required to avoid hiding or
185 conflicting with the builtins of the same name, but this additional
186 use of hidden symbols (where appropriate) avoids unnecessary PLT entries
187 on relevant targets. */
188
189 #if IFUNC_ALT
190 # define MAN(X) ASMNAME(C4(libat_,X,_i,IFUNC_ALT)) HIDDEN
191 #elif defined(HAVE_ATTRIBUTE_ALIAS)
192 # define MAN(X) HIDDEN
193 #else
194 # define MAN(X) ASMNAME(C2(__atomic_,X))
195 #endif
196
197 #if !defined(N) && HAVE_IFUNC
198 # define DECLARE_1(RET,NAME,ARGS) \
199 RET C2(libat_,NAME) ARGS MAN(NAME); \
200 RET C2(ifunc_,NAME) ARGS ASMNAME(C2(__atomic_,NAME))
201 #else
202 # define DECLARE_1(RET,NAME,ARGS) RET C2(libat_,NAME) ARGS MAN(NAME)
203 #endif
204
205 /* Prefix to use when calling internal, possibly ifunc'ed functions. */
206 #if HAVE_IFUNC
207 # define local_ ifunc_
208 #else
209 # define local_ libat_
210 #endif
211
212 DECLARE_ALL_SIZED(1);
213 DECLARE_ALL_SIZED(2);
214 DECLARE_ALL_SIZED(4);
215 DECLARE_ALL_SIZED(8);
216 DECLARE_ALL_SIZED(16);
217
218 #undef DECLARE_1
219 #undef DECLARE_ALL_SIZED
220 #undef DECLARE_ALL_SIZED_
221
222 /* And the generic sized versions. */
223 void libat_load (size_t, void *, void *, int) MAN(load);
224 void libat_store (size_t, void *, void *, int) MAN(store);
225 void libat_exchange (size_t, void *, void *, void *, int) MAN(exchange);
226 bool libat_compare_exchange (size_t, void *, void *, void *, int, int)
227 MAN(compare_exchange);
228 bool libat_is_lock_free (size_t, void *) MAN(is_lock_free);
229
230 #undef MAN
231
232 #include <host-config.h>
233
234 /* We don't have IFUNC_NCOND until after host-config.h. */
235 #if !HAVE_IFUNC
236 # define IFUNC_NCOND(N) 0
237 #endif
238
239 #if IFUNC_ALT
240 # define EXPORT_ALIAS(X) /* exported symbol in non-alternate file */
241 #elif defined(N) && IFUNC_NCOND(N)
242 # if IFUNC_NCOND(N) == 1
243 # define GEN_SELECTOR(X) \
244 extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
245 static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
246 { \
247 if (IFUNC_COND_1) \
248 return C3(libat_,X,_i1); \
249 return C2(libat_,X); \
250 }
251 # elif IFUNC_NCOND(N) == 2
252 # define GEN_SELECTOR(X) \
253 extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
254 extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \
255 static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
256 { \
257 if (IFUNC_COND_1) \
258 return C3(libat_,X,_i1); \
259 if (IFUNC_COND_2) \
260 return C3(libat_,X,_i2); \
261 return C2(libat_,X); \
262 }
263 # elif IFUNC_NCOND(N) == 3
264 # define GEN_SELECTOR(X) \
265 extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \
266 extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \
267 extern typeof(C2(libat_,X)) C3(libat_,X,_i3) HIDDEN; \
268 static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
269 { \
270 if (IFUNC_COND_1) \
271 return C3(libat_,X,_i1); \
272 if (IFUNC_COND_2) \
273 return C3(libat_,X,_i2); \
274 if (IFUNC_COND_3) \
275 return C3(libat_,X,_i3); \
276 return C2(libat_,X); \
277 }
278 # else
279 # error "Unsupported number of ifunc alternatives."
280 # endif
281 # define EXPORT_ALIAS(X) \
282 GEN_SELECTOR(X) \
283 typeof(C2(libat_,X)) C2(ifunc_,X) \
284 ASMNAME(C2(__atomic_,X)) \
285 __attribute__((ifunc(S(C2(select_,X)))))
286 #elif defined(HAVE_ATTRIBUTE_ALIAS)
287 # define EXPORT_ALIAS(X) \
288 extern typeof(C2(libat_,X)) C2(export_,X) \
289 ASMNAME(C2(__atomic_,X)) \
290 __attribute__((alias(S(C2(libat_,X)))))
291 #else
292 # define EXPORT_ALIAS(X) /* original symbol is exported */
293 #endif
294
295 #endif /* LIBATOMIC_H */