1 /* Linux-specific atomic operations for PA Linux.
2 Copyright (C) 2008-2023 Free Software Foundation, Inc.
3 Based on code contributed by CodeSourcery for ARM EABI Linux.
4 Modifications for PA Linux by Helge Deller <deller@gmx.de>
5
6 This file is part of GCC.
7
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
12
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
17
18 Under Section 7 of GPL version 3, you are granted additional
19 permissions described in the GCC Runtime Library Exception, version
20 3.1, as published by the Free Software Foundation.
21
22 You should have received a copy of the GNU General Public License and
23 a copy of the GCC Runtime Library Exception along with this program;
24 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
25 <http://www.gnu.org/licenses/>. */
26
27 #define EFAULT 14
28 #define EBUSY 16
29 #define ENOSYS 251
30
31 #define _ASM_EFAULT "-14"
32
33 typedef unsigned char u8;
34 typedef short unsigned int u16;
35 typedef unsigned int u32;
36 #ifdef __LP64__
37 typedef long unsigned int u64;
38 #else
39 typedef long long unsigned int u64;
40 #endif
41
42 /* PA-RISC 2.0 supports out-of-order execution for loads and stores.
43 Thus, we need to synchonize memory accesses. For more info, see:
44 "Advanced Performance Features of the 64-bit PA-8000" by Doug Hunt.
45
46 We implement byte, short and int versions of each atomic operation
47 using the kernel helper defined below. There is no support for
48 64-bit operations yet. */
49
50 /* Determine kernel LWS function call (0=32-bit, 1=64-bit userspace). */
51 #define LWS_CAS (sizeof(long) == 4 ? 0 : 1)
52
53 /* Kernel helper for compare-and-exchange a 32-bit value. */
54 static inline long
55 __kernel_cmpxchg (volatile void *mem, int oldval, int newval)
56 {
57 register unsigned long lws_mem asm("r26") = (unsigned long) (mem);
58 register int lws_old asm("r25") = oldval;
59 register int lws_new asm("r24") = newval;
60 register long lws_ret asm("r28");
61 register long lws_errno asm("r21");
62 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t"
63 "ldi %2, %%r20 \n\t"
64 "cmpiclr,<> " _ASM_EFAULT ", %%r21, %%r0\n\t"
65 "iitlbp %%r0,(%%sr0, %%r0) \n\t"
66 : "=r" (lws_ret), "=r" (lws_errno)
67 : "i" (LWS_CAS), "r" (lws_mem), "r" (lws_old), "r" (lws_new)
68 : "r1", "r20", "r22", "r23", "r29", "r31", "memory"
69 );
70
71 /* If the kernel LWS call succeeded (lws_errno == 0), lws_ret contains
72 the old value from memory. If this value is equal to OLDVAL, the
73 new value was written to memory. If not, return -EBUSY. */
74 if (!lws_errno && lws_ret != oldval)
75 return -EBUSY;
76
77 return lws_errno;
78 }
79
80 static inline long
81 __kernel_cmpxchg2 (volatile void *mem, const void *oldval, const void *newval,
82 int val_size)
83 {
84 register unsigned long lws_mem asm("r26") = (unsigned long) (mem);
85 register unsigned long lws_old asm("r25") = (unsigned long) oldval;
86 register unsigned long lws_new asm("r24") = (unsigned long) newval;
87 register int lws_size asm("r23") = val_size;
88 register long lws_ret asm("r28");
89 register long lws_errno asm("r21");
90 asm volatile ( "ble 0xb0(%%sr2, %%r0) \n\t"
91 "ldi %6, %%r20 \n\t"
92 "cmpiclr,<> " _ASM_EFAULT ", %%r21, %%r0\n\t"
93 "iitlbp %%r0,(%%sr0, %%r0) \n\t"
94 : "=r" (lws_ret), "=r" (lws_errno), "+r" (lws_mem),
95 "+r" (lws_old), "+r" (lws_new), "+r" (lws_size)
96 : "i" (2)
97 : "r1", "r20", "r22", "r29", "r31", "fr4", "memory"
98 );
99
100 /* If the kernel LWS call is successful, lws_ret contains 0. */
101 if (__builtin_expect (lws_ret == 0, 1))
102 return 0;
103
104 /* If the kernel LWS call fails with no error, return -EBUSY */
105 if (__builtin_expect (!lws_errno, 0))
106 return -EBUSY;
107
108 return lws_errno;
109 }
110 #define HIDDEN __attribute__ ((visibility ("hidden")))
111
112 /* Big endian masks */
113 #define INVERT_MASK_1 24
114 #define INVERT_MASK_2 16
115
116 #define MASK_1 0xffu
117 #define MASK_2 0xffffu
118
119 /* Load value with an atomic processor load if possible. */
120 #define ATOMIC_LOAD(TYPE, WIDTH) \
121 static inline TYPE \
122 atomic_load_##WIDTH (volatile void *ptr) \
123 { \
124 return *(volatile TYPE *)ptr; \
125 }
126
127 #if defined(__LP64__) || defined(__SOFTFP__)
128 ATOMIC_LOAD (u64, 8)
129 #else
130 static inline u64
131 atomic_load_8 (volatile void *ptr)
132 {
133 u64 result;
134 double tmp;
135
136 asm volatile ("{fldds|fldd} 0(%2),%1\n\t"
137 "{fstds|fstd} %1,-16(%%sp)\n\t"
138 "{ldws|ldw} -16(%%sp),%0\n\t"
139 "{ldws|ldw} -12(%%sp),%R0"
140 : "=r" (result), "=f" (tmp) : "r" (ptr): "memory");
141 return result;
142 }
143 #endif
144
145 ATOMIC_LOAD (u32, 4)
146 ATOMIC_LOAD (u16, 2)
147 ATOMIC_LOAD (u8, 1)
148
149 #define FETCH_AND_OP_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \
150 TYPE HIDDEN \
151 __sync_fetch_and_##OP##_##WIDTH (volatile void *ptr, TYPE val) \
152 { \
153 TYPE tmp, newval; \
154 long failure; \
155 \
156 do { \
157 tmp = atomic_load_##WIDTH ((volatile TYPE *)ptr); \
158 newval = PFX_OP (tmp INF_OP val); \
159 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \
160 } while (failure != 0); \
161 \
162 return tmp; \
163 }
164
165 FETCH_AND_OP_2 (add, , +, u64, 8, 3)
166 FETCH_AND_OP_2 (sub, , -, u64, 8, 3)
167 FETCH_AND_OP_2 (or, , |, u64, 8, 3)
168 FETCH_AND_OP_2 (and, , &, u64, 8, 3)
169 FETCH_AND_OP_2 (xor, , ^, u64, 8, 3)
170 FETCH_AND_OP_2 (nand, ~, &, u64, 8, 3)
171
172 FETCH_AND_OP_2 (add, , +, u16, 2, 1)
173 FETCH_AND_OP_2 (sub, , -, u16, 2, 1)
174 FETCH_AND_OP_2 (or, , |, u16, 2, 1)
175 FETCH_AND_OP_2 (and, , &, u16, 2, 1)
176 FETCH_AND_OP_2 (xor, , ^, u16, 2, 1)
177 FETCH_AND_OP_2 (nand, ~, &, u16, 2, 1)
178
179 FETCH_AND_OP_2 (add, , +, u8, 1, 0)
180 FETCH_AND_OP_2 (sub, , -, u8, 1, 0)
181 FETCH_AND_OP_2 (or, , |, u8, 1, 0)
182 FETCH_AND_OP_2 (and, , &, u8, 1, 0)
183 FETCH_AND_OP_2 (xor, , ^, u8, 1, 0)
184 FETCH_AND_OP_2 (nand, ~, &, u8, 1, 0)
185
186 #define OP_AND_FETCH_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX) \
187 TYPE HIDDEN \
188 __sync_##OP##_and_fetch_##WIDTH (volatile void *ptr, TYPE val) \
189 { \
190 TYPE tmp, newval; \
191 long failure; \
192 \
193 do { \
194 tmp = atomic_load_##WIDTH ((volatile TYPE *)ptr); \
195 newval = PFX_OP (tmp INF_OP val); \
196 failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX); \
197 } while (failure != 0); \
198 \
199 return PFX_OP (tmp INF_OP val); \
200 }
201
202 OP_AND_FETCH_2 (add, , +, u64, 8, 3)
203 OP_AND_FETCH_2 (sub, , -, u64, 8, 3)
204 OP_AND_FETCH_2 (or, , |, u64, 8, 3)
205 OP_AND_FETCH_2 (and, , &, u64, 8, 3)
206 OP_AND_FETCH_2 (xor, , ^, u64, 8, 3)
207 OP_AND_FETCH_2 (nand, ~, &, u64, 8, 3)
208
209 OP_AND_FETCH_2 (add, , +, u16, 2, 1)
210 OP_AND_FETCH_2 (sub, , -, u16, 2, 1)
211 OP_AND_FETCH_2 (or, , |, u16, 2, 1)
212 OP_AND_FETCH_2 (and, , &, u16, 2, 1)
213 OP_AND_FETCH_2 (xor, , ^, u16, 2, 1)
214 OP_AND_FETCH_2 (nand, ~, &, u16, 2, 1)
215
216 OP_AND_FETCH_2 (add, , +, u8, 1, 0)
217 OP_AND_FETCH_2 (sub, , -, u8, 1, 0)
218 OP_AND_FETCH_2 (or, , |, u8, 1, 0)
219 OP_AND_FETCH_2 (and, , &, u8, 1, 0)
220 OP_AND_FETCH_2 (xor, , ^, u8, 1, 0)
221 OP_AND_FETCH_2 (nand, ~, &, u8, 1, 0)
222
223 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \
224 unsigned int HIDDEN \
225 __sync_fetch_and_##OP##_4 (volatile void *ptr, unsigned int val) \
226 { \
227 unsigned int tmp; \
228 long failure; \
229 \
230 do { \
231 tmp = atomic_load_4 ((volatile unsigned int *)ptr); \
232 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \
233 } while (failure != 0); \
234 \
235 return tmp; \
236 }
237
238 FETCH_AND_OP_WORD (add, , +)
239 FETCH_AND_OP_WORD (sub, , -)
240 FETCH_AND_OP_WORD (or, , |)
241 FETCH_AND_OP_WORD (and, , &)
242 FETCH_AND_OP_WORD (xor, , ^)
243 FETCH_AND_OP_WORD (nand, ~, &)
244
245 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \
246 unsigned int HIDDEN \
247 __sync_##OP##_and_fetch_4 (volatile void *ptr, unsigned int val) \
248 { \
249 unsigned int tmp; \
250 long failure; \
251 \
252 do { \
253 tmp = atomic_load_4 ((volatile unsigned int *)ptr); \
254 failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val)); \
255 } while (failure != 0); \
256 \
257 return PFX_OP (tmp INF_OP val); \
258 }
259
260 OP_AND_FETCH_WORD (add, , +)
261 OP_AND_FETCH_WORD (sub, , -)
262 OP_AND_FETCH_WORD (or, , |)
263 OP_AND_FETCH_WORD (and, , &)
264 OP_AND_FETCH_WORD (xor, , ^)
265 OP_AND_FETCH_WORD (nand, ~, &)
266
267 typedef unsigned char bool;
268
269 #define COMPARE_AND_SWAP_2(TYPE, WIDTH, INDEX) \
270 TYPE HIDDEN \
271 __sync_val_compare_and_swap_##WIDTH (volatile void *ptr, TYPE oldval, \
272 TYPE newval) \
273 { \
274 TYPE actual_oldval; \
275 long fail; \
276 \
277 while (1) \
278 { \
279 actual_oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr); \
280 \
281 if (__builtin_expect (oldval != actual_oldval, 0)) \
282 return actual_oldval; \
283 \
284 fail = __kernel_cmpxchg2 (ptr, &actual_oldval, &newval, INDEX); \
285 \
286 if (__builtin_expect (!fail, 1)) \
287 return actual_oldval; \
288 } \
289 } \
290 \
291 _Bool HIDDEN \
292 __sync_bool_compare_and_swap_##WIDTH (volatile void *ptr, \
293 TYPE oldval, TYPE newval) \
294 { \
295 long failure = __kernel_cmpxchg2 (ptr, &oldval, &newval, INDEX); \
296 return (failure == 0); \
297 }
298
299 COMPARE_AND_SWAP_2 (u64, 8, 3)
300 COMPARE_AND_SWAP_2 (u16, 2, 1)
301 COMPARE_AND_SWAP_2 (u8, 1, 0)
302
303 unsigned int HIDDEN
304 __sync_val_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
305 unsigned int newval)
306 {
307 long fail;
308 unsigned int actual_oldval;
309
310 while (1)
311 {
312 actual_oldval = atomic_load_4 ((volatile unsigned int *)ptr);
313
314 if (__builtin_expect (oldval != actual_oldval, 0))
315 return actual_oldval;
316
317 fail = __kernel_cmpxchg (ptr, actual_oldval, newval);
318
319 if (__builtin_expect (!fail, 1))
320 return actual_oldval;
321 }
322 }
323
324 _Bool HIDDEN
325 __sync_bool_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
326 unsigned int newval)
327 {
328 long failure = __kernel_cmpxchg (ptr, oldval, newval);
329 return (failure == 0);
330 }
331
332 #define SYNC_LOCK_TEST_AND_SET_2(TYPE, WIDTH, INDEX) \
333 TYPE HIDDEN \
334 __sync_lock_test_and_set_##WIDTH (volatile void *ptr, TYPE val) \
335 { \
336 TYPE oldval; \
337 long failure; \
338 \
339 do { \
340 oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr); \
341 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \
342 } while (failure != 0); \
343 \
344 return oldval; \
345 }
346
347 SYNC_LOCK_TEST_AND_SET_2 (u64, 8, 3)
348 SYNC_LOCK_TEST_AND_SET_2 (u16, 2, 1)
349 SYNC_LOCK_TEST_AND_SET_2 (u8, 1, 0)
350
351 u32 HIDDEN
352 __sync_lock_test_and_set_4 (volatile void *ptr, unsigned int val)
353 {
354 long failure;
355 unsigned int oldval;
356
357 do {
358 oldval = atomic_load_4 ((volatile unsigned int *)ptr);
359 failure = __kernel_cmpxchg (ptr, oldval, val);
360 } while (failure != 0);
361
362 return oldval;
363 }
364
365 #define SYNC_LOCK_RELEASE_1(TYPE, WIDTH, INDEX) \
366 void HIDDEN \
367 __sync_lock_release_##WIDTH (volatile void *ptr) \
368 { \
369 TYPE oldval, val = 0; \
370 long failure; \
371 \
372 do { \
373 oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr); \
374 failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX); \
375 } while (failure != 0); \
376 }
377
378 SYNC_LOCK_RELEASE_1 (u64, 8, 3)
379 SYNC_LOCK_RELEASE_1 (u16, 2, 1)
380 SYNC_LOCK_RELEASE_1 (u8, 1, 0)
381
382 void HIDDEN
383 __sync_lock_release_4 (volatile void *ptr)
384 {
385 long failure;
386 unsigned int oldval;
387
388 do {
389 oldval = atomic_load_4 ((volatile unsigned int *)ptr);
390 failure = __kernel_cmpxchg (ptr, oldval, 0);
391 } while (failure != 0);
392 }
393
394 #ifndef __LP64__
395 #define SYNC_LOCK_LOAD_2(TYPE, WIDTH, INDEX) \
396 TYPE __sync_lock_load_##WIDTH (volatile void *) HIDDEN; \
397 TYPE \
398 __sync_lock_load_##WIDTH (volatile void *ptr) \
399 { \
400 TYPE oldval; \
401 long failure; \
402 \
403 do { \
404 oldval = atomic_load_##WIDTH ((volatile TYPE *)ptr); \
405 failure = __kernel_cmpxchg2 (ptr, &oldval, &oldval, INDEX); \
406 } while (failure != 0); \
407 \
408 return oldval; \
409 }
410
411 SYNC_LOCK_LOAD_2 (u64, 8, 3)
412 #endif