1 /* Copyright (C) 2002-2023 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3
4 The GNU C Library is free software; you can redistribute it and/or
5 modify it under the terms of the GNU Lesser General Public
6 License as published by the Free Software Foundation; either
7 version 2.1 of the License, or (at your option) any later version.
8
9 The GNU C Library is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 Lesser General Public License for more details.
13
14 You should have received a copy of the GNU Lesser General Public
15 License along with the GNU C Library; if not, see
16 <https://www.gnu.org/licenses/>. */
17
18 #include <assert.h>
19 #include <errno.h>
20 #include <stdlib.h>
21 #include "pthreadP.h"
22 #include <lowlevellock.h>
23 #include <stap-probe.h>
24 #include <futex-internal.h>
25 #include <shlib-compat.h>
26
27 static int
28 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
29 __attribute_noinline__;
30
31 /* lll_lock with single-thread optimization. */
32 static inline void
33 lll_mutex_unlock_optimized (pthread_mutex_t *mutex)
34 {
35 /* The single-threaded optimization is only valid for private
36 mutexes. For process-shared mutexes, the mutex could be in a
37 shared mapping, so synchronization with another process is needed
38 even without any threads. */
39 int private = PTHREAD_MUTEX_PSHARED (mutex);
40 if (private == LLL_PRIVATE && SINGLE_THREAD_P)
41 mutex->__data.__lock = 0;
42 else
43 lll_unlock (mutex->__data.__lock, private);
44 }
45
46 int
47 __pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
48 {
49 /* See concurrency notes regarding mutex type which is loaded from __kind
50 in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */
51 int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
52 if (__builtin_expect (type
53 & ~(PTHREAD_MUTEX_KIND_MASK_NP
54 |PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
55 return __pthread_mutex_unlock_full (mutex, decr);
56
57 if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
58 == PTHREAD_MUTEX_TIMED_NP)
59 {
60 /* Always reset the owner field. */
61 normal:
62 mutex->__data.__owner = 0;
63 if (decr)
64 /* One less user. */
65 --mutex->__data.__nusers;
66
67 /* Unlock. */
68 lll_mutex_unlock_optimized (mutex);
69
70 LIBC_PROBE (mutex_release, 1, mutex);
71
72 return 0;
73 }
74 else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
75 {
76 /* Don't reset the owner/users fields for elision. */
77 return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision,
78 PTHREAD_MUTEX_PSHARED (mutex));
79 }
80 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
81 == PTHREAD_MUTEX_RECURSIVE_NP, 1))
82 {
83 /* Recursive mutex. */
84 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
85 return EPERM;
86
87 if (--mutex->__data.__count != 0)
88 /* We still hold the mutex. */
89 return 0;
90 goto normal;
91 }
92 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
93 == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
94 goto normal;
95 else
96 {
97 /* Error checking mutex. */
98 assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
99 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
100 || ! lll_islocked (mutex->__data.__lock))
101 return EPERM;
102 goto normal;
103 }
104 }
105 libc_hidden_def (__pthread_mutex_unlock_usercnt)
106
107
108 static int
109 __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
110 {
111 int newowner = 0;
112 int private;
113
114 switch (PTHREAD_MUTEX_TYPE (mutex))
115 {
116 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
117 /* Recursive mutex. */
118 if ((mutex->__data.__lock & FUTEX_TID_MASK)
119 == THREAD_GETMEM (THREAD_SELF, tid)
120 && __builtin_expect (mutex->__data.__owner
121 == PTHREAD_MUTEX_INCONSISTENT, 0))
122 {
123 if (--mutex->__data.__count != 0)
124 /* We still hold the mutex. */
125 return ENOTRECOVERABLE;
126
127 goto notrecoverable;
128 }
129
130 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
131 return EPERM;
132
133 if (--mutex->__data.__count != 0)
134 /* We still hold the mutex. */
135 return 0;
136
137 goto robust;
138
139 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
140 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
141 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
142 if ((mutex->__data.__lock & FUTEX_TID_MASK)
143 != THREAD_GETMEM (THREAD_SELF, tid)
144 || ! lll_islocked (mutex->__data.__lock))
145 return EPERM;
146
147 /* If the previous owner died and the caller did not succeed in
148 making the state consistent, mark the mutex as unrecoverable
149 and make all waiters. */
150 if (__builtin_expect (mutex->__data.__owner
151 == PTHREAD_MUTEX_INCONSISTENT, 0))
152 notrecoverable:
153 newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
154
155 robust:
156 /* Remove mutex from the list. */
157 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
158 &mutex->__data.__list.__next);
159 /* We must set op_pending before we dequeue the mutex. Also see
160 comments at ENQUEUE_MUTEX. */
161 __asm ("" ::: "memory");
162 DEQUEUE_MUTEX (mutex);
163
164 mutex->__data.__owner = newowner;
165 if (decr)
166 /* One less user. */
167 --mutex->__data.__nusers;
168
169 /* Unlock by setting the lock to 0 (not acquired); if the lock had
170 FUTEX_WAITERS set previously, then wake any waiters.
171 The unlock operation must be the last access to the mutex to not
172 violate the mutex destruction requirements (see __lll_unlock). */
173 private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
174 if (__glibc_unlikely ((atomic_exchange_release (&mutex->__data.__lock, 0)
175 & FUTEX_WAITERS) != 0))
176 futex_wake ((unsigned int *) &mutex->__data.__lock, 1, private);
177
178 /* We must clear op_pending after we release the mutex.
179 FIXME However, this violates the mutex destruction requirements
180 because another thread could acquire the mutex, destroy it, and
181 reuse the memory for something else; then, if this thread crashes,
182 and the memory happens to have a value equal to the TID, the kernel
183 will believe it is still related to the mutex (which has been
184 destroyed already) and will modify some other random object. */
185 __asm ("" ::: "memory");
186 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
187 break;
188
189 /* The PI support requires the Linux futex system call. If that's not
190 available, pthread_mutex_init should never have allowed the type to
191 be set. So it will get the default case for an invalid type. */
192 #ifdef __NR_futex
193 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
194 /* Recursive mutex. */
195 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
196 return EPERM;
197
198 if (--mutex->__data.__count != 0)
199 /* We still hold the mutex. */
200 return 0;
201 goto continue_pi_non_robust;
202
203 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
204 /* Recursive mutex. */
205 if ((mutex->__data.__lock & FUTEX_TID_MASK)
206 == THREAD_GETMEM (THREAD_SELF, tid)
207 && __builtin_expect (mutex->__data.__owner
208 == PTHREAD_MUTEX_INCONSISTENT, 0))
209 {
210 if (--mutex->__data.__count != 0)
211 /* We still hold the mutex. */
212 return ENOTRECOVERABLE;
213
214 goto pi_notrecoverable;
215 }
216
217 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
218 return EPERM;
219
220 if (--mutex->__data.__count != 0)
221 /* We still hold the mutex. */
222 return 0;
223
224 goto continue_pi_robust;
225
226 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
227 case PTHREAD_MUTEX_PI_NORMAL_NP:
228 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
229 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
230 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
231 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
232 if ((mutex->__data.__lock & FUTEX_TID_MASK)
233 != THREAD_GETMEM (THREAD_SELF, tid)
234 || ! lll_islocked (mutex->__data.__lock))
235 return EPERM;
236
237 /* If the previous owner died and the caller did not succeed in
238 making the state consistent, mark the mutex as unrecoverable
239 and make all waiters. */
240 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
241 in sysdeps/nptl/bits/thread-shared-types.h. */
242 if ((atomic_load_relaxed (&(mutex->__data.__kind))
243 & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0
244 && __builtin_expect (mutex->__data.__owner
245 == PTHREAD_MUTEX_INCONSISTENT, 0))
246 pi_notrecoverable:
247 newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
248
249 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
250 in sysdeps/nptl/bits/thread-shared-types.h. */
251 if ((atomic_load_relaxed (&(mutex->__data.__kind))
252 & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0)
253 {
254 continue_pi_robust:
255 /* Remove mutex from the list.
256 Note: robust PI futexes are signaled by setting bit 0. */
257 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
258 (void *) (((uintptr_t) &mutex->__data.__list.__next)
259 | 1));
260 /* We must set op_pending before we dequeue the mutex. Also see
261 comments at ENQUEUE_MUTEX. */
262 __asm ("" ::: "memory");
263 DEQUEUE_MUTEX (mutex);
264 }
265
266 continue_pi_non_robust:
267 mutex->__data.__owner = newowner;
268 if (decr)
269 /* One less user. */
270 --mutex->__data.__nusers;
271
272 /* Unlock. Load all necessary mutex data before releasing the mutex
273 to not violate the mutex destruction requirements (see
274 lll_unlock). */
275 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
276 in sysdeps/nptl/bits/thread-shared-types.h. */
277 int robust = atomic_load_relaxed (&(mutex->__data.__kind))
278 & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
279 private = (robust
280 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
281 : PTHREAD_MUTEX_PSHARED (mutex));
282 /* Unlock the mutex using a CAS unless there are futex waiters or our
283 TID is not the value of __lock anymore, in which case we let the
284 kernel take care of the situation. Use release MO in the CAS to
285 synchronize with acquire MO in lock acquisitions. */
286 int l = atomic_load_relaxed (&mutex->__data.__lock);
287 do
288 {
289 if (((l & FUTEX_WAITERS) != 0)
290 || (l != THREAD_GETMEM (THREAD_SELF, tid)))
291 {
292 futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
293 private);
294 break;
295 }
296 }
297 while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
298 &l, 0));
299
300 /* This happens after the kernel releases the mutex but violates the
301 mutex destruction requirements; see comments in the code handling
302 PTHREAD_MUTEX_ROBUST_NORMAL_NP. */
303 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
304 break;
305 #endif /* __NR_futex. */
306
307 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
308 /* Recursive mutex. */
309 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
310 return EPERM;
311
312 if (--mutex->__data.__count != 0)
313 /* We still hold the mutex. */
314 return 0;
315 goto pp;
316
317 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
318 /* Error checking mutex. */
319 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
320 || (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0)
321 return EPERM;
322 /* FALLTHROUGH */
323
324 case PTHREAD_MUTEX_PP_NORMAL_NP:
325 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
326 /* Always reset the owner field. */
327 pp:
328 mutex->__data.__owner = 0;
329
330 if (decr)
331 /* One less user. */
332 --mutex->__data.__nusers;
333
334 /* Unlock. Use release MO in the CAS to synchronize with acquire MO in
335 lock acquisitions. */
336 int newval;
337 int oldval = atomic_load_relaxed (&mutex->__data.__lock);
338 do
339 {
340 newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK;
341 }
342 while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
343 &oldval, newval));
344
345 if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
346 futex_wake ((unsigned int *)&mutex->__data.__lock, 1,
347 PTHREAD_MUTEX_PSHARED (mutex));
348
349 int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
350
351 LIBC_PROBE (mutex_release, 1, mutex);
352
353 return __pthread_tpp_change_priority (oldprio, -1);
354
355 default:
356 /* Correct code cannot set any other type. */
357 return EINVAL;
358 }
359
360 LIBC_PROBE (mutex_release, 1, mutex);
361 return 0;
362 }
363
364
365 int
366 ___pthread_mutex_unlock (pthread_mutex_t *mutex)
367 {
368 return __pthread_mutex_unlock_usercnt (mutex, 1);
369 }
370 libc_hidden_ver (___pthread_mutex_unlock, __pthread_mutex_unlock)
371 #ifndef SHARED
372 strong_alias (___pthread_mutex_unlock, __pthread_mutex_unlock)
373 #endif
374 versioned_symbol (libpthread, ___pthread_mutex_unlock, pthread_mutex_unlock,
375 GLIBC_2_0);
376
377 #if OTHER_SHLIB_COMPAT (libpthread, GLIBC_2_0, GLIBC_2_34)
378 compat_symbol (libpthread, ___pthread_mutex_unlock, __pthread_mutex_unlock,
379 GLIBC_2_0);
380 #endif