1 /* Internal macros for atomic operations for GNU C Library.
2 Copyright (C) 2002-2006, 2009 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, write to the Free
18 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
24 /* This header defines three types of macros:
26 - atomic arithmetic and logic operation on memory. They all
27 have the prefix "atomic_".
29 - conditionally atomic operations of the same kinds. These
30 always behave identical but can be faster when atomicity
31 is not really needed since only one thread has access to
32 the memory location. In that case the code is slower in
33 the multi-thread case. The interfaces have the prefix
36 - support functions like barriers. They also have the preifx
39 Architectures must provide a few lowlevel macros (the compare
40 and exchange definitions). All others are optional. They
41 should only be provided if the architecture has specific
42 support for the operation.
44 As <atomic.h> macros are usually heavily nested and often use local
45 variables to make sure side-effects are evaluated properly, use for
46 macro local variables a per-macro unique prefix. This file uses
47 __atgN_ prefix where N is different in each macro. */
51 #include <bits/atomic.h>
53 /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the
54 bit width of *MEM. The calling macro puts parens around MEM
55 and following args. */
56 #define __atomic_val_bysize(pre, post, mem, ...) \
58 __typeof (*mem) __atg1_result; \
59 if (sizeof (*mem) == 1) \
60 __atg1_result = pre##_8_##post (mem, __VA_ARGS__); \
61 else if (sizeof (*mem) == 2) \
62 __atg1_result = pre##_16_##post (mem, __VA_ARGS__); \
63 else if (sizeof (*mem) == 4) \
64 __atg1_result = pre##_32_##post (mem, __VA_ARGS__); \
65 else if (sizeof (*mem) == 8) \
66 __atg1_result = pre##_64_##post (mem, __VA_ARGS__); \
71 #define __atomic_bool_bysize(pre, post, mem, ...) \
74 if (sizeof (*mem) == 1) \
75 __atg2_result = pre##_8_##post (mem, __VA_ARGS__); \
76 else if (sizeof (*mem) == 2) \
77 __atg2_result = pre##_16_##post (mem, __VA_ARGS__); \
78 else if (sizeof (*mem) == 4) \
79 __atg2_result = pre##_32_##post (mem, __VA_ARGS__); \
80 else if (sizeof (*mem) == 8) \
81 __atg2_result = pre##_64_##post (mem, __VA_ARGS__); \
88 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
89 Return the old *MEM value. */
90 #if !defined atomic_compare_and_exchange_val_acq \
91 && defined __arch_compare_and_exchange_val_32_acq
92 # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
93 __atomic_val_bysize (__arch_compare_and_exchange_val,acq, \
98 #ifndef catomic_compare_and_exchange_val_acq
99 # ifdef __arch_c_compare_and_exchange_val_32_acq
100 # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
101 __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq, \
104 # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
105 atomic_compare_and_exchange_val_acq (mem, newval, oldval)
110 #ifndef catomic_compare_and_exchange_val_rel
111 # ifndef atomic_compare_and_exchange_val_rel
112 # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \
113 catomic_compare_and_exchange_val_acq (mem, newval, oldval)
115 # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \
116 atomic_compare_and_exchange_val_rel (mem, newval, oldval)
121 #ifndef atomic_compare_and_exchange_val_rel
122 # define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \
123 atomic_compare_and_exchange_val_acq (mem, newval, oldval)
127 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
128 Return zero if *MEM was changed or non-zero if no exchange happened. */
129 #ifndef atomic_compare_and_exchange_bool_acq
130 # ifdef __arch_compare_and_exchange_bool_32_acq
131 # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
132 __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq, \
135 # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
136 ({ /* Cannot use __oldval here, because macros later in this file might \
137 call this macro with __oldval argument. */ \
138 __typeof (oldval) __atg3_old = (oldval); \
139 atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old) \
146 #ifndef catomic_compare_and_exchange_bool_acq
147 # ifdef __arch_c_compare_and_exchange_bool_32_acq
148 # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
149 __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq, \
152 # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
153 ({ /* Cannot use __oldval here, because macros later in this file might \
154 call this macro with __oldval argument. */ \
155 __typeof (oldval) __atg4_old = (oldval); \
156 catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old) \
163 #ifndef catomic_compare_and_exchange_bool_rel
164 # ifndef atomic_compare_and_exchange_bool_rel
165 # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
166 catomic_compare_and_exchange_bool_acq (mem, newval, oldval)
168 # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
169 atomic_compare_and_exchange_bool_rel (mem, newval, oldval)
174 #ifndef atomic_compare_and_exchange_bool_rel
175 # define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
176 atomic_compare_and_exchange_bool_acq (mem, newval, oldval)
180 /* Store NEWVALUE in *MEM and return the old value. */
181 #ifndef atomic_exchange_acq
182 # define atomic_exchange_acq(mem, newvalue) \
183 ({ __typeof (*(mem)) __atg5_oldval; \
184 __typeof (mem) __atg5_memp = (mem); \
185 __typeof (*(mem)) __atg5_value = (newvalue); \
188 __atg5_oldval = *__atg5_memp; \
189 while (__builtin_expect \
190 (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \
191 __atg5_oldval), 0)); \
196 #ifndef atomic_exchange_rel
197 # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue)
201 /* Add VALUE to *MEM and return the old value of *MEM. */
202 #ifndef atomic_exchange_and_add
203 # define atomic_exchange_and_add(mem, value) \
204 ({ __typeof (*(mem)) __atg6_oldval; \
205 __typeof (mem) __atg6_memp = (mem); \
206 __typeof (*(mem)) __atg6_value = (value); \
209 __atg6_oldval = *__atg6_memp; \
210 while (__builtin_expect \
211 (atomic_compare_and_exchange_bool_acq (__atg6_memp, \
214 __atg6_oldval), 0)); \
220 #ifndef catomic_exchange_and_add
221 # define catomic_exchange_and_add(mem, value) \
222 ({ __typeof (*(mem)) __atg7_oldv; \
223 __typeof (mem) __atg7_memp = (mem); \
224 __typeof (*(mem)) __atg7_value = (value); \
227 __atg7_oldv = *__atg7_memp; \
228 while (__builtin_expect \
229 (catomic_compare_and_exchange_bool_acq (__atg7_memp, \
239 # define atomic_max(mem, value) \
241 __typeof (*(mem)) __atg8_oldval; \
242 __typeof (mem) __atg8_memp = (mem); \
243 __typeof (*(mem)) __atg8_value = (value); \
245 __atg8_oldval = *__atg8_memp; \
246 if (__atg8_oldval >= __atg8_value) \
248 } while (__builtin_expect \
249 (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\
250 __atg8_oldval), 0)); \
256 # define catomic_max(mem, value) \
258 __typeof (*(mem)) __atg9_oldv; \
259 __typeof (mem) __atg9_memp = (mem); \
260 __typeof (*(mem)) __atg9_value = (value); \
262 __atg9_oldv = *__atg9_memp; \
263 if (__atg9_oldv >= __atg9_value) \
265 } while (__builtin_expect \
266 (catomic_compare_and_exchange_bool_acq (__atg9_memp, \
274 # define atomic_min(mem, value) \
276 __typeof (*(mem)) __atg10_oldval; \
277 __typeof (mem) __atg10_memp = (mem); \
278 __typeof (*(mem)) __atg10_value = (value); \
280 __atg10_oldval = *__atg10_memp; \
281 if (__atg10_oldval <= __atg10_value) \
283 } while (__builtin_expect \
284 (atomic_compare_and_exchange_bool_acq (__atg10_memp, \
286 __atg10_oldval), 0)); \
292 # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value))
297 # define catomic_add(mem, value) \
298 (void) catomic_exchange_and_add ((mem), (value))
302 #ifndef atomic_increment
303 # define atomic_increment(mem) atomic_add ((mem), 1)
307 #ifndef catomic_increment
308 # define catomic_increment(mem) catomic_add ((mem), 1)
312 #ifndef atomic_increment_val
313 # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1)
317 #ifndef catomic_increment_val
318 # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1)
322 /* Add one to *MEM and return true iff it's now zero. */
323 #ifndef atomic_increment_and_test
324 # define atomic_increment_and_test(mem) \
325 (atomic_exchange_and_add ((mem), 1) + 1 == 0)
329 #ifndef atomic_decrement
330 # define atomic_decrement(mem) atomic_add ((mem), -1)
334 #ifndef catomic_decrement
335 # define catomic_decrement(mem) catomic_add ((mem), -1)
339 #ifndef atomic_decrement_val
340 # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1)
344 #ifndef catomic_decrement_val
345 # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1)
349 /* Subtract 1 from *MEM and return true iff it's now zero. */
350 #ifndef atomic_decrement_and_test
351 # define atomic_decrement_and_test(mem) \
352 (atomic_exchange_and_add ((mem), -1) == 1)
356 /* Decrement *MEM if it is > 0, and return the old value. */
357 #ifndef atomic_decrement_if_positive
358 # define atomic_decrement_if_positive(mem) \
359 ({ __typeof (*(mem)) __atg11_oldval; \
360 __typeof (mem) __atg11_memp = (mem); \
364 __atg11_oldval = *__atg11_memp; \
365 if (__builtin_expect (__atg11_oldval <= 0, 0)) \
368 while (__builtin_expect \
369 (atomic_compare_and_exchange_bool_acq (__atg11_memp, \
370 __atg11_oldval - 1, \
371 __atg11_oldval), 0)); \
376 #ifndef atomic_add_negative
377 # define atomic_add_negative(mem, value) \
378 ({ __typeof (value) __atg12_value = (value); \
379 atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; })
383 #ifndef atomic_add_zero
384 # define atomic_add_zero(mem, value) \
385 ({ __typeof (value) __atg13_value = (value); \
386 atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; })
390 #ifndef atomic_bit_set
391 # define atomic_bit_set(mem, bit) \
392 (void) atomic_bit_test_set(mem, bit)
396 #ifndef atomic_bit_test_set
397 # define atomic_bit_test_set(mem, bit) \
398 ({ __typeof (*(mem)) __atg14_old; \
399 __typeof (mem) __atg14_memp = (mem); \
400 __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit)); \
403 __atg14_old = (*__atg14_memp); \
404 while (__builtin_expect \
405 (atomic_compare_and_exchange_bool_acq (__atg14_memp, \
406 __atg14_old | __atg14_mask,\
409 __atg14_old & __atg14_mask; })
412 /* Atomically *mem &= mask. */
414 # define atomic_and(mem, mask) \
416 __typeof (*(mem)) __atg15_old; \
417 __typeof (mem) __atg15_memp = (mem); \
418 __typeof (*(mem)) __atg15_mask = (mask); \
421 __atg15_old = (*__atg15_memp); \
422 while (__builtin_expect \
423 (atomic_compare_and_exchange_bool_acq (__atg15_memp, \
424 __atg15_old & __atg15_mask, \
430 # define catomic_and(mem, mask) \
432 __typeof (*(mem)) __atg20_old; \
433 __typeof (mem) __atg20_memp = (mem); \
434 __typeof (*(mem)) __atg20_mask = (mask); \
437 __atg20_old = (*__atg20_memp); \
438 while (__builtin_expect \
439 (catomic_compare_and_exchange_bool_acq (__atg20_memp, \
440 __atg20_old & __atg20_mask,\
445 /* Atomically *mem &= mask and return the old value of *mem. */
446 #ifndef atomic_and_val
447 # define atomic_and_val(mem, mask) \
448 ({ __typeof (*(mem)) __atg16_old; \
449 __typeof (mem) __atg16_memp = (mem); \
450 __typeof (*(mem)) __atg16_mask = (mask); \
453 __atg16_old = (*__atg16_memp); \
454 while (__builtin_expect \
455 (atomic_compare_and_exchange_bool_acq (__atg16_memp, \
456 __atg16_old & __atg16_mask,\
462 /* Atomically *mem |= mask and return the old value of *mem. */
464 # define atomic_or(mem, mask) \
466 __typeof (*(mem)) __atg17_old; \
467 __typeof (mem) __atg17_memp = (mem); \
468 __typeof (*(mem)) __atg17_mask = (mask); \
471 __atg17_old = (*__atg17_memp); \
472 while (__builtin_expect \
473 (atomic_compare_and_exchange_bool_acq (__atg17_memp, \
474 __atg17_old | __atg17_mask, \
480 # define catomic_or(mem, mask) \
482 __typeof (*(mem)) __atg18_old; \
483 __typeof (mem) __atg18_memp = (mem); \
484 __typeof (*(mem)) __atg18_mask = (mask); \
487 __atg18_old = (*__atg18_memp); \
488 while (__builtin_expect \
489 (catomic_compare_and_exchange_bool_acq (__atg18_memp, \
490 __atg18_old | __atg18_mask,\
495 /* Atomically *mem |= mask and return the old value of *mem. */
496 #ifndef atomic_or_val
497 # define atomic_or_val(mem, mask) \
498 ({ __typeof (*(mem)) __atg19_old; \
499 __typeof (mem) __atg19_memp = (mem); \
500 __typeof (*(mem)) __atg19_mask = (mask); \
503 __atg19_old = (*__atg19_memp); \
504 while (__builtin_expect \
505 (atomic_compare_and_exchange_bool_acq (__atg19_memp, \
506 __atg19_old | __atg19_mask,\
512 #ifndef atomic_full_barrier
513 # define atomic_full_barrier() __asm__ ("" ::: "memory")
517 #ifndef atomic_read_barrier
518 # define atomic_read_barrier() atomic_full_barrier ()
522 #ifndef atomic_write_barrier
523 # define atomic_write_barrier() atomic_full_barrier ()
527 #ifndef atomic_forced_read
528 # define atomic_forced_read(x) \
529 ({ __typeof (x) __x; __asm__ ("" : "=r" (__x) : "0" (x)); __x; })
534 # define atomic_delay() do { /* nothing */ } while (0)
537 #endif /* atomic.h */