include/atomic.h

   1 /* Internal macros for atomic operations for GNU C Library.
   2    Copyright (C) 2002-2006, 2009 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   5
   6    The GNU C Library is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU Lesser General Public
   8    License as published by the Free Software Foundation; either
   9    version 2.1 of the License, or (at your option) any later version.
  10
  11    The GNU C Library is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    Lesser General Public License for more details.
  15
  16    You should have received a copy of the GNU Lesser General Public
  17    License along with the GNU C Library; if not, see
  18    <http://www.gnu.org/licenses/>.  */
  19
  20 #ifndef _ATOMIC_H
  21 #define _ATOMIC_H       1
  22
  23 /* This header defines three types of macros:
  24
  25    - atomic arithmetic and logic operation on memory.  They all
  26      have the prefix "atomic_".
  27
  28    - conditionally atomic operations of the same kinds.  These
  29      always behave identical but can be faster when atomicity
  30      is not really needed since only one thread has access to
  31      the memory location.  In that case the code is slower in
  32      the multi-thread case.  The interfaces have the prefix
  33      "catomic_".
  34
  35    - support functions like barriers.  They also have the preifx
  36      "atomic_".
  37
  38    Architectures must provide a few lowlevel macros (the compare
  39    and exchange definitions).  All others are optional.  They
  40    should only be provided if the architecture has specific
  41    support for the operation.
  42
  43    As <atomic.h> macros are usually heavily nested and often use local
  44    variables to make sure side-effects are evaluated properly, use for
  45    macro local variables a per-macro unique prefix.  This file uses
  46    __atgN_ prefix where N is different in each macro.  */
  47
  48 #include <stdlib.h>
  49
  50 #include <bits/atomic.h>
  51
  52 /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the
  53    bit width of *MEM.  The calling macro puts parens around MEM
  54    and following args.  */
  55 #define __atomic_val_bysize(pre, post, mem, ...)                              \
  56   ({                                                                          \
  57     __typeof (*mem) __atg1_result;                                            \
  58     if (sizeof (*mem) == 1)                                                   \
  59       __atg1_result = pre##_8_##post (mem, __VA_ARGS__);                      \
  60     else if (sizeof (*mem) == 2)                                              \
  61       __atg1_result = pre##_16_##post (mem, __VA_ARGS__);                     \
  62     else if (sizeof (*mem) == 4)                                              \
  63       __atg1_result = pre##_32_##post (mem, __VA_ARGS__);                     \
  64     else if (sizeof (*mem) == 8)                                              \
  65       __atg1_result = pre##_64_##post (mem, __VA_ARGS__);                     \
  66     else                                                                      \
  67       abort ();                                                               \
  68     __atg1_result;                                                            \
  69   })
  70 #define __atomic_bool_bysize(pre, post, mem, ...)                             \
  71   ({                                                                          \
  72     int __atg2_result;                                                        \
  73     if (sizeof (*mem) == 1)                                                   \
  74       __atg2_result = pre##_8_##post (mem, __VA_ARGS__);                      \
  75     else if (sizeof (*mem) == 2)                                              \
  76       __atg2_result = pre##_16_##post (mem, __VA_ARGS__);                     \
  77     else if (sizeof (*mem) == 4)                                              \
  78       __atg2_result = pre##_32_##post (mem, __VA_ARGS__);                     \
  79     else if (sizeof (*mem) == 8)                                              \
  80       __atg2_result = pre##_64_##post (mem, __VA_ARGS__);                     \
  81     else                                                                      \
  82       abort ();                                                               \
  83     __atg2_result;                                                            \
  84   })
  85
  86
  87 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
  88    Return the old *MEM value.  */
  89 #if !defined atomic_compare_and_exchange_val_acq \
  90     && defined __arch_compare_and_exchange_val_32_acq
  91 # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \
  92   __atomic_val_bysize (__arch_compare_and_exchange_val,acq,                   \
  93                        mem, newval, oldval)
  94 #endif
  95
  96
  97 #ifndef catomic_compare_and_exchange_val_acq
  98 # ifdef __arch_c_compare_and_exchange_val_32_acq
  99 #  define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
 100   __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq,                 \
 101                        mem, newval, oldval)
 102 # else
 103 #  define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \
 104   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 105 # endif
 106 #endif
 107
 108
 109 #ifndef catomic_compare_and_exchange_val_rel
 110 # ifndef atomic_compare_and_exchange_val_rel
 111 #  define catomic_compare_and_exchange_val_rel(mem, newval, oldval)           \
 112   catomic_compare_and_exchange_val_acq (mem, newval, oldval)
 113 # else
 114 #  define catomic_compare_and_exchange_val_rel(mem, newval, oldval)           \
 115   atomic_compare_and_exchange_val_rel (mem, newval, oldval)
 116 # endif
 117 #endif
 118
 119
 120 #ifndef atomic_compare_and_exchange_val_rel
 121 # define atomic_compare_and_exchange_val_rel(mem, newval, oldval)             \
 122   atomic_compare_and_exchange_val_acq (mem, newval, oldval)
 123 #endif
 124
 125
 126 /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL.
 127    Return zero if *MEM was changed or non-zero if no exchange happened.  */
 128 #ifndef atomic_compare_and_exchange_bool_acq
 129 # ifdef __arch_compare_and_exchange_bool_32_acq
 130 #  define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 131   __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq,                 \
 132                         mem, newval, oldval)
 133 # else
 134 #  define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 135   ({ /* Cannot use __oldval here, because macros later in this file might     \
 136         call this macro with __oldval argument.  */                           \
 137      __typeof (oldval) __atg3_old = (oldval);                                 \
 138      atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old)            \
 139        != __atg3_old;                                                         \
 140   })
 141 # endif
 142 #endif
 143
 144
 145 #ifndef catomic_compare_and_exchange_bool_acq
 146 # ifdef __arch_c_compare_and_exchange_bool_32_acq
 147 #  define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 148   __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq,               \
 149                         mem, newval, oldval)
 150 # else
 151 #  define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \
 152   ({ /* Cannot use __oldval here, because macros later in this file might     \
 153         call this macro with __oldval argument.  */                           \
 154      __typeof (oldval) __atg4_old = (oldval);                                 \
 155      catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old)           \
 156        != __atg4_old;                                                         \
 157   })
 158 # endif
 159 #endif
 160
 161
 162 #ifndef catomic_compare_and_exchange_bool_rel
 163 # ifndef atomic_compare_and_exchange_bool_rel
 164 #  define catomic_compare_and_exchange_bool_rel(mem, newval, oldval)          \
 165   catomic_compare_and_exchange_bool_acq (mem, newval, oldval)
 166 # else
 167 #  define catomic_compare_and_exchange_bool_rel(mem, newval, oldval)          \
 168   atomic_compare_and_exchange_bool_rel (mem, newval, oldval)
 169 # endif
 170 #endif
 171
 172
 173 #ifndef atomic_compare_and_exchange_bool_rel
 174 # define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \
 175   atomic_compare_and_exchange_bool_acq (mem, newval, oldval)
 176 #endif
 177
 178
 179 /* Store NEWVALUE in *MEM and return the old value.  */
 180 #ifndef atomic_exchange_acq
 181 # define atomic_exchange_acq(mem, newvalue) \
 182   ({ __typeof (*(mem)) __atg5_oldval;                                         \
 183      __typeof (mem) __atg5_memp = (mem);                                      \
 184      __typeof (*(mem)) __atg5_value = (newvalue);                             \
 185                                                                               \
 186      do                                                                       \
 187        __atg5_oldval = *__atg5_memp;                                          \
 188      while (__builtin_expect                                                  \
 189             (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \
 190                                                    __atg5_oldval), 0));       \
 191                                                                               \
 192      __atg5_oldval; })
 193 #endif
 194
 195 #ifndef atomic_exchange_rel
 196 # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue)
 197 #endif
 198
 199
 200 /* Add VALUE to *MEM and return the old value of *MEM.  */
 201 #ifndef atomic_exchange_and_add
 202 # define atomic_exchange_and_add(mem, value) \
 203   ({ __typeof (*(mem)) __atg6_oldval;                                         \
 204      __typeof (mem) __atg6_memp = (mem);                                      \
 205      __typeof (*(mem)) __atg6_value = (value);                                \
 206                                                                               \
 207      do                                                                       \
 208        __atg6_oldval = *__atg6_memp;                                          \
 209      while (__builtin_expect                                                  \
 210             (atomic_compare_and_exchange_bool_acq (__atg6_memp,               \
 211                                                    __atg6_oldval              \
 212                                                    + __atg6_value,            \
 213                                                    __atg6_oldval), 0));       \
 214                                                                               \
 215      __atg6_oldval; })
 216 #endif
 217
 218
 219 #ifndef catomic_exchange_and_add
 220 # define catomic_exchange_and_add(mem, value) \
 221   ({ __typeof (*(mem)) __atg7_oldv;                                           \
 222      __typeof (mem) __atg7_memp = (mem);                                      \
 223      __typeof (*(mem)) __atg7_value = (value);                                \
 224                                                                               \
 225      do                                                                       \
 226        __atg7_oldv = *__atg7_memp;                                            \
 227      while (__builtin_expect                                                  \
 228             (catomic_compare_and_exchange_bool_acq (__atg7_memp,              \
 229                                                     __atg7_oldv               \
 230                                                     + __atg7_value,           \
 231                                                     __atg7_oldv), 0));        \
 232                                                                               \
 233      __atg7_oldv; })
 234 #endif
 235
 236
 237 #ifndef atomic_max
 238 # define atomic_max(mem, value) \
 239   do {                                                                        \
 240     __typeof (*(mem)) __atg8_oldval;                                          \
 241     __typeof (mem) __atg8_memp = (mem);                                       \
 242     __typeof (*(mem)) __atg8_value = (value);                                 \
 243     do {                                                                      \
 244       __atg8_oldval = *__atg8_memp;                                           \
 245       if (__atg8_oldval >= __atg8_value)                                      \
 246         break;                                                                \
 247     } while (__builtin_expect                                                 \
 248              (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\
 249                                                     __atg8_oldval), 0));      \
 250   } while (0)
 251 #endif
 252
 253
 254 #ifndef catomic_max
 255 # define catomic_max(mem, value) \
 256   do {                                                                        \
 257     __typeof (*(mem)) __atg9_oldv;                                            \
 258     __typeof (mem) __atg9_memp = (mem);                                       \
 259     __typeof (*(mem)) __atg9_value = (value);                                 \
 260     do {                                                                      \
 261       __atg9_oldv = *__atg9_memp;                                             \
 262       if (__atg9_oldv >= __atg9_value)                                        \
 263         break;                                                                \
 264     } while (__builtin_expect                                                 \
 265              (catomic_compare_and_exchange_bool_acq (__atg9_memp,             \
 266                                                      __atg9_value,            \
 267                                                      __atg9_oldv), 0));       \
 268   } while (0)
 269 #endif
 270
 271
 272 #ifndef atomic_min
 273 # define atomic_min(mem, value) \
 274   do {                                                                        \
 275     __typeof (*(mem)) __atg10_oldval;                                         \
 276     __typeof (mem) __atg10_memp = (mem);                                      \
 277     __typeof (*(mem)) __atg10_value = (value);                                \
 278     do {                                                                      \
 279       __atg10_oldval = *__atg10_memp;                                         \
 280       if (__atg10_oldval <= __atg10_value)                                    \
 281         break;                                                                \
 282     } while (__builtin_expect                                                 \
 283              (atomic_compare_and_exchange_bool_acq (__atg10_memp,             \
 284                                                     __atg10_value,            \
 285                                                     __atg10_oldval), 0));     \
 286   } while (0)
 287 #endif
 288
 289
 290 #ifndef atomic_add
 291 # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value))
 292 #endif
 293
 294
 295 #ifndef catomic_add
 296 # define catomic_add(mem, value) \
 297   (void) catomic_exchange_and_add ((mem), (value))
 298 #endif
 299
 300
 301 #ifndef atomic_increment
 302 # define atomic_increment(mem) atomic_add ((mem), 1)
 303 #endif
 304
 305
 306 #ifndef catomic_increment
 307 # define catomic_increment(mem) catomic_add ((mem), 1)
 308 #endif
 309
 310
 311 #ifndef atomic_increment_val
 312 # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1)
 313 #endif
 314
 315
 316 #ifndef catomic_increment_val
 317 # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1)
 318 #endif
 319
 320
 321 /* Add one to *MEM and return true iff it's now zero.  */
 322 #ifndef atomic_increment_and_test
 323 # define atomic_increment_and_test(mem) \
 324   (atomic_exchange_and_add ((mem), 1) + 1 == 0)
 325 #endif
 326
 327
 328 #ifndef atomic_decrement
 329 # define atomic_decrement(mem) atomic_add ((mem), -1)
 330 #endif
 331
 332
 333 #ifndef catomic_decrement
 334 # define catomic_decrement(mem) catomic_add ((mem), -1)
 335 #endif
 336
 337
 338 #ifndef atomic_decrement_val
 339 # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1)
 340 #endif
 341
 342
 343 #ifndef catomic_decrement_val
 344 # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1)
 345 #endif
 346
 347
 348 /* Subtract 1 from *MEM and return true iff it's now zero.  */
 349 #ifndef atomic_decrement_and_test
 350 # define atomic_decrement_and_test(mem) \
 351   (atomic_exchange_and_add ((mem), -1) == 1)
 352 #endif
 353
 354
 355 /* Decrement *MEM if it is > 0, and return the old value.  */
 356 #ifndef atomic_decrement_if_positive
 357 # define atomic_decrement_if_positive(mem) \
 358   ({ __typeof (*(mem)) __atg11_oldval;                                        \
 359      __typeof (mem) __atg11_memp = (mem);                                     \
 360                                                                               \
 361      do                                                                       \
 362        {                                                                      \
 363          __atg11_oldval = *__atg11_memp;                                      \
 364          if (__builtin_expect (__atg11_oldval <= 0, 0))                       \
 365            break;                                                             \
 366        }                                                                      \
 367      while (__builtin_expect                                                  \
 368             (atomic_compare_and_exchange_bool_acq (__atg11_memp,              \
 369                                                    __atg11_oldval - 1,        \
 370                                                    __atg11_oldval), 0));      \
 371      __atg11_oldval; })
 372 #endif
 373
 374
 375 #ifndef atomic_add_negative
 376 # define atomic_add_negative(mem, value)                                      \
 377   ({ __typeof (value) __atg12_value = (value);                                \
 378      atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; })
 379 #endif
 380
 381
 382 #ifndef atomic_add_zero
 383 # define atomic_add_zero(mem, value)                                          \
 384   ({ __typeof (value) __atg13_value = (value);                                \
 385      atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; })
 386 #endif
 387
 388
 389 #ifndef atomic_bit_set
 390 # define atomic_bit_set(mem, bit) \
 391   (void) atomic_bit_test_set(mem, bit)
 392 #endif
 393
 394
 395 #ifndef atomic_bit_test_set
 396 # define atomic_bit_test_set(mem, bit) \
 397   ({ __typeof (*(mem)) __atg14_old;                                           \
 398      __typeof (mem) __atg14_memp = (mem);                                     \
 399      __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit));       \
 400                                                                               \
 401      do                                                                       \
 402        __atg14_old = (*__atg14_memp);                                         \
 403      while (__builtin_expect                                                  \
 404             (atomic_compare_and_exchange_bool_acq (__atg14_memp,              \
 405                                                    __atg14_old | __atg14_mask,\
 406                                                    __atg14_old), 0));         \
 407                                                                               \
 408      __atg14_old & __atg14_mask; })
 409 #endif
 410
 411 /* Atomically *mem &= mask.  */
 412 #ifndef atomic_and
 413 # define atomic_and(mem, mask) \
 414   do {                                                                        \
 415     __typeof (*(mem)) __atg15_old;                                            \
 416     __typeof (mem) __atg15_memp = (mem);                                      \
 417     __typeof (*(mem)) __atg15_mask = (mask);                                  \
 418                                                                               \
 419     do                                                                        \
 420       __atg15_old = (*__atg15_memp);                                          \
 421     while (__builtin_expect                                                   \
 422            (atomic_compare_and_exchange_bool_acq (__atg15_memp,               \
 423                                                   __atg15_old & __atg15_mask, \
 424                                                   __atg15_old), 0));          \
 425   } while (0)
 426 #endif
 427
 428 #ifndef catomic_and
 429 # define catomic_and(mem, mask) \
 430   do {                                                                        \
 431     __typeof (*(mem)) __atg20_old;                                            \
 432     __typeof (mem) __atg20_memp = (mem);                                      \
 433     __typeof (*(mem)) __atg20_mask = (mask);                                  \
 434                                                                               \
 435     do                                                                        \
 436       __atg20_old = (*__atg20_memp);                                          \
 437     while (__builtin_expect                                                   \
 438            (catomic_compare_and_exchange_bool_acq (__atg20_memp,              \
 439                                                    __atg20_old & __atg20_mask,\
 440                                                    __atg20_old), 0));         \
 441   } while (0)
 442 #endif
 443
 444 /* Atomically *mem &= mask and return the old value of *mem.  */
 445 #ifndef atomic_and_val
 446 # define atomic_and_val(mem, mask) \
 447   ({ __typeof (*(mem)) __atg16_old;                                           \
 448      __typeof (mem) __atg16_memp = (mem);                                     \
 449      __typeof (*(mem)) __atg16_mask = (mask);                                 \
 450                                                                               \
 451      do                                                                       \
 452        __atg16_old = (*__atg16_memp);                                         \
 453      while (__builtin_expect                                                  \
 454             (atomic_compare_and_exchange_bool_acq (__atg16_memp,              \
 455                                                    __atg16_old & __atg16_mask,\
 456                                                    __atg16_old), 0));         \
 457                                                                               \
 458      __atg16_old; })
 459 #endif
 460
 461 /* Atomically *mem |= mask and return the old value of *mem.  */
 462 #ifndef atomic_or
 463 # define atomic_or(mem, mask) \
 464   do {                                                                        \
 465     __typeof (*(mem)) __atg17_old;                                            \
 466     __typeof (mem) __atg17_memp = (mem);                                      \
 467     __typeof (*(mem)) __atg17_mask = (mask);                                  \
 468                                                                               \
 469     do                                                                        \
 470       __atg17_old = (*__atg17_memp);                                          \
 471     while (__builtin_expect                                                   \
 472            (atomic_compare_and_exchange_bool_acq (__atg17_memp,               \
 473                                                   __atg17_old | __atg17_mask, \
 474                                                   __atg17_old), 0));          \
 475   } while (0)
 476 #endif
 477
 478 #ifndef catomic_or
 479 # define catomic_or(mem, mask) \
 480   do {                                                                        \
 481     __typeof (*(mem)) __atg18_old;                                            \
 482     __typeof (mem) __atg18_memp = (mem);                                      \
 483     __typeof (*(mem)) __atg18_mask = (mask);                                  \
 484                                                                               \
 485     do                                                                        \
 486       __atg18_old = (*__atg18_memp);                                          \
 487     while (__builtin_expect                                                   \
 488            (catomic_compare_and_exchange_bool_acq (__atg18_memp,              \
 489                                                    __atg18_old | __atg18_mask,\
 490                                                    __atg18_old), 0));         \
 491   } while (0)
 492 #endif
 493
 494 /* Atomically *mem |= mask and return the old value of *mem.  */
 495 #ifndef atomic_or_val
 496 # define atomic_or_val(mem, mask) \
 497   ({ __typeof (*(mem)) __atg19_old;                                           \
 498      __typeof (mem) __atg19_memp = (mem);                                     \
 499      __typeof (*(mem)) __atg19_mask = (mask);                                 \
 500                                                                               \
 501      do                                                                       \
 502        __atg19_old = (*__atg19_memp);                                         \
 503      while (__builtin_expect                                                  \
 504             (atomic_compare_and_exchange_bool_acq (__atg19_memp,              \
 505                                                    __atg19_old | __atg19_mask,\
 506                                                    __atg19_old), 0));         \
 507                                                                               \
 508      __atg19_old; })
 509 #endif
 510
 511 #ifndef atomic_full_barrier
 512 # define atomic_full_barrier() __asm__ ("" ::: "memory")
 513 #endif
 514
 515
 516 #ifndef atomic_read_barrier
 517 # define atomic_read_barrier() atomic_full_barrier ()
 518 #endif
 519
 520
 521 #ifndef atomic_write_barrier
 522 # define atomic_write_barrier() atomic_full_barrier ()
 523 #endif
 524
 525
 526 #ifndef atomic_forced_read
 527 # define atomic_forced_read(x) \
 528   ({ __typeof (x) __x; __asm__ ("" : "=r" (__x) : "0" (x)); __x; })
 529 #endif
 530
 531
 532 #ifndef atomic_delay
 533 # define atomic_delay() do { /* nothing */ } while (0)
 534 #endif
 535
 536 #endif  /* atomic.h */