accel/tcg/atomic_template.h

   1 /*
   2  * Atomic helper templates
   3  * Included from tcg-runtime.c and cputlb.c.
   4  *
   5  * Copyright (c) 2016 Red Hat, Inc
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Lesser General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Lesser General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public
  18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #include "trace/mem.h"
  22
  23 #if DATA_SIZE == 16
  24 # define SUFFIX     o
  25 # define DATA_TYPE  Int128
  26 # define BSWAP      bswap128
  27 # define SHIFT      4
  28 #elif DATA_SIZE == 8
  29 # define SUFFIX     q
  30 # define DATA_TYPE  uint64_t
  31 # define SDATA_TYPE int64_t
  32 # define BSWAP      bswap64
  33 # define SHIFT      3
  34 #elif DATA_SIZE == 4
  35 # define SUFFIX     l
  36 # define DATA_TYPE  uint32_t
  37 # define SDATA_TYPE int32_t
  38 # define BSWAP      bswap32
  39 # define SHIFT      2
  40 #elif DATA_SIZE == 2
  41 # define SUFFIX     w
  42 # define DATA_TYPE  uint16_t
  43 # define SDATA_TYPE int16_t
  44 # define BSWAP      bswap16
  45 # define SHIFT      1
  46 #elif DATA_SIZE == 1
  47 # define SUFFIX     b
  48 # define DATA_TYPE  uint8_t
  49 # define SDATA_TYPE int8_t
  50 # define BSWAP
  51 # define SHIFT      0
  52 #else
  53 # error unsupported data size
  54 #endif
  55
  56 #if DATA_SIZE >= 4
  57 # define ABI_TYPE  DATA_TYPE
  58 #else
  59 # define ABI_TYPE  uint32_t
  60 #endif
  61
  62 #define ATOMIC_TRACE_RMW do {                                           \
  63         uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
  64                                                                         \
  65         trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
  66         trace_guest_mem_before_exec(ENV_GET_CPU(env), addr,             \
  67                                     info | TRACE_MEM_ST);               \
  68     } while (0)
  69
  70 #define ATOMIC_TRACE_LD do {                                            \
  71         uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
  72                                                                         \
  73         trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
  74     } while (0)
  75
  76 # define ATOMIC_TRACE_ST do {                                           \
  77         uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \
  78                                                                         \
  79         trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info);      \
  80     } while (0)
  81
  82 /* Define host-endian atomic operations.  Note that END is used within
  83    the ATOMIC_NAME macro, and redefined below.  */
  84 #if DATA_SIZE == 1
  85 # define END
  86 # define MEND _be /* either le or be would be fine */
  87 #elif defined(HOST_WORDS_BIGENDIAN)
  88 # define END  _be
  89 # define MEND _be
  90 #else
  91 # define END  _le
  92 # define MEND _le
  93 #endif
  94
  95 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
  96                               ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
  97 {
  98     ATOMIC_MMU_DECLS;
  99     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 100     DATA_TYPE ret;
 101
 102     ATOMIC_TRACE_RMW;
 103     ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
 104     ATOMIC_MMU_CLEANUP;
 105     return ret;
 106 }
 107
 108 #if DATA_SIZE >= 16
 109 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 110 {
 111     ATOMIC_MMU_DECLS;
 112     DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 113
 114     ATOMIC_TRACE_LD;
 115     __atomic_load(haddr, &val, __ATOMIC_RELAXED);
 116     ATOMIC_MMU_CLEANUP;
 117     return val;
 118 }
 119
 120 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 121                      ABI_TYPE val EXTRA_ARGS)
 122 {
 123     ATOMIC_MMU_DECLS;
 124     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 125
 126     ATOMIC_TRACE_ST;
 127     __atomic_store(haddr, &val, __ATOMIC_RELAXED);
 128     ATOMIC_MMU_CLEANUP;
 129 }
 130 #else
 131 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 132                            ABI_TYPE val EXTRA_ARGS)
 133 {
 134     ATOMIC_MMU_DECLS;
 135     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 136     DATA_TYPE ret;
 137
 138     ATOMIC_TRACE_RMW;
 139     ret = atomic_xchg__nocheck(haddr, val);
 140     ATOMIC_MMU_CLEANUP;
 141     return ret;
 142 }
 143
 144 #define GEN_ATOMIC_HELPER(X)                                        \
 145 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 146                  ABI_TYPE val EXTRA_ARGS)                           \
 147 {                                                                   \
 148     ATOMIC_MMU_DECLS;                                               \
 149     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 150     DATA_TYPE ret;                                                  \
 151                                                                     \
 152     ATOMIC_TRACE_RMW;                                               \
 153     ret = atomic_##X(haddr, val);                                   \
 154     ATOMIC_MMU_CLEANUP;                                             \
 155     return ret;                                                     \
 156 }
 157
 158 GEN_ATOMIC_HELPER(fetch_add)
 159 GEN_ATOMIC_HELPER(fetch_and)
 160 GEN_ATOMIC_HELPER(fetch_or)
 161 GEN_ATOMIC_HELPER(fetch_xor)
 162 GEN_ATOMIC_HELPER(add_fetch)
 163 GEN_ATOMIC_HELPER(and_fetch)
 164 GEN_ATOMIC_HELPER(or_fetch)
 165 GEN_ATOMIC_HELPER(xor_fetch)
 166
 167 #undef GEN_ATOMIC_HELPER
 168
 169 /* These helpers are, as a whole, full barriers.  Within the helper,
 170  * the leading barrier is explicit and the trailing barrier is within
 171  * cmpxchg primitive.
 172  *
 173  * Trace this load + RMW loop as a single RMW op. This way, regardless
 174  * of CF_PARALLEL's value, we'll trace just a read and a write.
 175  */
 176 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 177 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 178                         ABI_TYPE xval EXTRA_ARGS)                   \
 179 {                                                                   \
 180     ATOMIC_MMU_DECLS;                                               \
 181     XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
 182     XDATA_TYPE cmp, old, new, val = xval;                           \
 183                                                                     \
 184     ATOMIC_TRACE_RMW;                                               \
 185     smp_mb();                                                       \
 186     cmp = atomic_read__nocheck(haddr);                              \
 187     do {                                                            \
 188         old = cmp; new = FN(old, val);                              \
 189         cmp = atomic_cmpxchg__nocheck(haddr, old, new);             \
 190     } while (cmp != old);                                           \
 191     ATOMIC_MMU_CLEANUP;                                             \
 192     return RET;                                                     \
 193 }
 194
 195 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 196 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 197 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 198 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 199
 200 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 201 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 202 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 203 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 204
 205 #undef GEN_ATOMIC_HELPER_FN
 206 #endif /* DATA SIZE >= 16 */
 207
 208 #undef END
 209 #undef MEND
 210
 211 #if DATA_SIZE > 1
 212
 213 /* Define reverse-host-endian atomic operations.  Note that END is used
 214    within the ATOMIC_NAME macro.  */
 215 #ifdef HOST_WORDS_BIGENDIAN
 216 # define END  _le
 217 # define MEND _le
 218 #else
 219 # define END  _be
 220 # define MEND _be
 221 #endif
 222
 223 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
 224                               ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
 225 {
 226     ATOMIC_MMU_DECLS;
 227     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 228     DATA_TYPE ret;
 229
 230     ATOMIC_TRACE_RMW;
 231     ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 232     ATOMIC_MMU_CLEANUP;
 233     return BSWAP(ret);
 234 }
 235
 236 #if DATA_SIZE >= 16
 237 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
 238 {
 239     ATOMIC_MMU_DECLS;
 240     DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
 241
 242     ATOMIC_TRACE_LD;
 243     __atomic_load(haddr, &val, __ATOMIC_RELAXED);
 244     ATOMIC_MMU_CLEANUP;
 245     return BSWAP(val);
 246 }
 247
 248 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
 249                      ABI_TYPE val EXTRA_ARGS)
 250 {
 251     ATOMIC_MMU_DECLS;
 252     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 253
 254     ATOMIC_TRACE_ST;
 255     val = BSWAP(val);
 256     __atomic_store(haddr, &val, __ATOMIC_RELAXED);
 257     ATOMIC_MMU_CLEANUP;
 258 }
 259 #else
 260 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
 261                            ABI_TYPE val EXTRA_ARGS)
 262 {
 263     ATOMIC_MMU_DECLS;
 264     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
 265     ABI_TYPE ret;
 266
 267     ATOMIC_TRACE_RMW;
 268     ret = atomic_xchg__nocheck(haddr, BSWAP(val));
 269     ATOMIC_MMU_CLEANUP;
 270     return BSWAP(ret);
 271 }
 272
 273 #define GEN_ATOMIC_HELPER(X)                                        \
 274 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 275                  ABI_TYPE val EXTRA_ARGS)                           \
 276 {                                                                   \
 277     ATOMIC_MMU_DECLS;                                               \
 278     DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
 279     DATA_TYPE ret;                                                  \
 280                                                                     \
 281     ATOMIC_TRACE_RMW;                                               \
 282     ret = atomic_##X(haddr, BSWAP(val));                            \
 283     ATOMIC_MMU_CLEANUP;                                             \
 284     return BSWAP(ret);                                              \
 285 }
 286
 287 GEN_ATOMIC_HELPER(fetch_and)
 288 GEN_ATOMIC_HELPER(fetch_or)
 289 GEN_ATOMIC_HELPER(fetch_xor)
 290 GEN_ATOMIC_HELPER(and_fetch)
 291 GEN_ATOMIC_HELPER(or_fetch)
 292 GEN_ATOMIC_HELPER(xor_fetch)
 293
 294 #undef GEN_ATOMIC_HELPER
 295
 296 /* These helpers are, as a whole, full barriers.  Within the helper,
 297  * the leading barrier is explicit and the trailing barrier is within
 298  * cmpxchg primitive.
 299  *
 300  * Trace this load + RMW loop as a single RMW op. This way, regardless
 301  * of CF_PARALLEL's value, we'll trace just a read and a write.
 302  */
 303 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
 304 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
 305                         ABI_TYPE xval EXTRA_ARGS)                   \
 306 {                                                                   \
 307     ATOMIC_MMU_DECLS;                                               \
 308     XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
 309     XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
 310                                                                     \
 311     ATOMIC_TRACE_RMW;                                               \
 312     smp_mb();                                                       \
 313     ldn = atomic_read__nocheck(haddr);                              \
 314     do {                                                            \
 315         ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
 316         ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));      \
 317     } while (ldo != ldn);                                           \
 318     ATOMIC_MMU_CLEANUP;                                             \
 319     return RET;                                                     \
 320 }
 321
 322 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
 323 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
 324 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
 325 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)
 326
 327 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
 328 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
 329 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 330 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 331
 332 /* Note that for addition, we need to use a separate cmpxchg loop instead
 333    of bswaps for the reverse-host-endian helpers.  */
 334 #define ADD(X, Y)   (X + Y)
 335 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
 336 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 337 #undef ADD
 338
 339 #undef GEN_ATOMIC_HELPER_FN
 340 #endif /* DATA_SIZE >= 16 */
 341
 342 #undef END
 343 #undef MEND
 344 #endif /* DATA_SIZE > 1 */
 345
 346 #undef ATOMIC_TRACE_ST
 347 #undef ATOMIC_TRACE_LD
 348 #undef ATOMIC_TRACE_RMW
 349
 350 #undef BSWAP
 351 #undef ABI_TYPE
 352 #undef DATA_TYPE
 353 #undef SDATA_TYPE
 354 #undef SUFFIX
 355 #undef DATA_SIZE
 356 #undef SHIFT