2 * Atomic helper templates
3 * Included from tcg-runtime.c and cputlb.c.
5 * Copyright (c) 2016 Red Hat, Inc
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
21 #include "trace/mem.h"
25 # define DATA_TYPE Int128
26 # define BSWAP bswap128
30 # define DATA_TYPE uint64_t
31 # define SDATA_TYPE int64_t
32 # define BSWAP bswap64
36 # define DATA_TYPE uint32_t
37 # define SDATA_TYPE int32_t
38 # define BSWAP bswap32
42 # define DATA_TYPE uint16_t
43 # define SDATA_TYPE int16_t
44 # define BSWAP bswap16
48 # define DATA_TYPE uint8_t
49 # define SDATA_TYPE int8_t
53 # error unsupported data size
57 # define ABI_TYPE DATA_TYPE
59 # define ABI_TYPE uint32_t
62 #define ATOMIC_TRACE_RMW do { \
63 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
65 trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \
66 trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, \
67 info | TRACE_MEM_ST); \
70 #define ATOMIC_TRACE_LD do { \
71 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
73 trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \
76 # define ATOMIC_TRACE_ST do { \
77 uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \
79 trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \
82 /* Define host-endian atomic operations. Note that END is used within
83 the ATOMIC_NAME macro, and redefined below. */
86 # define MEND _be /* either le or be would be fine */
87 #elif defined(HOST_WORDS_BIGENDIAN)
95 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
96 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
99 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
103 ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
109 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
112 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
115 __atomic_load(haddr, &val, __ATOMIC_RELAXED);
120 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
121 ABI_TYPE val EXTRA_ARGS)
124 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
127 __atomic_store(haddr, &val, __ATOMIC_RELAXED);
131 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
132 ABI_TYPE val EXTRA_ARGS)
135 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
139 ret = atomic_xchg__nocheck(haddr, val);
144 #define GEN_ATOMIC_HELPER(X) \
145 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
146 ABI_TYPE val EXTRA_ARGS) \
149 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
153 ret = atomic_##X(haddr, val); \
154 ATOMIC_MMU_CLEANUP; \
158 GEN_ATOMIC_HELPER(fetch_add)
159 GEN_ATOMIC_HELPER(fetch_and)
160 GEN_ATOMIC_HELPER(fetch_or)
161 GEN_ATOMIC_HELPER(fetch_xor)
162 GEN_ATOMIC_HELPER(add_fetch)
163 GEN_ATOMIC_HELPER(and_fetch)
164 GEN_ATOMIC_HELPER(or_fetch)
165 GEN_ATOMIC_HELPER(xor_fetch)
167 #undef GEN_ATOMIC_HELPER
169 /* These helpers are, as a whole, full barriers. Within the helper,
170 * the leading barrier is explicit and the trailing barrier is within
173 * Trace this load + RMW loop as a single RMW op. This way, regardless
174 * of CF_PARALLEL's value, we'll trace just a read and a write.
176 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
177 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
178 ABI_TYPE xval EXTRA_ARGS) \
181 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
182 XDATA_TYPE cmp, old, new, val = xval; \
186 cmp = atomic_read__nocheck(haddr); \
188 old = cmp; new = FN(old, val); \
189 cmp = atomic_cmpxchg__nocheck(haddr, old, new); \
190 } while (cmp != old); \
191 ATOMIC_MMU_CLEANUP; \
195 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
196 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
197 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
198 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
200 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
201 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
202 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
203 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
205 #undef GEN_ATOMIC_HELPER_FN
206 #endif /* DATA SIZE >= 16 */
213 /* Define reverse-host-endian atomic operations. Note that END is used
214 within the ATOMIC_NAME macro. */
215 #ifdef HOST_WORDS_BIGENDIAN
223 ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
224 ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
227 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
231 ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
237 ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
240 DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
243 __atomic_load(haddr, &val, __ATOMIC_RELAXED);
248 void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
249 ABI_TYPE val EXTRA_ARGS)
252 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
256 __atomic_store(haddr, &val, __ATOMIC_RELAXED);
260 ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
261 ABI_TYPE val EXTRA_ARGS)
264 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
268 ret = atomic_xchg__nocheck(haddr, BSWAP(val));
273 #define GEN_ATOMIC_HELPER(X) \
274 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
275 ABI_TYPE val EXTRA_ARGS) \
278 DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
282 ret = atomic_##X(haddr, BSWAP(val)); \
283 ATOMIC_MMU_CLEANUP; \
287 GEN_ATOMIC_HELPER(fetch_and)
288 GEN_ATOMIC_HELPER(fetch_or)
289 GEN_ATOMIC_HELPER(fetch_xor)
290 GEN_ATOMIC_HELPER(and_fetch)
291 GEN_ATOMIC_HELPER(or_fetch)
292 GEN_ATOMIC_HELPER(xor_fetch)
294 #undef GEN_ATOMIC_HELPER
296 /* These helpers are, as a whole, full barriers. Within the helper,
297 * the leading barrier is explicit and the trailing barrier is within
300 * Trace this load + RMW loop as a single RMW op. This way, regardless
301 * of CF_PARALLEL's value, we'll trace just a read and a write.
303 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
304 ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
305 ABI_TYPE xval EXTRA_ARGS) \
308 XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
309 XDATA_TYPE ldo, ldn, old, new, val = xval; \
313 ldn = atomic_read__nocheck(haddr); \
315 ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
316 ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
317 } while (ldo != ldn); \
318 ATOMIC_MMU_CLEANUP; \
322 GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
323 GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
324 GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
325 GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
327 GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
328 GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
329 GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
330 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
332 /* Note that for addition, we need to use a separate cmpxchg loop instead
333 of bswaps for the reverse-host-endian helpers. */
334 #define ADD(X, Y) (X + Y)
335 GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
336 GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
339 #undef GEN_ATOMIC_HELPER_FN
340 #endif /* DATA_SIZE >= 16 */
344 #endif /* DATA_SIZE > 1 */
346 #undef ATOMIC_TRACE_ST
347 #undef ATOMIC_TRACE_LD
348 #undef ATOMIC_TRACE_RMW