[qemu.git] / accel / tcg / atomic_template.h

/*
 * Atomic helper templates
 * Included from tcg-runtime.c and cputlb.c.
 *
 * Copyright (c) 2016 Red Hat, Inc
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

#include "qemu/plugin.h"
#include "trace/mem.h"

#if DATA_SIZE == 16
# define SUFFIX     o
# define DATA_TYPE  Int128
# define BSWAP      bswap128
# define SHIFT      4
#elif DATA_SIZE == 8
# define SUFFIX     q
# define DATA_TYPE  uint64_t
# define SDATA_TYPE int64_t
# define BSWAP      bswap64
# define SHIFT      3
#elif DATA_SIZE == 4
# define SUFFIX     l
# define DATA_TYPE  uint32_t
# define SDATA_TYPE int32_t
# define BSWAP      bswap32
# define SHIFT      2
#elif DATA_SIZE == 2
# define SUFFIX     w
# define DATA_TYPE  uint16_t
# define SDATA_TYPE int16_t
# define BSWAP      bswap16
# define SHIFT      1
#elif DATA_SIZE == 1
# define SUFFIX     b
# define DATA_TYPE  uint8_t
# define SDATA_TYPE int8_t
# define BSWAP
# define SHIFT      0
#else
# error unsupported data size
#endif

#if DATA_SIZE >= 4
# define ABI_TYPE  DATA_TYPE
#else
# define ABI_TYPE  uint32_t
#endif

/* Define host-endian atomic operations.  Note that END is used within
   the ATOMIC_NAME macro, and redefined below.  */
#if DATA_SIZE == 1
# define END
# define MEND _be /* either le or be would be fine */
#elif defined(HOST_WORDS_BIGENDIAN)
# define END  _be
# define MEND _be
#else
# define END  _le
# define MEND _le
#endif

ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
                                                           ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
#if DATA_SIZE == 16
    ret = atomic16_cmpxchg(haddr, cmpv, newv);
#else
    ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
#endif
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return ret;
}

#if DATA_SIZE >= 16
#if HAVE_ATOMIC128
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
                                                           ATOMIC_MMU_IDX);

    atomic_trace_ld_pre(env, addr, info);
    val = atomic16_read(haddr);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_ld_post(env, addr, info);
    return val;
}

void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
                     ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true,
                                                          ATOMIC_MMU_IDX);

    atomic_trace_st_pre(env, addr, info);
    atomic16_set(haddr, val);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_st_post(env, addr, info);
}
#endif
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                           ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
                                                          ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
    ret = atomic_xchg__nocheck(haddr, val);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return ret;
}

#define GEN_ATOMIC_HELPER(X)                                        \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE val EXTRA_ARGS)                    \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    DATA_TYPE ret;                                                  \
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,   \
                                                           false,   \
                                                           ATOMIC_MMU_IDX); \
                                                                    \
    atomic_trace_rmw_pre(env, addr, info);                          \
    ret = atomic_##X(haddr, val);                                   \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return ret;                                                     \
}

GEN_ATOMIC_HELPER(fetch_add)
GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(fetch_or)
GEN_ATOMIC_HELPER(fetch_xor)
GEN_ATOMIC_HELPER(add_fetch)
GEN_ATOMIC_HELPER(and_fetch)
GEN_ATOMIC_HELPER(or_fetch)
GEN_ATOMIC_HELPER(xor_fetch)

#undef GEN_ATOMIC_HELPER

/* These helpers are, as a whole, full barriers.  Within the helper,
 * the leading barrier is explicit and the trailing barrier is within
 * cmpxchg primitive.
 *
 * Trace this load + RMW loop as a single RMW op. This way, regardless
 * of CF_PARALLEL's value, we'll trace just a read and a write.
 */
#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE xval EXTRA_ARGS)                   \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
    XDATA_TYPE cmp, old, new, val = xval;                           \
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,   \
                                                           false,   \
                                                           ATOMIC_MMU_IDX); \
                                                                    \
    atomic_trace_rmw_pre(env, addr, info);                          \
    smp_mb();                                                       \
    cmp = atomic_read__nocheck(haddr);                              \
    do {                                                            \
        old = cmp; new = FN(old, val);                              \
        cmp = atomic_cmpxchg__nocheck(haddr, old, new);             \
    } while (cmp != old);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return RET;                                                     \
}

GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)

GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)

#undef GEN_ATOMIC_HELPER_FN
#endif /* DATA SIZE >= 16 */

#undef END
#undef MEND

#if DATA_SIZE > 1

/* Define reverse-host-endian atomic operations.  Note that END is used
   within the ATOMIC_NAME macro.  */
#ifdef HOST_WORDS_BIGENDIAN
# define END  _le
# define MEND _le
#else
# define END  _be
# define MEND _be
#endif

ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
                              ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    DATA_TYPE ret;
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
                                                           false,
                                                           ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
#if DATA_SIZE == 16
    ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
#else
    ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
#endif
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return BSWAP(ret);
}

#if DATA_SIZE >= 16
#if HAVE_ATOMIC128
ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
                                                           false,
                                                           ATOMIC_MMU_IDX);

    atomic_trace_ld_pre(env, addr, info);
    val = atomic16_read(haddr);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_ld_post(env, addr, info);
    return BSWAP(val);
}

void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
                     ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
                                                           true,
                                                           ATOMIC_MMU_IDX);

    val = BSWAP(val);
    atomic_trace_st_pre(env, addr, info);
    val = BSWAP(val);
    atomic16_set(haddr, val);
    ATOMIC_MMU_CLEANUP;
    atomic_trace_st_post(env, addr, info);
}
#endif
#else
ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
                           ABI_TYPE val EXTRA_ARGS)
{
    ATOMIC_MMU_DECLS;
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
    ABI_TYPE ret;
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
                                                           false,
                                                           ATOMIC_MMU_IDX);

    atomic_trace_rmw_pre(env, addr, info);
    ret = atomic_xchg__nocheck(haddr, BSWAP(val));
    ATOMIC_MMU_CLEANUP;
    atomic_trace_rmw_post(env, addr, info);
    return BSWAP(ret);
}

#define GEN_ATOMIC_HELPER(X)                                        \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE val EXTRA_ARGS)                    \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                           \
    DATA_TYPE ret;                                                  \
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,   \
                                                           false,   \
                                                           ATOMIC_MMU_IDX); \
                                                                    \
    atomic_trace_rmw_pre(env, addr, info);                          \
    ret = atomic_##X(haddr, BSWAP(val));                            \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return BSWAP(ret);                                              \
}

GEN_ATOMIC_HELPER(fetch_and)
GEN_ATOMIC_HELPER(fetch_or)
GEN_ATOMIC_HELPER(fetch_xor)
GEN_ATOMIC_HELPER(and_fetch)
GEN_ATOMIC_HELPER(or_fetch)
GEN_ATOMIC_HELPER(xor_fetch)

#undef GEN_ATOMIC_HELPER

/* These helpers are, as a whole, full barriers.  Within the helper,
 * the leading barrier is explicit and the trailing barrier is within
 * cmpxchg primitive.
 *
 * Trace this load + RMW loop as a single RMW op. This way, regardless
 * of CF_PARALLEL's value, we'll trace just a read and a write.
 */
#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr,       \
                        ABI_TYPE xval EXTRA_ARGS)                   \
{                                                                   \
    ATOMIC_MMU_DECLS;                                               \
    XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;                          \
    XDATA_TYPE ldo, ldn, old, new, val = xval;                      \
    uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,   \
                                                           false,   \
                                                           ATOMIC_MMU_IDX); \
                                                                    \
    atomic_trace_rmw_pre(env, addr, info);                          \
    smp_mb();                                                       \
    ldn = atomic_read__nocheck(haddr);                              \
    do {                                                            \
        ldo = ldn; old = BSWAP(ldo); new = FN(old, val);            \
        ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));      \
    } while (ldo != ldn);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
    atomic_trace_rmw_post(env, addr, info);                         \
    return RET;                                                     \
}

GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umin, MIN,  DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(fetch_umax, MAX,  DATA_TYPE, old)

GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umin_fetch, MIN,  DATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)

/* Note that for addition, we need to use a separate cmpxchg loop instead
   of bswaps for the reverse-host-endian helpers.  */
#define ADD(X, Y)   (X + Y)
GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
#undef ADD

#undef GEN_ATOMIC_HELPER_FN
#endif /* DATA_SIZE >= 16 */

#undef END
#undef MEND
#endif /* DATA_SIZE > 1 */

#undef BSWAP
#undef ABI_TYPE
#undef DATA_TYPE
#undef SDATA_TYPE
#undef SUFFIX
#undef DATA_SIZE
#undef SHIFT
Commit	Line	Data
c482cb11 RH	1	/*
	2	* Atomic helper templates
	3	* Included from tcg-runtime.c and cputlb.c.
	4	*
	5	* Copyright (c) 2016 Red Hat, Inc
	6	*
	7	* This library is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU Lesser General Public
	9	* License as published by the Free Software Foundation; either
fb0343d5	10	* version 2.1 of the License, or (at your option) any later version.
c482cb11 RH	11	*
	12	* This library is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	15	* Lesser General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU Lesser General Public
	18	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
e6d86bed	21	#include "qemu/plugin.h"
d071f4cd EC	22	#include "trace/mem.h"
d071f4cd EC	23
7ebee43e RH	24	#if DATA_SIZE == 16
	25	# define SUFFIX o
	26	# define DATA_TYPE Int128
	27	# define BSWAP bswap128
d071f4cd	28	# define SHIFT 4
7ebee43e	29	#elif DATA_SIZE == 8
c482cb11 RH	30	# define SUFFIX q
c482cb11 RH	31	# define DATA_TYPE uint64_t
5507c2bf	32	# define SDATA_TYPE int64_t
c482cb11	33	# define BSWAP bswap64
d071f4cd	34	# define SHIFT 3
c482cb11 RH	35	#elif DATA_SIZE == 4
	36	# define SUFFIX l
	37	# define DATA_TYPE uint32_t
5507c2bf	38	# define SDATA_TYPE int32_t
c482cb11	39	# define BSWAP bswap32
d071f4cd	40	# define SHIFT 2
c482cb11 RH	41	#elif DATA_SIZE == 2
	42	# define SUFFIX w
	43	# define DATA_TYPE uint16_t
5507c2bf	44	# define SDATA_TYPE int16_t
c482cb11	45	# define BSWAP bswap16
d071f4cd	46	# define SHIFT 1
c482cb11 RH	47	#elif DATA_SIZE == 1
	48	# define SUFFIX b
	49	# define DATA_TYPE uint8_t
5507c2bf	50	# define SDATA_TYPE int8_t
c482cb11	51	# define BSWAP
d071f4cd	52	# define SHIFT 0
c482cb11 RH	53	#else
	54	# error unsupported data size
	55	#endif
	56
	57	#if DATA_SIZE >= 4
	58	# define ABI_TYPE DATA_TYPE
	59	#else
	60	# define ABI_TYPE uint32_t
	61	#endif
	62
	63	/* Define host-endian atomic operations. Note that END is used within
	64	the ATOMIC_NAME macro, and redefined below. */
	65	#if DATA_SIZE == 1
	66	# define END
d071f4cd	67	# define MEND _be /* either le or be would be fine */
c482cb11 RH	68	#elif defined(HOST_WORDS_BIGENDIAN)
c482cb11 RH	69	# define END _be
d071f4cd	70	# define MEND _be
c482cb11 RH	71	#else
c482cb11 RH	72	# define END _le
d071f4cd	73	# define MEND _le
c482cb11 RH	74	#endif
	75
	76	ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
	77	ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
	78	{
34d49937	79	ATOMIC_MMU_DECLS;
c482cb11	80	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd	81	DATA_TYPE ret;
cfec3885 EC	82	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
cfec3885 EC	83	ATOMIC_MMU_IDX);
d071f4cd	84
cfec3885	85	atomic_trace_rmw_pre(env, addr, info);
e6cd4bb5 RH	86	#if DATA_SIZE == 16
	87	ret = atomic16_cmpxchg(haddr, cmpv, newv);
	88	#else
d071f4cd	89	ret = atomic_cmpxchg__nocheck(haddr, cmpv, newv);
e6cd4bb5	90	#endif
ec603b55	91	ATOMIC_MMU_CLEANUP;
cfec3885	92	atomic_trace_rmw_post(env, addr, info);
ec603b55	93	return ret;
c482cb11 RH	94	}
c482cb11 RH	95
7ebee43e	96	#if DATA_SIZE >= 16
e6cd4bb5	97	#if HAVE_ATOMIC128
7ebee43e RH	98	ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
7ebee43e RH	99	{
34d49937	100	ATOMIC_MMU_DECLS;
7ebee43e	101	DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
cfec3885 EC	102	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
cfec3885 EC	103	ATOMIC_MMU_IDX);
d071f4cd	104
cfec3885	105	atomic_trace_ld_pre(env, addr, info);
e6cd4bb5	106	val = atomic16_read(haddr);
ec603b55	107	ATOMIC_MMU_CLEANUP;
cfec3885	108	atomic_trace_ld_post(env, addr, info);
7ebee43e RH	109	return val;
	110	}
	111
	112	void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
	113	ABI_TYPE val EXTRA_ARGS)
	114	{
34d49937	115	ATOMIC_MMU_DECLS;
7ebee43e	116	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
cfec3885 EC	117	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true,
cfec3885 EC	118	ATOMIC_MMU_IDX);
d071f4cd	119
cfec3885	120	atomic_trace_st_pre(env, addr, info);
e6cd4bb5	121	atomic16_set(haddr, val);
ec603b55	122	ATOMIC_MMU_CLEANUP;
cfec3885	123	atomic_trace_st_post(env, addr, info);
7ebee43e	124	}
e6cd4bb5	125	#endif
7ebee43e	126	#else
c482cb11 RH	127	ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
	128	ABI_TYPE val EXTRA_ARGS)
	129	{
34d49937	130	ATOMIC_MMU_DECLS;
c482cb11	131	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd	132	DATA_TYPE ret;
cfec3885 EC	133	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false,
cfec3885 EC	134	ATOMIC_MMU_IDX);
d071f4cd	135
cfec3885	136	atomic_trace_rmw_pre(env, addr, info);
d071f4cd	137	ret = atomic_xchg__nocheck(haddr, val);
ec603b55	138	ATOMIC_MMU_CLEANUP;
cfec3885	139	atomic_trace_rmw_post(env, addr, info);
ec603b55	140	return ret;
c482cb11 RH	141	}
	142
	143	#define GEN_ATOMIC_HELPER(X) \
	144	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
d2fac5f6	145	ABI_TYPE val EXTRA_ARGS) \
c482cb11	146	{ \
34d49937	147	ATOMIC_MMU_DECLS; \
c482cb11	148	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
d071f4cd	149	DATA_TYPE ret; \
cfec3885 EC	150	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \
	151	false, \
	152	ATOMIC_MMU_IDX); \
d071f4cd	153	\
cfec3885	154	atomic_trace_rmw_pre(env, addr, info); \
d071f4cd	155	ret = atomic_##X(haddr, val); \
ec603b55	156	ATOMIC_MMU_CLEANUP; \
cfec3885	157	atomic_trace_rmw_post(env, addr, info); \
ec603b55 RH	158	return ret; \
ec603b55 RH	159	}
c482cb11 RH	160
	161	GEN_ATOMIC_HELPER(fetch_add)
	162	GEN_ATOMIC_HELPER(fetch_and)
	163	GEN_ATOMIC_HELPER(fetch_or)
	164	GEN_ATOMIC_HELPER(fetch_xor)
	165	GEN_ATOMIC_HELPER(add_fetch)
	166	GEN_ATOMIC_HELPER(and_fetch)
	167	GEN_ATOMIC_HELPER(or_fetch)
	168	GEN_ATOMIC_HELPER(xor_fetch)
	169
	170	#undef GEN_ATOMIC_HELPER
5507c2bf RH	171
	172	/* These helpers are, as a whole, full barriers. Within the helper,
	173	* the leading barrier is explicit and the trailing barrier is within
	174	* cmpxchg primitive.
d071f4cd EC	175	*
	176	* Trace this load + RMW loop as a single RMW op. This way, regardless
	177	* of CF_PARALLEL's value, we'll trace just a read and a write.
5507c2bf RH	178	*/
	179	#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
	180	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	181	ABI_TYPE xval EXTRA_ARGS) \
	182	{ \
	183	ATOMIC_MMU_DECLS; \
	184	XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	185	XDATA_TYPE cmp, old, new, val = xval; \
cfec3885 EC	186	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \
	187	false, \
	188	ATOMIC_MMU_IDX); \
d071f4cd	189	\
cfec3885	190	atomic_trace_rmw_pre(env, addr, info); \
5507c2bf RH	191	smp_mb(); \
	192	cmp = atomic_read__nocheck(haddr); \
	193	do { \
	194	old = cmp; new = FN(old, val); \
	195	cmp = atomic_cmpxchg__nocheck(haddr, old, new); \
	196	} while (cmp != old); \
	197	ATOMIC_MMU_CLEANUP; \
cfec3885	198	atomic_trace_rmw_post(env, addr, info); \
5507c2bf RH	199	return RET; \
	200	}
	201
	202	GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
	203	GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
	204	GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
	205	GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
	206
	207	GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
	208	GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
	209	GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
	210	GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
	211
	212	#undef GEN_ATOMIC_HELPER_FN
7ebee43e RH	213	#endif /* DATA SIZE >= 16 */
7ebee43e RH	214
c482cb11	215	#undef END
d071f4cd	216	#undef MEND
c482cb11 RH	217
	218	#if DATA_SIZE > 1
	219
	220	/* Define reverse-host-endian atomic operations. Note that END is used
	221	within the ATOMIC_NAME macro. */
	222	#ifdef HOST_WORDS_BIGENDIAN
	223	# define END _le
d071f4cd	224	# define MEND _le
c482cb11 RH	225	#else
c482cb11 RH	226	# define END _be
d071f4cd	227	# define MEND _be
c482cb11 RH	228	#endif
	229
	230	ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, target_ulong addr,
	231	ABI_TYPE cmpv, ABI_TYPE newv EXTRA_ARGS)
	232	{
34d49937	233	ATOMIC_MMU_DECLS;
c482cb11	234	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd	235	DATA_TYPE ret;
cfec3885 EC	236	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
	237	false,
	238	ATOMIC_MMU_IDX);
d071f4cd	239
cfec3885	240	atomic_trace_rmw_pre(env, addr, info);
e6cd4bb5 RH	241	#if DATA_SIZE == 16
	242	ret = atomic16_cmpxchg(haddr, BSWAP(cmpv), BSWAP(newv));
	243	#else
d071f4cd	244	ret = atomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
e6cd4bb5	245	#endif
ec603b55	246	ATOMIC_MMU_CLEANUP;
cfec3885	247	atomic_trace_rmw_post(env, addr, info);
ec603b55	248	return BSWAP(ret);
c482cb11 RH	249	}
c482cb11 RH	250
7ebee43e	251	#if DATA_SIZE >= 16
e6cd4bb5	252	#if HAVE_ATOMIC128
7ebee43e RH	253	ABI_TYPE ATOMIC_NAME(ld)(CPUArchState *env, target_ulong addr EXTRA_ARGS)
7ebee43e RH	254	{
34d49937	255	ATOMIC_MMU_DECLS;
7ebee43e	256	DATA_TYPE val, *haddr = ATOMIC_MMU_LOOKUP;
cfec3885 EC	257	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
	258	false,
	259	ATOMIC_MMU_IDX);
d071f4cd	260
cfec3885	261	atomic_trace_ld_pre(env, addr, info);
e6cd4bb5	262	val = atomic16_read(haddr);
ec603b55	263	ATOMIC_MMU_CLEANUP;
cfec3885	264	atomic_trace_ld_post(env, addr, info);
7ebee43e RH	265	return BSWAP(val);
	266	}
	267
	268	void ATOMIC_NAME(st)(CPUArchState *env, target_ulong addr,
	269	ABI_TYPE val EXTRA_ARGS)
	270	{
34d49937	271	ATOMIC_MMU_DECLS;
7ebee43e	272	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
cfec3885 EC	273	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
	274	true,
	275	ATOMIC_MMU_IDX);
d071f4cd	276
cfec3885 EC	277	val = BSWAP(val);
cfec3885 EC	278	atomic_trace_st_pre(env, addr, info);
7ebee43e	279	val = BSWAP(val);
e6cd4bb5	280	atomic16_set(haddr, val);
ec603b55	281	ATOMIC_MMU_CLEANUP;
cfec3885	282	atomic_trace_st_post(env, addr, info);
7ebee43e	283	}
e6cd4bb5	284	#endif
7ebee43e	285	#else
c482cb11 RH	286	ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, target_ulong addr,
	287	ABI_TYPE val EXTRA_ARGS)
	288	{
34d49937	289	ATOMIC_MMU_DECLS;
c482cb11	290	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP;
d071f4cd	291	ABI_TYPE ret;
cfec3885 EC	292	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT,
	293	false,
	294	ATOMIC_MMU_IDX);
d071f4cd	295
cfec3885	296	atomic_trace_rmw_pre(env, addr, info);
d071f4cd	297	ret = atomic_xchg__nocheck(haddr, BSWAP(val));
ec603b55	298	ATOMIC_MMU_CLEANUP;
cfec3885	299	atomic_trace_rmw_post(env, addr, info);
ec603b55	300	return BSWAP(ret);
c482cb11 RH	301	}
	302
	303	#define GEN_ATOMIC_HELPER(X) \
	304	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
358f6348	305	ABI_TYPE val EXTRA_ARGS) \
c482cb11	306	{ \
34d49937	307	ATOMIC_MMU_DECLS; \
c482cb11	308	DATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
d071f4cd	309	DATA_TYPE ret; \
cfec3885 EC	310	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \
	311	false, \
	312	ATOMIC_MMU_IDX); \
d071f4cd	313	\
cfec3885	314	atomic_trace_rmw_pre(env, addr, info); \
d071f4cd	315	ret = atomic_##X(haddr, BSWAP(val)); \
ec603b55	316	ATOMIC_MMU_CLEANUP; \
cfec3885	317	atomic_trace_rmw_post(env, addr, info); \
ec603b55	318	return BSWAP(ret); \
c482cb11 RH	319	}
	320
	321	GEN_ATOMIC_HELPER(fetch_and)
	322	GEN_ATOMIC_HELPER(fetch_or)
	323	GEN_ATOMIC_HELPER(fetch_xor)
	324	GEN_ATOMIC_HELPER(and_fetch)
	325	GEN_ATOMIC_HELPER(or_fetch)
	326	GEN_ATOMIC_HELPER(xor_fetch)
	327
	328	#undef GEN_ATOMIC_HELPER
	329
5507c2bf RH	330	/* These helpers are, as a whole, full barriers. Within the helper,
	331	* the leading barrier is explicit and the trailing barrier is within
	332	* cmpxchg primitive.
d071f4cd EC	333	*
	334	* Trace this load + RMW loop as a single RMW op. This way, regardless
	335	* of CF_PARALLEL's value, we'll trace just a read and a write.
5507c2bf RH	336	*/
	337	#define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET) \
	338	ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, target_ulong addr, \
	339	ABI_TYPE xval EXTRA_ARGS) \
	340	{ \
	341	ATOMIC_MMU_DECLS; \
	342	XDATA_TYPE *haddr = ATOMIC_MMU_LOOKUP; \
	343	XDATA_TYPE ldo, ldn, old, new, val = xval; \
cfec3885 EC	344	uint16_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, \
	345	false, \
	346	ATOMIC_MMU_IDX); \
d071f4cd	347	\
cfec3885	348	atomic_trace_rmw_pre(env, addr, info); \
5507c2bf RH	349	smp_mb(); \
	350	ldn = atomic_read__nocheck(haddr); \
	351	do { \
	352	ldo = ldn; old = BSWAP(ldo); new = FN(old, val); \
	353	ldn = atomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new)); \
	354	} while (ldo != ldn); \
	355	ATOMIC_MMU_CLEANUP; \
cfec3885	356	atomic_trace_rmw_post(env, addr, info); \
5507c2bf RH	357	return RET; \
	358	}
	359
	360	GEN_ATOMIC_HELPER_FN(fetch_smin, MIN, SDATA_TYPE, old)
	361	GEN_ATOMIC_HELPER_FN(fetch_umin, MIN, DATA_TYPE, old)
	362	GEN_ATOMIC_HELPER_FN(fetch_smax, MAX, SDATA_TYPE, old)
	363	GEN_ATOMIC_HELPER_FN(fetch_umax, MAX, DATA_TYPE, old)
	364
	365	GEN_ATOMIC_HELPER_FN(smin_fetch, MIN, SDATA_TYPE, new)
	366	GEN_ATOMIC_HELPER_FN(umin_fetch, MIN, DATA_TYPE, new)
	367	GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
	368	GEN_ATOMIC_HELPER_FN(umax_fetch, MAX, DATA_TYPE, new)
	369
58edf9ee RH	370	/* Note that for addition, we need to use a separate cmpxchg loop instead
	371	of bswaps for the reverse-host-endian helpers. */
	372	#define ADD(X, Y) (X + Y)
	373	GEN_ATOMIC_HELPER_FN(fetch_add, ADD, DATA_TYPE, old)
	374	GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
	375	#undef ADD
	376
5507c2bf	377	#undef GEN_ATOMIC_HELPER_FN
7ebee43e	378	#endif /* DATA_SIZE >= 16 */
c482cb11 RH	379
c482cb11 RH	380	#undef END
d071f4cd	381	#undef MEND
c482cb11 RH	382	#endif /* DATA_SIZE > 1 */
	383
	384	#undef BSWAP
	385	#undef ABI_TYPE
	386	#undef DATA_TYPE
5507c2bf	387	#undef SDATA_TYPE
c482cb11 RH	388	#undef SUFFIX
c482cb11 RH	389	#undef DATA_SIZE
d071f4cd	390	#undef SHIFT