[qemu.git] / memory_ldst.inc.c

/*
 *  Physical memory access templates
 *
 *  Copyright (c) 2003 Fabrice Bellard
 *  Copyright (c) 2015 Linaro, Inc.
 *  Copyright (c) 2016 Red Hat, Inc.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

/* warning: addr must be aligned */
static inline uint32_t glue(address_space_ldl_internal, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result,
    enum device_endian endian)
{
    uint8_t *ptr;
    uint64_t val;
    MemoryRegion *mr;
    hwaddr l = 4;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    RCU_READ_LOCK();
    mr = TRANSLATE(addr, &addr1, &l, false);
    if (l < 4 || !IS_DIRECT(mr, false)) {
        release_lock |= prepare_mmio_access(mr);

        /* I/O case */
        r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap32(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap32(val);
        }
#endif
    } else {
        /* RAM case */
        ptr = MAP_RAM(mr, addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            val = ldl_le_p(ptr);
            break;
        case DEVICE_BIG_ENDIAN:
            val = ldl_be_p(ptr);
            break;
        default:
            val = ldl_p(ptr);
            break;
        }
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    RCU_READ_UNLOCK();
    return val;
}

uint32_t glue(address_space_ldl, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
{
    return glue(address_space_ldl_internal, SUFFIX)(ARG1, addr, attrs, result,
                                                    DEVICE_NATIVE_ENDIAN);
}

uint32_t glue(address_space_ldl_le, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
{
    return glue(address_space_ldl_internal, SUFFIX)(ARG1, addr, attrs, result,
                                                    DEVICE_LITTLE_ENDIAN);
}

uint32_t glue(address_space_ldl_be, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
{
    return glue(address_space_ldl_internal, SUFFIX)(ARG1, addr, attrs, result,
                                                    DEVICE_BIG_ENDIAN);
}

/* warning: addr must be aligned */
static inline uint64_t glue(address_space_ldq_internal, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result,
    enum device_endian endian)
{
    uint8_t *ptr;
    uint64_t val;
    MemoryRegion *mr;
    hwaddr l = 8;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    RCU_READ_LOCK();
    mr = TRANSLATE(addr, &addr1, &l, false);
    if (l < 8 || !IS_DIRECT(mr, false)) {
        release_lock |= prepare_mmio_access(mr);

        /* I/O case */
        r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap64(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap64(val);
        }
#endif
    } else {
        /* RAM case */
        ptr = MAP_RAM(mr, addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            val = ldq_le_p(ptr);
            break;
        case DEVICE_BIG_ENDIAN:
            val = ldq_be_p(ptr);
            break;
        default:
            val = ldq_p(ptr);
            break;
        }
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    RCU_READ_UNLOCK();
    return val;
}

uint64_t glue(address_space_ldq, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
{
    return glue(address_space_ldq_internal, SUFFIX)(ARG1, addr, attrs, result,
                                                    DEVICE_NATIVE_ENDIAN);
}

uint64_t glue(address_space_ldq_le, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
{
    return glue(address_space_ldq_internal, SUFFIX)(ARG1, addr, attrs, result,
                                                    DEVICE_LITTLE_ENDIAN);
}

uint64_t glue(address_space_ldq_be, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
{
    return glue(address_space_ldq_internal, SUFFIX)(ARG1, addr, attrs, result,
                                                    DEVICE_BIG_ENDIAN);
}

uint32_t glue(address_space_ldub, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
{
    uint8_t *ptr;
    uint64_t val;
    MemoryRegion *mr;
    hwaddr l = 1;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    RCU_READ_LOCK();
    mr = TRANSLATE(addr, &addr1, &l, false);
    if (!IS_DIRECT(mr, false)) {
        release_lock |= prepare_mmio_access(mr);

        /* I/O case */
        r = memory_region_dispatch_read(mr, addr1, &val, 1, attrs);
    } else {
        /* RAM case */
        ptr = MAP_RAM(mr, addr1);
        val = ldub_p(ptr);
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    RCU_READ_UNLOCK();
    return val;
}

/* warning: addr must be aligned */
static inline uint32_t glue(address_space_lduw_internal, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result,
    enum device_endian endian)
{
    uint8_t *ptr;
    uint64_t val;
    MemoryRegion *mr;
    hwaddr l = 2;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    RCU_READ_LOCK();
    mr = TRANSLATE(addr, &addr1, &l, false);
    if (l < 2 || !IS_DIRECT(mr, false)) {
        release_lock |= prepare_mmio_access(mr);

        /* I/O case */
        r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap16(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap16(val);
        }
#endif
    } else {
        /* RAM case */
        ptr = MAP_RAM(mr, addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            val = lduw_le_p(ptr);
            break;
        case DEVICE_BIG_ENDIAN:
            val = lduw_be_p(ptr);
            break;
        default:
            val = lduw_p(ptr);
            break;
        }
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    RCU_READ_UNLOCK();
    return val;
}

uint32_t glue(address_space_lduw, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
{
    return glue(address_space_lduw_internal, SUFFIX)(ARG1, addr, attrs, result,
                                                     DEVICE_NATIVE_ENDIAN);
}

uint32_t glue(address_space_lduw_le, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
{
    return glue(address_space_lduw_internal, SUFFIX)(ARG1, addr, attrs, result,
                                                     DEVICE_LITTLE_ENDIAN);
}

uint32_t glue(address_space_lduw_be, SUFFIX)(ARG1_DECL,
    hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
{
    return glue(address_space_lduw_internal, SUFFIX)(ARG1, addr, attrs, result,
                                       DEVICE_BIG_ENDIAN);
}

/* warning: addr must be aligned. The ram page is not masked as dirty
   and the code inside is not invalidated. It is useful if the dirty
   bits are used to track modified PTEs */
void glue(address_space_stl_notdirty, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
{
    uint8_t *ptr;
    MemoryRegion *mr;
    hwaddr l = 4;
    hwaddr addr1;
    MemTxResult r;
    uint8_t dirty_log_mask;
    bool release_lock = false;

    RCU_READ_LOCK();
    mr = TRANSLATE(addr, &addr1, &l, true);
    if (l < 4 || !IS_DIRECT(mr, true)) {
        release_lock |= prepare_mmio_access(mr);

        r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
    } else {
        ptr = MAP_RAM(mr, addr1);
        stl_p(ptr, val);

        dirty_log_mask = memory_region_get_dirty_log_mask(mr);
        dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
        cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
                                            4, dirty_log_mask);
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    RCU_READ_UNLOCK();
}

/* warning: addr must be aligned */
static inline void glue(address_space_stl_internal, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint32_t val, MemTxAttrs attrs,
    MemTxResult *result, enum device_endian endian)
{
    uint8_t *ptr;
    MemoryRegion *mr;
    hwaddr l = 4;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    RCU_READ_LOCK();
    mr = TRANSLATE(addr, &addr1, &l, true);
    if (l < 4 || !IS_DIRECT(mr, true)) {
        release_lock |= prepare_mmio_access(mr);

#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap32(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap32(val);
        }
#endif
        r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
    } else {
        /* RAM case */
        ptr = MAP_RAM(mr, addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            stl_le_p(ptr, val);
            break;
        case DEVICE_BIG_ENDIAN:
            stl_be_p(ptr, val);
            break;
        default:
            stl_p(ptr, val);
            break;
        }
        INVALIDATE(mr, addr1, 4);
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    RCU_READ_UNLOCK();
}

void glue(address_space_stl, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
{
    glue(address_space_stl_internal, SUFFIX)(ARG1, addr, val, attrs,
                                             result, DEVICE_NATIVE_ENDIAN);
}

void glue(address_space_stl_le, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
{
    glue(address_space_stl_internal, SUFFIX)(ARG1, addr, val, attrs,
                                             result, DEVICE_LITTLE_ENDIAN);
}

void glue(address_space_stl_be, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
{
    glue(address_space_stl_internal, SUFFIX)(ARG1, addr, val, attrs,
                                             result, DEVICE_BIG_ENDIAN);
}

void glue(address_space_stb, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
{
    uint8_t *ptr;
    MemoryRegion *mr;
    hwaddr l = 1;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    RCU_READ_LOCK();
    mr = TRANSLATE(addr, &addr1, &l, true);
    if (!IS_DIRECT(mr, true)) {
        release_lock |= prepare_mmio_access(mr);
        r = memory_region_dispatch_write(mr, addr1, val, 1, attrs);
    } else {
        /* RAM case */
        ptr = MAP_RAM(mr, addr1);
        stb_p(ptr, val);
        INVALIDATE(mr, addr1, 1);
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    RCU_READ_UNLOCK();
}

/* warning: addr must be aligned */
static inline void glue(address_space_stw_internal, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint32_t val, MemTxAttrs attrs,
    MemTxResult *result, enum device_endian endian)
{
    uint8_t *ptr;
    MemoryRegion *mr;
    hwaddr l = 2;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    RCU_READ_LOCK();
    mr = TRANSLATE(addr, &addr1, &l, true);
    if (l < 2 || !IS_DIRECT(mr, true)) {
        release_lock |= prepare_mmio_access(mr);

#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap16(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap16(val);
        }
#endif
        r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
    } else {
        /* RAM case */
        ptr = MAP_RAM(mr, addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            stw_le_p(ptr, val);
            break;
        case DEVICE_BIG_ENDIAN:
            stw_be_p(ptr, val);
            break;
        default:
            stw_p(ptr, val);
            break;
        }
        INVALIDATE(mr, addr1, 2);
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    RCU_READ_UNLOCK();
}

void glue(address_space_stw, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
{
    glue(address_space_stw_internal, SUFFIX)(ARG1, addr, val, attrs, result,
                                             DEVICE_NATIVE_ENDIAN);
}

void glue(address_space_stw_le, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
{
    glue(address_space_stw_internal, SUFFIX)(ARG1, addr, val, attrs, result,
                                             DEVICE_LITTLE_ENDIAN);
}

void glue(address_space_stw_be, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
{
    glue(address_space_stw_internal, SUFFIX)(ARG1, addr, val, attrs, result,
                               DEVICE_BIG_ENDIAN);
}

static void glue(address_space_stq_internal, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint64_t val, MemTxAttrs attrs,
    MemTxResult *result, enum device_endian endian)
{
    uint8_t *ptr;
    MemoryRegion *mr;
    hwaddr l = 8;
    hwaddr addr1;
    MemTxResult r;
    bool release_lock = false;

    RCU_READ_LOCK();
    mr = TRANSLATE(addr, &addr1, &l, true);
    if (l < 8 || !IS_DIRECT(mr, true)) {
        release_lock |= prepare_mmio_access(mr);

#if defined(TARGET_WORDS_BIGENDIAN)
        if (endian == DEVICE_LITTLE_ENDIAN) {
            val = bswap64(val);
        }
#else
        if (endian == DEVICE_BIG_ENDIAN) {
            val = bswap64(val);
        }
#endif
        r = memory_region_dispatch_write(mr, addr1, val, 8, attrs);
    } else {
        /* RAM case */
        ptr = MAP_RAM(mr, addr1);
        switch (endian) {
        case DEVICE_LITTLE_ENDIAN:
            stq_le_p(ptr, val);
            break;
        case DEVICE_BIG_ENDIAN:
            stq_be_p(ptr, val);
            break;
        default:
            stq_p(ptr, val);
            break;
        }
        INVALIDATE(mr, addr1, 8);
        r = MEMTX_OK;
    }
    if (result) {
        *result = r;
    }
    if (release_lock) {
        qemu_mutex_unlock_iothread();
    }
    RCU_READ_UNLOCK();
}

void glue(address_space_stq, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint64_t val, MemTxAttrs attrs, MemTxResult *result)
{
    glue(address_space_stq_internal, SUFFIX)(ARG1, addr, val, attrs, result,
                                             DEVICE_NATIVE_ENDIAN);
}

void glue(address_space_stq_le, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint64_t val, MemTxAttrs attrs, MemTxResult *result)
{
    glue(address_space_stq_internal, SUFFIX)(ARG1, addr, val, attrs, result,
                                             DEVICE_LITTLE_ENDIAN);
}

void glue(address_space_stq_be, SUFFIX)(ARG1_DECL,
    hwaddr addr, uint64_t val, MemTxAttrs attrs, MemTxResult *result)
{
    glue(address_space_stq_internal, SUFFIX)(ARG1, addr, val, attrs, result,
                                             DEVICE_BIG_ENDIAN);
}

#undef ARG1_DECL
#undef ARG1
#undef SUFFIX
#undef TRANSLATE
#undef IS_DIRECT
#undef MAP_RAM
#undef INVALIDATE
#undef RCU_READ_LOCK
#undef RCU_READ_UNLOCK
Commit	Line	Data
0ce265ff PB	1	/*
	2	* Physical memory access templates
	3	*
	4	* Copyright (c) 2003 Fabrice Bellard
	5	* Copyright (c) 2015 Linaro, Inc.
	6	* Copyright (c) 2016 Red Hat, Inc.
	7	*
	8	* This library is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU Lesser General Public
	10	* License as published by the Free Software Foundation; either
	11	* version 2 of the License, or (at your option) any later version.
	12	*
	13	* This library is distributed in the hope that it will be useful,
	14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* Lesser General Public License for more details.
	17	*
	18	* You should have received a copy of the GNU Lesser General Public
	19	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
	20	*/
	21
	22	/* warning: addr must be aligned */
	23	static inline uint32_t glue(address_space_ldl_internal, SUFFIX)(ARG1_DECL,
	24	hwaddr addr, MemTxAttrs attrs, MemTxResult *result,
	25	enum device_endian endian)
	26	{
	27	uint8_t *ptr;
	28	uint64_t val;
	29	MemoryRegion *mr;
	30	hwaddr l = 4;
	31	hwaddr addr1;
	32	MemTxResult r;
	33	bool release_lock = false;
	34
	35	RCU_READ_LOCK();
	36	mr = TRANSLATE(addr, &addr1, &l, false);
	37	if (l < 4 \|\| !IS_DIRECT(mr, false)) {
	38	release_lock \|= prepare_mmio_access(mr);
	39
	40	/* I/O case */
	41	r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
	42	#if defined(TARGET_WORDS_BIGENDIAN)
	43	if (endian == DEVICE_LITTLE_ENDIAN) {
	44	val = bswap32(val);
	45	}
	46	#else
	47	if (endian == DEVICE_BIG_ENDIAN) {
	48	val = bswap32(val);
	49	}
	50	#endif
	51	} else {
	52	/* RAM case */
	53	ptr = MAP_RAM(mr, addr1);
	54	switch (endian) {
	55	case DEVICE_LITTLE_ENDIAN:
	56	val = ldl_le_p(ptr);
	57	break;
	58	case DEVICE_BIG_ENDIAN:
	59	val = ldl_be_p(ptr);
	60	break;
	61	default:
	62	val = ldl_p(ptr);
	63	break;
	64	}
65	r = MEMTX_OK;
66	}
67	if (result) {
68	*result = r;
69	}
70	if (release_lock) {
71	qemu_mutex_unlock_iothread();
72	}
73	RCU_READ_UNLOCK();
74	return val;
75	}
76
77	uint32_t glue(address_space_ldl, SUFFIX)(ARG1_DECL,
78	hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
79	{
80	return glue(address_space_ldl_internal, SUFFIX)(ARG1, addr, attrs, result,
81	DEVICE_NATIVE_ENDIAN);
82	}
83
84	uint32_t glue(address_space_ldl_le, SUFFIX)(ARG1_DECL,
85	hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
86	{
87	return glue(address_space_ldl_internal, SUFFIX)(ARG1, addr, attrs, result,
88	DEVICE_LITTLE_ENDIAN);
89	}
90
91	uint32_t glue(address_space_ldl_be, SUFFIX)(ARG1_DECL,
92	hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
93	{
94	return glue(address_space_ldl_internal, SUFFIX)(ARG1, addr, attrs, result,
95	DEVICE_BIG_ENDIAN);
96	}
97
0ce265ff PB	98	/* warning: addr must be aligned */
	99	static inline uint64_t glue(address_space_ldq_internal, SUFFIX)(ARG1_DECL,
	100	hwaddr addr, MemTxAttrs attrs, MemTxResult *result,
	101	enum device_endian endian)
	102	{
	103	uint8_t *ptr;
	104	uint64_t val;
	105	MemoryRegion *mr;
	106	hwaddr l = 8;
	107	hwaddr addr1;
	108	MemTxResult r;
	109	bool release_lock = false;
	110
	111	RCU_READ_LOCK();
	112	mr = TRANSLATE(addr, &addr1, &l, false);
	113	if (l < 8 \|\| !IS_DIRECT(mr, false)) {
	114	release_lock \|= prepare_mmio_access(mr);
	115
	116	/* I/O case */
	117	r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
	118	#if defined(TARGET_WORDS_BIGENDIAN)
	119	if (endian == DEVICE_LITTLE_ENDIAN) {
	120	val = bswap64(val);
	121	}
	122	#else
	123	if (endian == DEVICE_BIG_ENDIAN) {
	124	val = bswap64(val);
	125	}
	126	#endif
	127	} else {
	128	/* RAM case */
	129	ptr = MAP_RAM(mr, addr1);
	130	switch (endian) {
	131	case DEVICE_LITTLE_ENDIAN:
	132	val = ldq_le_p(ptr);
	133	break;
	134	case DEVICE_BIG_ENDIAN:
	135	val = ldq_be_p(ptr);
	136	break;
	137	default:
	138	val = ldq_p(ptr);
	139	break;
	140	}
	141	r = MEMTX_OK;
	142	}
	143	if (result) {
	144	*result = r;
	145	}
	146	if (release_lock) {
	147	qemu_mutex_unlock_iothread();
	148	}
	149	RCU_READ_UNLOCK();
	150	return val;
	151	}
	152
	153	uint64_t glue(address_space_ldq, SUFFIX)(ARG1_DECL,
	154	hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
	155	{
	156	return glue(address_space_ldq_internal, SUFFIX)(ARG1, addr, attrs, result,
	157	DEVICE_NATIVE_ENDIAN);
	158	}
	159
	160	uint64_t glue(address_space_ldq_le, SUFFIX)(ARG1_DECL,
	161	hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
162	{
163	return glue(address_space_ldq_internal, SUFFIX)(ARG1, addr, attrs, result,
164	DEVICE_LITTLE_ENDIAN);
165	}
166
167	uint64_t glue(address_space_ldq_be, SUFFIX)(ARG1_DECL,
168	hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
169	{
170	return glue(address_space_ldq_internal, SUFFIX)(ARG1, addr, attrs, result,
171	DEVICE_BIG_ENDIAN);
172	}
173
0ce265ff PB	174	uint32_t glue(address_space_ldub, SUFFIX)(ARG1_DECL,
	175	hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
	176	{
	177	uint8_t *ptr;
	178	uint64_t val;
	179	MemoryRegion *mr;
	180	hwaddr l = 1;
	181	hwaddr addr1;
	182	MemTxResult r;
	183	bool release_lock = false;
	184
	185	RCU_READ_LOCK();
	186	mr = TRANSLATE(addr, &addr1, &l, false);
	187	if (!IS_DIRECT(mr, false)) {
	188	release_lock \|= prepare_mmio_access(mr);
	189
	190	/* I/O case */
	191	r = memory_region_dispatch_read(mr, addr1, &val, 1, attrs);
	192	} else {
	193	/* RAM case */
	194	ptr = MAP_RAM(mr, addr1);
	195	val = ldub_p(ptr);
	196	r = MEMTX_OK;
	197	}
	198	if (result) {
	199	*result = r;
	200	}
	201	if (release_lock) {
	202	qemu_mutex_unlock_iothread();
	203	}
	204	RCU_READ_UNLOCK();
	205	return val;
	206	}
	207
0ce265ff PB	208	/* warning: addr must be aligned */
	209	static inline uint32_t glue(address_space_lduw_internal, SUFFIX)(ARG1_DECL,
	210	hwaddr addr, MemTxAttrs attrs, MemTxResult *result,
	211	enum device_endian endian)
	212	{
	213	uint8_t *ptr;
	214	uint64_t val;
	215	MemoryRegion *mr;
	216	hwaddr l = 2;
	217	hwaddr addr1;
	218	MemTxResult r;
	219	bool release_lock = false;
	220
	221	RCU_READ_LOCK();
	222	mr = TRANSLATE(addr, &addr1, &l, false);
	223	if (l < 2 \|\| !IS_DIRECT(mr, false)) {
	224	release_lock \|= prepare_mmio_access(mr);
	225
	226	/* I/O case */
	227	r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
	228	#if defined(TARGET_WORDS_BIGENDIAN)
	229	if (endian == DEVICE_LITTLE_ENDIAN) {
	230	val = bswap16(val);
	231	}
	232	#else
	233	if (endian == DEVICE_BIG_ENDIAN) {
	234	val = bswap16(val);
	235	}
	236	#endif
	237	} else {
	238	/* RAM case */
	239	ptr = MAP_RAM(mr, addr1);
	240	switch (endian) {
	241	case DEVICE_LITTLE_ENDIAN:
	242	val = lduw_le_p(ptr);
	243	break;
	244	case DEVICE_BIG_ENDIAN:
	245	val = lduw_be_p(ptr);
	246	break;
	247	default:
	248	val = lduw_p(ptr);
	249	break;
	250	}
	251	r = MEMTX_OK;
	252	}
	253	if (result) {
	254	*result = r;
	255	}
	256	if (release_lock) {
	257	qemu_mutex_unlock_iothread();
	258	}
	259	RCU_READ_UNLOCK();
	260	return val;
	261	}
	262
	263	uint32_t glue(address_space_lduw, SUFFIX)(ARG1_DECL,
	264	hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
	265	{
	266	return glue(address_space_lduw_internal, SUFFIX)(ARG1, addr, attrs, result,
	267	DEVICE_NATIVE_ENDIAN);
	268	}
	269
	270	uint32_t glue(address_space_lduw_le, SUFFIX)(ARG1_DECL,
	271	hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
272	{
273	return glue(address_space_lduw_internal, SUFFIX)(ARG1, addr, attrs, result,
274	DEVICE_LITTLE_ENDIAN);
275	}
276
277	uint32_t glue(address_space_lduw_be, SUFFIX)(ARG1_DECL,
278	hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
279	{
280	return glue(address_space_lduw_internal, SUFFIX)(ARG1, addr, attrs, result,
281	DEVICE_BIG_ENDIAN);
282	}
283
0ce265ff PB	284	/* warning: addr must be aligned. The ram page is not masked as dirty
	285	and the code inside is not invalidated. It is useful if the dirty
	286	bits are used to track modified PTEs */
	287	void glue(address_space_stl_notdirty, SUFFIX)(ARG1_DECL,
	288	hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
	289	{
	290	uint8_t *ptr;
	291	MemoryRegion *mr;
	292	hwaddr l = 4;
	293	hwaddr addr1;
	294	MemTxResult r;
	295	uint8_t dirty_log_mask;
	296	bool release_lock = false;
	297
	298	RCU_READ_LOCK();
	299	mr = TRANSLATE(addr, &addr1, &l, true);
	300	if (l < 4 \|\| !IS_DIRECT(mr, true)) {
	301	release_lock \|= prepare_mmio_access(mr);
	302
	303	r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
	304	} else {
	305	ptr = MAP_RAM(mr, addr1);
	306	stl_p(ptr, val);
	307
	308	dirty_log_mask = memory_region_get_dirty_log_mask(mr);
	309	dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
	310	cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
	311	4, dirty_log_mask);
	312	r = MEMTX_OK;
	313	}
	314	if (result) {
	315	*result = r;
	316	}
	317	if (release_lock) {
	318	qemu_mutex_unlock_iothread();
	319	}
	320	RCU_READ_UNLOCK();
	321	}
	322
0ce265ff PB	323	/* warning: addr must be aligned */
	324	static inline void glue(address_space_stl_internal, SUFFIX)(ARG1_DECL,
	325	hwaddr addr, uint32_t val, MemTxAttrs attrs,
	326	MemTxResult *result, enum device_endian endian)
	327	{
	328	uint8_t *ptr;
	329	MemoryRegion *mr;
	330	hwaddr l = 4;
	331	hwaddr addr1;
	332	MemTxResult r;
	333	bool release_lock = false;
	334
	335	RCU_READ_LOCK();
	336	mr = TRANSLATE(addr, &addr1, &l, true);
	337	if (l < 4 \|\| !IS_DIRECT(mr, true)) {
	338	release_lock \|= prepare_mmio_access(mr);
	339
	340	#if defined(TARGET_WORDS_BIGENDIAN)
	341	if (endian == DEVICE_LITTLE_ENDIAN) {
	342	val = bswap32(val);
	343	}
	344	#else
	345	if (endian == DEVICE_BIG_ENDIAN) {
	346	val = bswap32(val);
	347	}
	348	#endif
	349	r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
	350	} else {
	351	/* RAM case */
	352	ptr = MAP_RAM(mr, addr1);
	353	switch (endian) {
	354	case DEVICE_LITTLE_ENDIAN:
	355	stl_le_p(ptr, val);
	356	break;
	357	case DEVICE_BIG_ENDIAN:
	358	stl_be_p(ptr, val);
	359	break;
	360	default:
	361	stl_p(ptr, val);
	362	break;
	363	}
	364	INVALIDATE(mr, addr1, 4);
	365	r = MEMTX_OK;
	366	}
	367	if (result) {
	368	*result = r;
	369	}
	370	if (release_lock) {
	371	qemu_mutex_unlock_iothread();
	372	}
	373	RCU_READ_UNLOCK();
	374	}
	375
	376	void glue(address_space_stl, SUFFIX)(ARG1_DECL,
	377	hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
	378	{
	379	glue(address_space_stl_internal, SUFFIX)(ARG1, addr, val, attrs,
	380	result, DEVICE_NATIVE_ENDIAN);
	381	}
	382
	383	void glue(address_space_stl_le, SUFFIX)(ARG1_DECL,
	384	hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
	385	{
	386	glue(address_space_stl_internal, SUFFIX)(ARG1, addr, val, attrs,
387	result, DEVICE_LITTLE_ENDIAN);
388	}
389
390	void glue(address_space_stl_be, SUFFIX)(ARG1_DECL,
391	hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
392	{
393	glue(address_space_stl_internal, SUFFIX)(ARG1, addr, val, attrs,
394	result, DEVICE_BIG_ENDIAN);
395	}
396
0ce265ff PB	397	void glue(address_space_stb, SUFFIX)(ARG1_DECL,
	398	hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
	399	{
	400	uint8_t *ptr;
	401	MemoryRegion *mr;
	402	hwaddr l = 1;
	403	hwaddr addr1;
	404	MemTxResult r;
	405	bool release_lock = false;
	406
	407	RCU_READ_LOCK();
	408	mr = TRANSLATE(addr, &addr1, &l, true);
	409	if (!IS_DIRECT(mr, true)) {
	410	release_lock \|= prepare_mmio_access(mr);
	411	r = memory_region_dispatch_write(mr, addr1, val, 1, attrs);
	412	} else {
	413	/* RAM case */
	414	ptr = MAP_RAM(mr, addr1);
	415	stb_p(ptr, val);
	416	INVALIDATE(mr, addr1, 1);
	417	r = MEMTX_OK;
	418	}
	419	if (result) {
	420	*result = r;
	421	}
	422	if (release_lock) {
	423	qemu_mutex_unlock_iothread();
	424	}
	425	RCU_READ_UNLOCK();
	426	}
	427
0ce265ff PB	428	/* warning: addr must be aligned */
	429	static inline void glue(address_space_stw_internal, SUFFIX)(ARG1_DECL,
	430	hwaddr addr, uint32_t val, MemTxAttrs attrs,
	431	MemTxResult *result, enum device_endian endian)
	432	{
	433	uint8_t *ptr;
	434	MemoryRegion *mr;
	435	hwaddr l = 2;
	436	hwaddr addr1;
	437	MemTxResult r;
	438	bool release_lock = false;
	439
	440	RCU_READ_LOCK();
	441	mr = TRANSLATE(addr, &addr1, &l, true);
	442	if (l < 2 \|\| !IS_DIRECT(mr, true)) {
	443	release_lock \|= prepare_mmio_access(mr);
	444
	445	#if defined(TARGET_WORDS_BIGENDIAN)
	446	if (endian == DEVICE_LITTLE_ENDIAN) {
	447	val = bswap16(val);
	448	}
	449	#else
	450	if (endian == DEVICE_BIG_ENDIAN) {
	451	val = bswap16(val);
	452	}
	453	#endif
	454	r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
	455	} else {
	456	/* RAM case */
	457	ptr = MAP_RAM(mr, addr1);
	458	switch (endian) {
	459	case DEVICE_LITTLE_ENDIAN:
	460	stw_le_p(ptr, val);
	461	break;
	462	case DEVICE_BIG_ENDIAN:
	463	stw_be_p(ptr, val);
	464	break;
	465	default:
	466	stw_p(ptr, val);
	467	break;
	468	}
	469	INVALIDATE(mr, addr1, 2);
	470	r = MEMTX_OK;
	471	}
	472	if (result) {
	473	*result = r;
	474	}
	475	if (release_lock) {
	476	qemu_mutex_unlock_iothread();
	477	}
	478	RCU_READ_UNLOCK();
	479	}
	480
	481	void glue(address_space_stw, SUFFIX)(ARG1_DECL,
	482	hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
	483	{
	484	glue(address_space_stw_internal, SUFFIX)(ARG1, addr, val, attrs, result,
	485	DEVICE_NATIVE_ENDIAN);
	486	}
	487
	488	void glue(address_space_stw_le, SUFFIX)(ARG1_DECL,
	489	hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
	490	{
	491	glue(address_space_stw_internal, SUFFIX)(ARG1, addr, val, attrs, result,
492	DEVICE_LITTLE_ENDIAN);
493	}
494
495	void glue(address_space_stw_be, SUFFIX)(ARG1_DECL,
496	hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
497	{
498	glue(address_space_stw_internal, SUFFIX)(ARG1, addr, val, attrs, result,
499	DEVICE_BIG_ENDIAN);
500	}
501
0ce265ff PB	502	static void glue(address_space_stq_internal, SUFFIX)(ARG1_DECL,
	503	hwaddr addr, uint64_t val, MemTxAttrs attrs,
	504	MemTxResult *result, enum device_endian endian)
	505	{
	506	uint8_t *ptr;
	507	MemoryRegion *mr;
	508	hwaddr l = 8;
	509	hwaddr addr1;
	510	MemTxResult r;
	511	bool release_lock = false;
	512
	513	RCU_READ_LOCK();
	514	mr = TRANSLATE(addr, &addr1, &l, true);
	515	if (l < 8 \|\| !IS_DIRECT(mr, true)) {
	516	release_lock \|= prepare_mmio_access(mr);
	517
	518	#if defined(TARGET_WORDS_BIGENDIAN)
	519	if (endian == DEVICE_LITTLE_ENDIAN) {
	520	val = bswap64(val);
	521	}
	522	#else
	523	if (endian == DEVICE_BIG_ENDIAN) {
	524	val = bswap64(val);
	525	}
	526	#endif
	527	r = memory_region_dispatch_write(mr, addr1, val, 8, attrs);
	528	} else {
	529	/* RAM case */
	530	ptr = MAP_RAM(mr, addr1);
	531	switch (endian) {
	532	case DEVICE_LITTLE_ENDIAN:
	533	stq_le_p(ptr, val);
	534	break;
	535	case DEVICE_BIG_ENDIAN:
	536	stq_be_p(ptr, val);
	537	break;
	538	default:
	539	stq_p(ptr, val);
	540	break;
	541	}
	542	INVALIDATE(mr, addr1, 8);
	543	r = MEMTX_OK;
	544	}
	545	if (result) {
	546	*result = r;
	547	}
	548	if (release_lock) {
	549	qemu_mutex_unlock_iothread();
	550	}
	551	RCU_READ_UNLOCK();
	552	}
	553
	554	void glue(address_space_stq, SUFFIX)(ARG1_DECL,
	555	hwaddr addr, uint64_t val, MemTxAttrs attrs, MemTxResult *result)
	556	{
	557	glue(address_space_stq_internal, SUFFIX)(ARG1, addr, val, attrs, result,
	558	DEVICE_NATIVE_ENDIAN);
	559	}
	560
	561	void glue(address_space_stq_le, SUFFIX)(ARG1_DECL,
	562	hwaddr addr, uint64_t val, MemTxAttrs attrs, MemTxResult *result)
	563	{
	564	glue(address_space_stq_internal, SUFFIX)(ARG1, addr, val, attrs, result,
	565	DEVICE_LITTLE_ENDIAN);
566	}
567
568	void glue(address_space_stq_be, SUFFIX)(ARG1_DECL,
569	hwaddr addr, uint64_t val, MemTxAttrs attrs, MemTxResult *result)
570	{
571	glue(address_space_stq_internal, SUFFIX)(ARG1, addr, val, attrs, result,
572	DEVICE_BIG_ENDIAN);
573	}
574
0ce265ff PB	575	#undef ARG1_DECL
	576	#undef ARG1
	577	#undef SUFFIX
	578	#undef TRANSLATE
	579	#undef IS_DIRECT
	580	#undef MAP_RAM
	581	#undef INVALIDATE
	582	#undef RCU_READ_LOCK
	583	#undef RCU_READ_UNLOCK