[qemu.git] / include / exec / softmmu_template.h

/*
 *  Software MMU support
 *
 * Generate helpers used by TCG for qemu_ld/st ops and code load
 * functions.
 *
 * Included from target op helpers and exec.c.
 *
 *  Copyright (c) 2003 Fabrice Bellard
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */
#include "qemu/timer.h"
#include "exec/address-spaces.h"
#include "exec/memory.h"

#define DATA_SIZE (1 << SHIFT)

#if DATA_SIZE == 8
#define SUFFIX q
#define LSUFFIX q
#define SDATA_TYPE  int64_t
#define DATA_TYPE  uint64_t
#elif DATA_SIZE == 4
#define SUFFIX l
#define LSUFFIX l
#define SDATA_TYPE  int32_t
#define DATA_TYPE  uint32_t
#elif DATA_SIZE == 2
#define SUFFIX w
#define LSUFFIX uw
#define SDATA_TYPE  int16_t
#define DATA_TYPE  uint16_t
#elif DATA_SIZE == 1
#define SUFFIX b
#define LSUFFIX ub
#define SDATA_TYPE  int8_t
#define DATA_TYPE  uint8_t
#else
#error unsupported data size
#endif


/* For the benefit of TCG generated code, we want to avoid the complication
   of ABI-specific return type promotion and always return a value extended
   to the register size of the host.  This is tcg_target_long, except in the
   case of a 32-bit host and 64-bit data, and for that we always have
   uint64_t.  Don't bother with this widened value for SOFTMMU_CODE_ACCESS.  */
#if defined(SOFTMMU_CODE_ACCESS) || DATA_SIZE == 8
# define WORD_TYPE  DATA_TYPE
# define USUFFIX    SUFFIX
#else
# define WORD_TYPE  tcg_target_ulong
# define USUFFIX    glue(u, SUFFIX)
# define SSUFFIX    glue(s, SUFFIX)
#endif

#ifdef SOFTMMU_CODE_ACCESS
#define READ_ACCESS_TYPE 2
#define ADDR_READ addr_code
#else
#define READ_ACCESS_TYPE 0
#define ADDR_READ addr_read
#endif

#if DATA_SIZE == 8
# define BSWAP(X)  bswap64(X)
#elif DATA_SIZE == 4
# define BSWAP(X)  bswap32(X)
#elif DATA_SIZE == 2
# define BSWAP(X)  bswap16(X)
#else
# define BSWAP(X)  (X)
#endif

#ifdef TARGET_WORDS_BIGENDIAN
# define TGT_BE(X)  (X)
# define TGT_LE(X)  BSWAP(X)
#else
# define TGT_BE(X)  BSWAP(X)
# define TGT_LE(X)  (X)
#endif

#if DATA_SIZE == 1
# define helper_le_ld_name  glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
# define helper_be_ld_name  helper_le_ld_name
# define helper_le_lds_name glue(glue(helper_ret_ld, SSUFFIX), MMUSUFFIX)
# define helper_be_lds_name helper_le_lds_name
# define helper_le_st_name  glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)
# define helper_be_st_name  helper_le_st_name
#else
# define helper_le_ld_name  glue(glue(helper_le_ld, USUFFIX), MMUSUFFIX)
# define helper_be_ld_name  glue(glue(helper_be_ld, USUFFIX), MMUSUFFIX)
# define helper_le_lds_name glue(glue(helper_le_ld, SSUFFIX), MMUSUFFIX)
# define helper_be_lds_name glue(glue(helper_be_ld, SSUFFIX), MMUSUFFIX)
# define helper_le_st_name  glue(glue(helper_le_st, SUFFIX), MMUSUFFIX)
# define helper_be_st_name  glue(glue(helper_be_st, SUFFIX), MMUSUFFIX)
#endif

#ifdef TARGET_WORDS_BIGENDIAN
# define helper_te_ld_name  helper_be_ld_name
# define helper_te_st_name  helper_be_st_name
#else
# define helper_te_ld_name  helper_le_ld_name
# define helper_te_st_name  helper_le_st_name
#endif

static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
                                              hwaddr physaddr,
                                              target_ulong addr,
                                              uintptr_t retaddr)
{
    uint64_t val;
    CPUState *cpu = ENV_GET_CPU(env);
    MemoryRegion *mr = iotlb_to_region(cpu->as, physaddr);

    physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
    cpu->mem_io_pc = retaddr;
    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu_can_do_io(cpu)) {
        cpu_io_recompile(cpu, retaddr);
    }

    cpu->mem_io_vaddr = addr;
    io_mem_read(mr, physaddr, &val, 1 << SHIFT);
    return val;
}

#ifdef SOFTMMU_CODE_ACCESS
static __attribute__((unused))
#endif
WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
                            uintptr_t retaddr)
{
    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
    uintptr_t haddr;
    DATA_TYPE res;

    /* Adjust the given return address.  */
    retaddr -= GETPC_ADJ;

    /* If the TLB entry is for a different page, reload and try again.  */
    if ((addr & TARGET_PAGE_MASK)
         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
#ifdef ALIGNED_ONLY
        if ((addr & (DATA_SIZE - 1)) != 0) {
            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
        }
#endif
        tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
        tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
    }

    /* Handle an IO access.  */
    if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
        hwaddr ioaddr;
        if ((addr & (DATA_SIZE - 1)) != 0) {
            goto do_unaligned_access;
        }
        ioaddr = env->iotlb[mmu_idx][index];

        /* ??? Note that the io helpers always read data in the target
           byte ordering.  We should push the LE/BE request down into io.  */
        res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
        res = TGT_LE(res);
        return res;
    }

    /* Handle slow unaligned access (it spans two pages or IO).  */
    if (DATA_SIZE > 1
        && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
                    >= TARGET_PAGE_SIZE)) {
        target_ulong addr1, addr2;
        DATA_TYPE res1, res2;
        unsigned shift;
    do_unaligned_access:
#ifdef ALIGNED_ONLY
        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
#endif
        addr1 = addr & ~(DATA_SIZE - 1);
        addr2 = addr1 + DATA_SIZE;
        /* Note the adjustment at the beginning of the function.
           Undo that for the recursion.  */
        res1 = helper_le_ld_name(env, addr1, mmu_idx, retaddr + GETPC_ADJ);
        res2 = helper_le_ld_name(env, addr2, mmu_idx, retaddr + GETPC_ADJ);
        shift = (addr & (DATA_SIZE - 1)) * 8;

        /* Little-endian combine.  */
        res = (res1 >> shift) | (res2 << ((DATA_SIZE * 8) - shift));
        return res;
    }

    /* Handle aligned access or unaligned access in the same page.  */
#ifdef ALIGNED_ONLY
    if ((addr & (DATA_SIZE - 1)) != 0) {
        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
    }
#endif

    haddr = addr + env->tlb_table[mmu_idx][index].addend;
#if DATA_SIZE == 1
    res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
#else
    res = glue(glue(ld, LSUFFIX), _le_p)((uint8_t *)haddr);
#endif
    return res;
}

#if DATA_SIZE > 1
#ifdef SOFTMMU_CODE_ACCESS
static __attribute__((unused))
#endif
WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
                            uintptr_t retaddr)
{
    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
    uintptr_t haddr;
    DATA_TYPE res;

    /* Adjust the given return address.  */
    retaddr -= GETPC_ADJ;

    /* If the TLB entry is for a different page, reload and try again.  */
    if ((addr & TARGET_PAGE_MASK)
         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
#ifdef ALIGNED_ONLY
        if ((addr & (DATA_SIZE - 1)) != 0) {
            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
        }
#endif
        tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
        tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
    }

    /* Handle an IO access.  */
    if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
        hwaddr ioaddr;
        if ((addr & (DATA_SIZE - 1)) != 0) {
            goto do_unaligned_access;
        }
        ioaddr = env->iotlb[mmu_idx][index];

        /* ??? Note that the io helpers always read data in the target
           byte ordering.  We should push the LE/BE request down into io.  */
        res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
        res = TGT_BE(res);
        return res;
    }

    /* Handle slow unaligned access (it spans two pages or IO).  */
    if (DATA_SIZE > 1
        && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
                    >= TARGET_PAGE_SIZE)) {
        target_ulong addr1, addr2;
        DATA_TYPE res1, res2;
        unsigned shift;
    do_unaligned_access:
#ifdef ALIGNED_ONLY
        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
#endif
        addr1 = addr & ~(DATA_SIZE - 1);
        addr2 = addr1 + DATA_SIZE;
        /* Note the adjustment at the beginning of the function.
           Undo that for the recursion.  */
        res1 = helper_be_ld_name(env, addr1, mmu_idx, retaddr + GETPC_ADJ);
        res2 = helper_be_ld_name(env, addr2, mmu_idx, retaddr + GETPC_ADJ);
        shift = (addr & (DATA_SIZE - 1)) * 8;

        /* Big-endian combine.  */
        res = (res1 << shift) | (res2 >> ((DATA_SIZE * 8) - shift));
        return res;
    }

    /* Handle aligned access or unaligned access in the same page.  */
#ifdef ALIGNED_ONLY
    if ((addr & (DATA_SIZE - 1)) != 0) {
        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
    }
#endif

    haddr = addr + env->tlb_table[mmu_idx][index].addend;
    res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);
    return res;
}
#endif /* DATA_SIZE > 1 */

DATA_TYPE
glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
                                         int mmu_idx)
{
    return helper_te_ld_name (env, addr, mmu_idx, GETRA());
}

#ifndef SOFTMMU_CODE_ACCESS

/* Provide signed versions of the load routines as well.  We can of course
   avoid this for 64-bit data, or for 32-bit data on 32-bit host.  */
#if DATA_SIZE * 8 < TCG_TARGET_REG_BITS
WORD_TYPE helper_le_lds_name(CPUArchState *env, target_ulong addr,
                             int mmu_idx, uintptr_t retaddr)
{
    return (SDATA_TYPE)helper_le_ld_name(env, addr, mmu_idx, retaddr);
}

# if DATA_SIZE > 1
WORD_TYPE helper_be_lds_name(CPUArchState *env, target_ulong addr,
                             int mmu_idx, uintptr_t retaddr)
{
    return (SDATA_TYPE)helper_be_ld_name(env, addr, mmu_idx, retaddr);
}
# endif
#endif

static inline void glue(io_write, SUFFIX)(CPUArchState *env,
                                          hwaddr physaddr,
                                          DATA_TYPE val,
                                          target_ulong addr,
                                          uintptr_t retaddr)
{
    CPUState *cpu = ENV_GET_CPU(env);
    MemoryRegion *mr = iotlb_to_region(cpu->as, physaddr);

    physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu_can_do_io(cpu)) {
        cpu_io_recompile(cpu, retaddr);
    }

    cpu->mem_io_vaddr = addr;
    cpu->mem_io_pc = retaddr;
    io_mem_write(mr, physaddr, val, 1 << SHIFT);
}

void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
                       int mmu_idx, uintptr_t retaddr)
{
    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
    uintptr_t haddr;

    /* Adjust the given return address.  */
    retaddr -= GETPC_ADJ;

    /* If the TLB entry is for a different page, reload and try again.  */
    if ((addr & TARGET_PAGE_MASK)
        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
#ifdef ALIGNED_ONLY
        if ((addr & (DATA_SIZE - 1)) != 0) {
            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
        }
#endif
        tlb_fill(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
        tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
    }

    /* Handle an IO access.  */
    if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
        hwaddr ioaddr;
        if ((addr & (DATA_SIZE - 1)) != 0) {
            goto do_unaligned_access;
        }
        ioaddr = env->iotlb[mmu_idx][index];

        /* ??? Note that the io helpers always read data in the target
           byte ordering.  We should push the LE/BE request down into io.  */
        val = TGT_LE(val);
        glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
        return;
    }

    /* Handle slow unaligned access (it spans two pages or IO).  */
    if (DATA_SIZE > 1
        && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
                     >= TARGET_PAGE_SIZE)) {
        int i;
    do_unaligned_access:
#ifdef ALIGNED_ONLY
        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
#endif
        /* XXX: not efficient, but simple */
        /* Note: relies on the fact that tlb_fill() does not remove the
         * previous page from the TLB cache.  */
        for (i = DATA_SIZE - 1; i >= 0; i--) {
            /* Little-endian extract.  */
            uint8_t val8 = val >> (i * 8);
            /* Note the adjustment at the beginning of the function.
               Undo that for the recursion.  */
            glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
                                            mmu_idx, retaddr + GETPC_ADJ);
        }
        return;
    }

    /* Handle aligned access or unaligned access in the same page.  */
#ifdef ALIGNED_ONLY
    if ((addr & (DATA_SIZE - 1)) != 0) {
        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
    }
#endif

    haddr = addr + env->tlb_table[mmu_idx][index].addend;
#if DATA_SIZE == 1
    glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
#else
    glue(glue(st, SUFFIX), _le_p)((uint8_t *)haddr, val);
#endif
}

#if DATA_SIZE > 1
void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
                       int mmu_idx, uintptr_t retaddr)
{
    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
    uintptr_t haddr;

    /* Adjust the given return address.  */
    retaddr -= GETPC_ADJ;

    /* If the TLB entry is for a different page, reload and try again.  */
    if ((addr & TARGET_PAGE_MASK)
        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
#ifdef ALIGNED_ONLY
        if ((addr & (DATA_SIZE - 1)) != 0) {
            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
        }
#endif
        tlb_fill(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
        tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
    }

    /* Handle an IO access.  */
    if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
        hwaddr ioaddr;
        if ((addr & (DATA_SIZE - 1)) != 0) {
            goto do_unaligned_access;
        }
        ioaddr = env->iotlb[mmu_idx][index];

        /* ??? Note that the io helpers always read data in the target
           byte ordering.  We should push the LE/BE request down into io.  */
        val = TGT_BE(val);
        glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
        return;
    }

    /* Handle slow unaligned access (it spans two pages or IO).  */
    if (DATA_SIZE > 1
        && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
                     >= TARGET_PAGE_SIZE)) {
        int i;
    do_unaligned_access:
#ifdef ALIGNED_ONLY
        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
#endif
        /* XXX: not efficient, but simple */
        /* Note: relies on the fact that tlb_fill() does not remove the
         * previous page from the TLB cache.  */
        for (i = DATA_SIZE - 1; i >= 0; i--) {
            /* Big-endian extract.  */
            uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));
            /* Note the adjustment at the beginning of the function.
               Undo that for the recursion.  */
            glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
                                            mmu_idx, retaddr + GETPC_ADJ);
        }
        return;
    }

    /* Handle aligned access or unaligned access in the same page.  */
#ifdef ALIGNED_ONLY
    if ((addr & (DATA_SIZE - 1)) != 0) {
        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
    }
#endif

    haddr = addr + env->tlb_table[mmu_idx][index].addend;
    glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);
}
#endif /* DATA_SIZE > 1 */

void
glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
                                         DATA_TYPE val, int mmu_idx)
{
    helper_te_st_name(env, addr, val, mmu_idx, GETRA());
}

#endif /* !defined(SOFTMMU_CODE_ACCESS) */

#undef READ_ACCESS_TYPE
#undef SHIFT
#undef DATA_TYPE
#undef SUFFIX
#undef LSUFFIX
#undef DATA_SIZE
#undef ADDR_READ
#undef WORD_TYPE
#undef SDATA_TYPE
#undef USUFFIX
#undef SSUFFIX
#undef BSWAP
#undef TGT_BE
#undef TGT_LE
#undef CPU_BE
#undef CPU_LE
#undef helper_le_ld_name
#undef helper_be_ld_name
#undef helper_le_lds_name
#undef helper_be_lds_name
#undef helper_le_st_name
#undef helper_be_st_name
#undef helper_te_ld_name
#undef helper_te_st_name
Commit	Line	Data
b92e5a22 FB	1	/*
b92e5a22 FB	2	* Software MMU support
5fafdf24	3	*
efbf29b6 BS	4	* Generate helpers used by TCG for qemu_ld/st ops and code load
	5	* functions.
	6	*
	7	* Included from target op helpers and exec.c.
	8	*
b92e5a22 FB	9	* Copyright (c) 2003 Fabrice Bellard
	10	*
	11	* This library is free software; you can redistribute it and/or
	12	* modify it under the terms of the GNU Lesser General Public
	13	* License as published by the Free Software Foundation; either
	14	* version 2 of the License, or (at your option) any later version.
	15	*
	16	* This library is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* Lesser General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU Lesser General Public
8167ee88	22	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
b92e5a22	23	*/
1de7afc9	24	#include "qemu/timer.h"
77717094	25	#include "exec/address-spaces.h"
022c62cb	26	#include "exec/memory.h"
29e922b6	27
b92e5a22 FB	28	#define DATA_SIZE (1 << SHIFT)
	29
	30	#if DATA_SIZE == 8
	31	#define SUFFIX q
701e3a5c	32	#define LSUFFIX q
c8f94df5	33	#define SDATA_TYPE int64_t
dc9a353c	34	#define DATA_TYPE uint64_t
b92e5a22 FB	35	#elif DATA_SIZE == 4
b92e5a22 FB	36	#define SUFFIX l
701e3a5c	37	#define LSUFFIX l
c8f94df5	38	#define SDATA_TYPE int32_t
dc9a353c	39	#define DATA_TYPE uint32_t
b92e5a22 FB	40	#elif DATA_SIZE == 2
b92e5a22 FB	41	#define SUFFIX w
701e3a5c	42	#define LSUFFIX uw
c8f94df5	43	#define SDATA_TYPE int16_t
dc9a353c	44	#define DATA_TYPE uint16_t
b92e5a22 FB	45	#elif DATA_SIZE == 1
b92e5a22 FB	46	#define SUFFIX b
701e3a5c	47	#define LSUFFIX ub
c8f94df5	48	#define SDATA_TYPE int8_t
dc9a353c	49	#define DATA_TYPE uint8_t
b92e5a22 FB	50	#else
	51	#error unsupported data size
	52	#endif
	53
c8f94df5 RH	54
	55	/* For the benefit of TCG generated code, we want to avoid the complication
	56	of ABI-specific return type promotion and always return a value extended
	57	to the register size of the host. This is tcg_target_long, except in the
	58	case of a 32-bit host and 64-bit data, and for that we always have
	59	uint64_t. Don't bother with this widened value for SOFTMMU_CODE_ACCESS. */
	60	#if defined(SOFTMMU_CODE_ACCESS) \|\| DATA_SIZE == 8
	61	# define WORD_TYPE DATA_TYPE
	62	# define USUFFIX SUFFIX
	63	#else
	64	# define WORD_TYPE tcg_target_ulong
	65	# define USUFFIX glue(u, SUFFIX)
	66	# define SSUFFIX glue(s, SUFFIX)
	67	#endif
	68
b769d8fe FB	69	#ifdef SOFTMMU_CODE_ACCESS
b769d8fe FB	70	#define READ_ACCESS_TYPE 2
84b7b8e7	71	#define ADDR_READ addr_code
b769d8fe FB	72	#else
b769d8fe FB	73	#define READ_ACCESS_TYPE 0
84b7b8e7	74	#define ADDR_READ addr_read
b769d8fe FB	75	#endif
b769d8fe FB	76
867b3201 RH	77	#if DATA_SIZE == 8
	78	# define BSWAP(X) bswap64(X)
	79	#elif DATA_SIZE == 4
	80	# define BSWAP(X) bswap32(X)
	81	#elif DATA_SIZE == 2
	82	# define BSWAP(X) bswap16(X)
	83	#else
	84	# define BSWAP(X) (X)
	85	#endif
	86
	87	#ifdef TARGET_WORDS_BIGENDIAN
	88	# define TGT_BE(X) (X)
	89	# define TGT_LE(X) BSWAP(X)
	90	#else
	91	# define TGT_BE(X) BSWAP(X)
	92	# define TGT_LE(X) (X)
	93	#endif
	94
	95	#if DATA_SIZE == 1
	96	# define helper_le_ld_name glue(glue(helper_ret_ld, USUFFIX), MMUSUFFIX)
	97	# define helper_be_ld_name helper_le_ld_name
	98	# define helper_le_lds_name glue(glue(helper_ret_ld, SSUFFIX), MMUSUFFIX)
	99	# define helper_be_lds_name helper_le_lds_name
	100	# define helper_le_st_name glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)
	101	# define helper_be_st_name helper_le_st_name
	102	#else
	103	# define helper_le_ld_name glue(glue(helper_le_ld, USUFFIX), MMUSUFFIX)
	104	# define helper_be_ld_name glue(glue(helper_be_ld, USUFFIX), MMUSUFFIX)
	105	# define helper_le_lds_name glue(glue(helper_le_ld, SSUFFIX), MMUSUFFIX)
	106	# define helper_be_lds_name glue(glue(helper_be_ld, SSUFFIX), MMUSUFFIX)
	107	# define helper_le_st_name glue(glue(helper_le_st, SUFFIX), MMUSUFFIX)
	108	# define helper_be_st_name glue(glue(helper_be_st, SUFFIX), MMUSUFFIX)
	109	#endif
	110
	111	#ifdef TARGET_WORDS_BIGENDIAN
	112	# define helper_te_ld_name helper_be_ld_name
	113	# define helper_te_st_name helper_be_st_name
	114	#else
	115	# define helper_te_ld_name helper_le_ld_name
	116	# define helper_te_st_name helper_le_st_name
	117	#endif
	118
89c33337	119	static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
a8170e5e	120	hwaddr physaddr,
2e70f6ef	121	target_ulong addr,
20503968	122	uintptr_t retaddr)
b92e5a22	123	{
791af8c8	124	uint64_t val;
09daed84 EI	125	CPUState *cpu = ENV_GET_CPU(env);
09daed84 EI	126	MemoryRegion *mr = iotlb_to_region(cpu->as, physaddr);
37ec01d4	127
0f459d16	128	physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
93afeade	129	cpu->mem_io_pc = retaddr;
99df7dce	130	if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu_can_do_io(cpu)) {
90b40a69	131	cpu_io_recompile(cpu, retaddr);
2e70f6ef	132	}
b92e5a22	133
93afeade	134	cpu->mem_io_vaddr = addr;
791af8c8 PB	135	io_mem_read(mr, physaddr, &val, 1 << SHIFT);
791af8c8 PB	136	return val;
b92e5a22 FB	137	}
b92e5a22 FB	138
e25c3887	139	#ifdef SOFTMMU_CODE_ACCESS
867b3201	140	static __attribute__((unused))
e25c3887	141	#endif
867b3201 RH	142	WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
867b3201 RH	143	uintptr_t retaddr)
b92e5a22	144	{
aac1fb05 RH	145	int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
	146	target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
	147	uintptr_t haddr;
867b3201	148	DATA_TYPE res;
3b46e624	149
0f842f8a RH	150	/* Adjust the given return address. */
	151	retaddr -= GETPC_ADJ;
	152
aac1fb05 RH	153	/* If the TLB entry is for a different page, reload and try again. */
	154	if ((addr & TARGET_PAGE_MASK)
	155	!= (tlb_addr & (TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
a64d4718	156	#ifdef ALIGNED_ONLY
aac1fb05	157	if ((addr & (DATA_SIZE - 1)) != 0) {
89c33337	158	do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
aac1fb05	159	}
a64d4718	160	#endif
d5a11fef	161	tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
aac1fb05 RH	162	tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
	163	}
	164
	165	/* Handle an IO access. */
	166	if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
	167	hwaddr ioaddr;
	168	if ((addr & (DATA_SIZE - 1)) != 0) {
	169	goto do_unaligned_access;
b92e5a22	170	}
aac1fb05	171	ioaddr = env->iotlb[mmu_idx][index];
867b3201 RH	172
	173	/* ??? Note that the io helpers always read data in the target
	174	byte ordering. We should push the LE/BE request down into io. */
	175	res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
	176	res = TGT_LE(res);
	177	return res;
aac1fb05 RH	178	}
	179
	180	/* Handle slow unaligned access (it spans two pages or IO). */
	181	if (DATA_SIZE > 1
	182	&& unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
	183	>= TARGET_PAGE_SIZE)) {
	184	target_ulong addr1, addr2;
867b3201	185	DATA_TYPE res1, res2;
aac1fb05 RH	186	unsigned shift;
aac1fb05 RH	187	do_unaligned_access:
a64d4718	188	#ifdef ALIGNED_ONLY
aac1fb05	189	do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
a64d4718	190	#endif
aac1fb05 RH	191	addr1 = addr & ~(DATA_SIZE - 1);
aac1fb05 RH	192	addr2 = addr1 + DATA_SIZE;
0f842f8a RH	193	/* Note the adjustment at the beginning of the function.
0f842f8a RH	194	Undo that for the recursion. */
867b3201 RH	195	res1 = helper_le_ld_name(env, addr1, mmu_idx, retaddr + GETPC_ADJ);
867b3201 RH	196	res2 = helper_le_ld_name(env, addr2, mmu_idx, retaddr + GETPC_ADJ);
aac1fb05	197	shift = (addr & (DATA_SIZE - 1)) * 8;
867b3201 RH	198
867b3201 RH	199	/* Little-endian combine. */
aac1fb05	200	res = (res1 >> shift) \| (res2 << ((DATA_SIZE * 8) - shift));
867b3201 RH	201	return res;
	202	}
	203
	204	/* Handle aligned access or unaligned access in the same page. */
	205	#ifdef ALIGNED_ONLY
	206	if ((addr & (DATA_SIZE - 1)) != 0) {
	207	do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
	208	}
	209	#endif
	210
	211	haddr = addr + env->tlb_table[mmu_idx][index].addend;
	212	#if DATA_SIZE == 1
	213	res = glue(glue(ld, LSUFFIX), _p)((uint8_t *)haddr);
	214	#else
	215	res = glue(glue(ld, LSUFFIX), _le_p)((uint8_t *)haddr);
	216	#endif
	217	return res;
	218	}
	219
	220	#if DATA_SIZE > 1
	221	#ifdef SOFTMMU_CODE_ACCESS
	222	static __attribute__((unused))
	223	#endif
	224	WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr, int mmu_idx,
	225	uintptr_t retaddr)
	226	{
	227	int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
	228	target_ulong tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
	229	uintptr_t haddr;
	230	DATA_TYPE res;
	231
	232	/* Adjust the given return address. */
	233	retaddr -= GETPC_ADJ;
	234
	235	/* If the TLB entry is for a different page, reload and try again. */
	236	if ((addr & TARGET_PAGE_MASK)
	237	!= (tlb_addr & (TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
	238	#ifdef ALIGNED_ONLY
	239	if ((addr & (DATA_SIZE - 1)) != 0) {
	240	do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
	241	}
	242	#endif
d5a11fef	243	tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
867b3201 RH	244	tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
	245	}
	246
	247	/* Handle an IO access. */
	248	if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
	249	hwaddr ioaddr;
	250	if ((addr & (DATA_SIZE - 1)) != 0) {
	251	goto do_unaligned_access;
	252	}
	253	ioaddr = env->iotlb[mmu_idx][index];
	254
	255	/* ??? Note that the io helpers always read data in the target
	256	byte ordering. We should push the LE/BE request down into io. */
	257	res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
	258	res = TGT_BE(res);
	259	return res;
	260	}
	261
	262	/* Handle slow unaligned access (it spans two pages or IO). */
	263	if (DATA_SIZE > 1
	264	&& unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
	265	>= TARGET_PAGE_SIZE)) {
	266	target_ulong addr1, addr2;
	267	DATA_TYPE res1, res2;
	268	unsigned shift;
	269	do_unaligned_access:
	270	#ifdef ALIGNED_ONLY
	271	do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
aac1fb05	272	#endif
867b3201 RH	273	addr1 = addr & ~(DATA_SIZE - 1);
	274	addr2 = addr1 + DATA_SIZE;
	275	/* Note the adjustment at the beginning of the function.
	276	Undo that for the recursion. */
	277	res1 = helper_be_ld_name(env, addr1, mmu_idx, retaddr + GETPC_ADJ);
	278	res2 = helper_be_ld_name(env, addr2, mmu_idx, retaddr + GETPC_ADJ);
	279	shift = (addr & (DATA_SIZE - 1)) * 8;
	280
	281	/* Big-endian combine. */
	282	res = (res1 << shift) \| (res2 >> ((DATA_SIZE * 8) - shift));
aac1fb05 RH	283	return res;
	284	}
	285
	286	/* Handle aligned access or unaligned access in the same page. */
	287	#ifdef ALIGNED_ONLY
	288	if ((addr & (DATA_SIZE - 1)) != 0) {
	289	do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
b92e5a22	290	}
aac1fb05 RH	291	#endif
	292
	293	haddr = addr + env->tlb_table[mmu_idx][index].addend;
867b3201 RH	294	res = glue(glue(ld, LSUFFIX), _be_p)((uint8_t *)haddr);
867b3201 RH	295	return res;
b92e5a22	296	}
867b3201	297	#endif /* DATA_SIZE > 1 */
b92e5a22	298
e25c3887 RH	299	DATA_TYPE
	300	glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
	301	int mmu_idx)
	302	{
867b3201	303	return helper_te_ld_name (env, addr, mmu_idx, GETRA());
e25c3887 RH	304	}
e25c3887 RH	305
b769d8fe FB	306	#ifndef SOFTMMU_CODE_ACCESS
b769d8fe FB	307
c8f94df5 RH	308	/* Provide signed versions of the load routines as well. We can of course
	309	avoid this for 64-bit data, or for 32-bit data on 32-bit host. */
	310	#if DATA_SIZE * 8 < TCG_TARGET_REG_BITS
867b3201 RH	311	WORD_TYPE helper_le_lds_name(CPUArchState *env, target_ulong addr,
	312	int mmu_idx, uintptr_t retaddr)
	313	{
	314	return (SDATA_TYPE)helper_le_ld_name(env, addr, mmu_idx, retaddr);
	315	}
	316
	317	# if DATA_SIZE > 1
	318	WORD_TYPE helper_be_lds_name(CPUArchState *env, target_ulong addr,
	319	int mmu_idx, uintptr_t retaddr)
c8f94df5	320	{
867b3201	321	return (SDATA_TYPE)helper_be_ld_name(env, addr, mmu_idx, retaddr);
c8f94df5	322	}
867b3201	323	# endif
c8f94df5 RH	324	#endif
c8f94df5 RH	325
89c33337	326	static inline void glue(io_write, SUFFIX)(CPUArchState *env,
a8170e5e	327	hwaddr physaddr,
b769d8fe	328	DATA_TYPE val,
0f459d16	329	target_ulong addr,
20503968	330	uintptr_t retaddr)
b769d8fe	331	{
09daed84 EI	332	CPUState *cpu = ENV_GET_CPU(env);
09daed84 EI	333	MemoryRegion *mr = iotlb_to_region(cpu->as, physaddr);
37ec01d4	334
0f459d16	335	physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
99df7dce	336	if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu_can_do_io(cpu)) {
90b40a69	337	cpu_io_recompile(cpu, retaddr);
2e70f6ef	338	}
b769d8fe	339
93afeade AF	340	cpu->mem_io_vaddr = addr;
93afeade AF	341	cpu->mem_io_pc = retaddr;
37ec01d4	342	io_mem_write(mr, physaddr, val, 1 << SHIFT);
b769d8fe	343	}
b92e5a22	344
867b3201 RH	345	void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
867b3201 RH	346	int mmu_idx, uintptr_t retaddr)
b92e5a22	347	{
aac1fb05 RH	348	int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
	349	target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
	350	uintptr_t haddr;
3b46e624	351
0f842f8a RH	352	/* Adjust the given return address. */
	353	retaddr -= GETPC_ADJ;
	354
aac1fb05 RH	355	/* If the TLB entry is for a different page, reload and try again. */
	356	if ((addr & TARGET_PAGE_MASK)
	357	!= (tlb_addr & (TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
a64d4718	358	#ifdef ALIGNED_ONLY
aac1fb05	359	if ((addr & (DATA_SIZE - 1)) != 0) {
89c33337	360	do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
aac1fb05	361	}
a64d4718	362	#endif
d5a11fef	363	tlb_fill(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
aac1fb05 RH	364	tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
	365	}
	366
	367	/* Handle an IO access. */
	368	if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
	369	hwaddr ioaddr;
	370	if ((addr & (DATA_SIZE - 1)) != 0) {
	371	goto do_unaligned_access;
	372	}
	373	ioaddr = env->iotlb[mmu_idx][index];
867b3201 RH	374
	375	/* ??? Note that the io helpers always read data in the target
	376	byte ordering. We should push the LE/BE request down into io. */
	377	val = TGT_LE(val);
aac1fb05 RH	378	glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
	379	return;
	380	}
	381
	382	/* Handle slow unaligned access (it spans two pages or IO). */
	383	if (DATA_SIZE > 1
	384	&& unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
	385	>= TARGET_PAGE_SIZE)) {
	386	int i;
	387	do_unaligned_access:
a64d4718	388	#ifdef ALIGNED_ONLY
aac1fb05 RH	389	do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
	390	#endif
	391	/* XXX: not efficient, but simple */
	392	/* Note: relies on the fact that tlb_fill() does not remove the
	393	* previous page from the TLB cache. */
	394	for (i = DATA_SIZE - 1; i >= 0; i--) {
867b3201	395	/* Little-endian extract. */
aac1fb05	396	uint8_t val8 = val >> (i * 8);
867b3201 RH	397	/* Note the adjustment at the beginning of the function.
	398	Undo that for the recursion. */
	399	glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
	400	mmu_idx, retaddr + GETPC_ADJ);
	401	}
	402	return;
	403	}
	404
	405	/* Handle aligned access or unaligned access in the same page. */
	406	#ifdef ALIGNED_ONLY
	407	if ((addr & (DATA_SIZE - 1)) != 0) {
	408	do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
	409	}
	410	#endif
	411
	412	haddr = addr + env->tlb_table[mmu_idx][index].addend;
	413	#if DATA_SIZE == 1
	414	glue(glue(st, SUFFIX), _p)((uint8_t *)haddr, val);
	415	#else
	416	glue(glue(st, SUFFIX), _le_p)((uint8_t *)haddr, val);
a64d4718	417	#endif
867b3201 RH	418	}
	419
	420	#if DATA_SIZE > 1
	421	void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
	422	int mmu_idx, uintptr_t retaddr)
	423	{
	424	int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
	425	target_ulong tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
	426	uintptr_t haddr;
	427
	428	/* Adjust the given return address. */
	429	retaddr -= GETPC_ADJ;
	430
	431	/* If the TLB entry is for a different page, reload and try again. */
	432	if ((addr & TARGET_PAGE_MASK)
	433	!= (tlb_addr & (TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
	434	#ifdef ALIGNED_ONLY
	435	if ((addr & (DATA_SIZE - 1)) != 0) {
	436	do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
	437	}
	438	#endif
d5a11fef	439	tlb_fill(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
867b3201 RH	440	tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
	441	}
	442
	443	/* Handle an IO access. */
	444	if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
	445	hwaddr ioaddr;
	446	if ((addr & (DATA_SIZE - 1)) != 0) {
	447	goto do_unaligned_access;
	448	}
	449	ioaddr = env->iotlb[mmu_idx][index];
	450
	451	/* ??? Note that the io helpers always read data in the target
	452	byte ordering. We should push the LE/BE request down into io. */
	453	val = TGT_BE(val);
	454	glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
	455	return;
	456	}
	457
	458	/* Handle slow unaligned access (it spans two pages or IO). */
	459	if (DATA_SIZE > 1
	460	&& unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
	461	>= TARGET_PAGE_SIZE)) {
	462	int i;
	463	do_unaligned_access:
	464	#ifdef ALIGNED_ONLY
	465	do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
	466	#endif
	467	/* XXX: not efficient, but simple */
	468	/* Note: relies on the fact that tlb_fill() does not remove the
	469	* previous page from the TLB cache. */
	470	for (i = DATA_SIZE - 1; i >= 0; i--) {
	471	/* Big-endian extract. */
	472	uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));
0f842f8a RH	473	/* Note the adjustment at the beginning of the function.
0f842f8a RH	474	Undo that for the recursion. */
aac1fb05	475	glue(helper_ret_stb, MMUSUFFIX)(env, addr + i, val8,
0f842f8a	476	mmu_idx, retaddr + GETPC_ADJ);
b92e5a22	477	}
aac1fb05 RH	478	return;
	479	}
	480
	481	/* Handle aligned access or unaligned access in the same page. */
a64d4718	482	#ifdef ALIGNED_ONLY
aac1fb05 RH	483	if ((addr & (DATA_SIZE - 1)) != 0) {
aac1fb05 RH	484	do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
b92e5a22	485	}
aac1fb05 RH	486	#endif
	487
	488	haddr = addr + env->tlb_table[mmu_idx][index].addend;
867b3201	489	glue(glue(st, SUFFIX), _be_p)((uint8_t *)haddr, val);
b92e5a22	490	}
867b3201	491	#endif /* DATA_SIZE > 1 */
b92e5a22	492
e25c3887 RH	493	void
	494	glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
	495	DATA_TYPE val, int mmu_idx)
	496	{
867b3201	497	helper_te_st_name(env, addr, val, mmu_idx, GETRA());
e25c3887 RH	498	}
e25c3887 RH	499
b769d8fe FB	500	#endif /* !defined(SOFTMMU_CODE_ACCESS) */
	501
	502	#undef READ_ACCESS_TYPE
b92e5a22 FB	503	#undef SHIFT
	504	#undef DATA_TYPE
	505	#undef SUFFIX
701e3a5c	506	#undef LSUFFIX
b92e5a22	507	#undef DATA_SIZE
84b7b8e7	508	#undef ADDR_READ
c8f94df5 RH	509	#undef WORD_TYPE
	510	#undef SDATA_TYPE
	511	#undef USUFFIX
	512	#undef SSUFFIX
867b3201 RH	513	#undef BSWAP
	514	#undef TGT_BE
	515	#undef TGT_LE
	516	#undef CPU_BE
	517	#undef CPU_LE
	518	#undef helper_le_ld_name
	519	#undef helper_be_ld_name
	520	#undef helper_le_lds_name
	521	#undef helper_be_lds_name
	522	#undef helper_le_st_name
	523	#undef helper_be_st_name
	524	#undef helper_te_ld_name
	525	#undef helper_te_st_name