[qemu.git] / softmmu_header.h

/*
 *  Software MMU support
 *
 *  Copyright (c) 2003 Fabrice Bellard
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
#if DATA_SIZE == 8
#define SUFFIX q
#define USUFFIX q
#define DATA_TYPE uint64_t
#elif DATA_SIZE == 4
#define SUFFIX l
#define USUFFIX l
#define DATA_TYPE uint32_t
#elif DATA_SIZE == 2
#define SUFFIX w
#define USUFFIX uw
#define DATA_TYPE uint16_t
#define DATA_STYPE int16_t
#elif DATA_SIZE == 1
#define SUFFIX b
#define USUFFIX ub
#define DATA_TYPE uint8_t
#define DATA_STYPE int8_t
#else
#error unsupported data size
#endif

#if ACCESS_TYPE == 0

#define CPU_MEM_INDEX 0
#define MMUSUFFIX _mmu

#elif ACCESS_TYPE == 1

#define CPU_MEM_INDEX 1
#define MMUSUFFIX _mmu

#elif ACCESS_TYPE == 2

#ifdef TARGET_I386
#define CPU_MEM_INDEX ((env->hflags & HF_CPL_MASK) == 3)
#elif defined (TARGET_PPC)
#define CPU_MEM_INDEX (msr_pr)
#elif defined (TARGET_MIPS)
#define CPU_MEM_INDEX ((env->hflags & MIPS_HFLAG_MODE) == MIPS_HFLAG_UM)
#elif defined (TARGET_SPARC)
#define CPU_MEM_INDEX ((env->psrs) == 0)
#elif defined (TARGET_ARM)
#define CPU_MEM_INDEX ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR)
#elif defined (TARGET_SH4)
#define CPU_MEM_INDEX ((env->sr & SR_MD) == 0)
#elif defined (TARGET_ALPHA)
#define CPU_MEM_INDEX ((env->ps >> 3) & 3)
#elif defined (TARGET_M68K)
#define CPU_MEM_INDEX ((env->sr & SR_S) == 0)
#else
#error unsupported CPU
#endif
#define MMUSUFFIX _mmu

#elif ACCESS_TYPE == 3

#ifdef TARGET_I386
#define CPU_MEM_INDEX ((env->hflags & HF_CPL_MASK) == 3)
#elif defined (TARGET_PPC)
#define CPU_MEM_INDEX (msr_pr)
#elif defined (TARGET_MIPS)
#define CPU_MEM_INDEX ((env->hflags & MIPS_HFLAG_MODE) == MIPS_HFLAG_UM)
#elif defined (TARGET_SPARC)
#define CPU_MEM_INDEX ((env->psrs) == 0)
#elif defined (TARGET_ARM)
#define CPU_MEM_INDEX ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR)
#elif defined (TARGET_SH4)
#define CPU_MEM_INDEX ((env->sr & SR_MD) == 0)
#elif defined (TARGET_ALPHA)
#define CPU_MEM_INDEX ((env->ps >> 3) & 3)
#elif defined (TARGET_M68K)
#define CPU_MEM_INDEX ((env->sr & SR_S) == 0)
#else
#error unsupported CPU
#endif
#define MMUSUFFIX _cmmu

#else
#error invalid ACCESS_TYPE
#endif

#if DATA_SIZE == 8
#define RES_TYPE uint64_t
#else
#define RES_TYPE int
#endif

#if ACCESS_TYPE == 3
#define ADDR_READ addr_code
#else
#define ADDR_READ addr_read
#endif

DATA_TYPE REGPARM(1) glue(glue(__ld, SUFFIX), MMUSUFFIX)(target_ulong addr,
                                                         int is_user);
void REGPARM(2) glue(glue(__st, SUFFIX), MMUSUFFIX)(target_ulong addr, DATA_TYPE v, int is_user);

#if (DATA_SIZE <= 4) && (TARGET_LONG_BITS == 32) && defined(__i386__) && \
    (ACCESS_TYPE <= 1) && defined(ASM_SOFTMMU)

#define CPU_TLB_ENTRY_BITS 4

static inline RES_TYPE glue(glue(ld, USUFFIX), MEMSUFFIX)(target_ulong ptr)
{
    int res;

    asm volatile ("movl %1, %%edx\n"
                  "movl %1, %%eax\n"
                  "shrl %3, %%edx\n"
                  "andl %4, %%eax\n"
                  "andl %2, %%edx\n"
                  "leal %5(%%edx, %%ebp), %%edx\n"
                  "cmpl (%%edx), %%eax\n"
                  "movl %1, %%eax\n"
                  "je 1f\n"
                  "pushl %6\n"
                  "call %7\n"
                  "popl %%edx\n"
                  "movl %%eax, %0\n"
                  "jmp 2f\n"
                  "1:\n"
                  "addl 12(%%edx), %%eax\n"
#if DATA_SIZE == 1
                  "movzbl (%%eax), %0\n"
#elif DATA_SIZE == 2
                  "movzwl (%%eax), %0\n"
#elif DATA_SIZE == 4
                  "movl (%%eax), %0\n"
#else
#error unsupported size
#endif
                  "2:\n"
                  : "=r" (res)
                  : "r" (ptr),
                  "i" ((CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS),
                  "i" (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
                  "i" (TARGET_PAGE_MASK | (DATA_SIZE - 1)),
                  "m" (*(uint32_t *)offsetof(CPUState, tlb_table[CPU_MEM_INDEX][0].addr_read)),
                  "i" (CPU_MEM_INDEX),
                  "m" (*(uint8_t *)&glue(glue(__ld, SUFFIX), MMUSUFFIX))
                  : "%eax", "%ecx", "%edx", "memory", "cc");
    return res;
}

#if DATA_SIZE <= 2
static inline int glue(glue(lds, SUFFIX), MEMSUFFIX)(target_ulong ptr)
{
    int res;

    asm volatile ("movl %1, %%edx\n"
                  "movl %1, %%eax\n"
                  "shrl %3, %%edx\n"
                  "andl %4, %%eax\n"
                  "andl %2, %%edx\n"
                  "leal %5(%%edx, %%ebp), %%edx\n"
                  "cmpl (%%edx), %%eax\n"
                  "movl %1, %%eax\n"
                  "je 1f\n"
                  "pushl %6\n"
                  "call %7\n"
                  "popl %%edx\n"
#if DATA_SIZE == 1
                  "movsbl %%al, %0\n"
#elif DATA_SIZE == 2
                  "movswl %%ax, %0\n"
#else
#error unsupported size
#endif
                  "jmp 2f\n"
                  "1:\n"
                  "addl 12(%%edx), %%eax\n"
#if DATA_SIZE == 1
                  "movsbl (%%eax), %0\n"
#elif DATA_SIZE == 2
                  "movswl (%%eax), %0\n"
#else
#error unsupported size
#endif
                  "2:\n"
                  : "=r" (res)
                  : "r" (ptr),
                  "i" ((CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS),
                  "i" (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
                  "i" (TARGET_PAGE_MASK | (DATA_SIZE - 1)),
                  "m" (*(uint32_t *)offsetof(CPUState, tlb_table[CPU_MEM_INDEX][0].addr_read)),
                  "i" (CPU_MEM_INDEX),
                  "m" (*(uint8_t *)&glue(glue(__ld, SUFFIX), MMUSUFFIX))
                  : "%eax", "%ecx", "%edx", "memory", "cc");
    return res;
}
#endif

static inline void glue(glue(st, SUFFIX), MEMSUFFIX)(target_ulong ptr, RES_TYPE v)
{
    asm volatile ("movl %0, %%edx\n"
                  "movl %0, %%eax\n"
                  "shrl %3, %%edx\n"
                  "andl %4, %%eax\n"
                  "andl %2, %%edx\n"
                  "leal %5(%%edx, %%ebp), %%edx\n"
                  "cmpl (%%edx), %%eax\n"
                  "movl %0, %%eax\n"
                  "je 1f\n"
#if DATA_SIZE == 1
                  "movzbl %b1, %%edx\n"
#elif DATA_SIZE == 2
                  "movzwl %w1, %%edx\n"
#elif DATA_SIZE == 4
                  "movl %1, %%edx\n"
#else
#error unsupported size
#endif
                  "pushl %6\n"
                  "call %7\n"
                  "popl %%eax\n"
                  "jmp 2f\n"
                  "1:\n"
                  "addl 8(%%edx), %%eax\n"
#if DATA_SIZE == 1
                  "movb %b1, (%%eax)\n"
#elif DATA_SIZE == 2
                  "movw %w1, (%%eax)\n"
#elif DATA_SIZE == 4
                  "movl %1, (%%eax)\n"
#else
#error unsupported size
#endif
                  "2:\n"
                  :
                  : "r" (ptr),
/* NOTE: 'q' would be needed as constraint, but we could not use it
   with T1 ! */
                  "r" (v),
                  "i" ((CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS),
                  "i" (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
                  "i" (TARGET_PAGE_MASK | (DATA_SIZE - 1)),
                  "m" (*(uint32_t *)offsetof(CPUState, tlb_table[CPU_MEM_INDEX][0].addr_write)),
                  "i" (CPU_MEM_INDEX),
                  "m" (*(uint8_t *)&glue(glue(__st, SUFFIX), MMUSUFFIX))
                  : "%eax", "%ecx", "%edx", "memory", "cc");
}

#else

/* generic load/store macros */

static inline RES_TYPE glue(glue(ld, USUFFIX), MEMSUFFIX)(target_ulong ptr)
{
    int index;
    RES_TYPE res;
    target_ulong addr;
    unsigned long physaddr;
    int is_user;

    addr = ptr;
    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    is_user = CPU_MEM_INDEX;
    if (__builtin_expect(env->tlb_table[is_user][index].ADDR_READ !=
                         (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))), 0)) {
        res = glue(glue(__ld, SUFFIX), MMUSUFFIX)(addr, is_user);
    } else {
        physaddr = addr + env->tlb_table[is_user][index].addend;
        res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)physaddr);
    }
    return res;
}

#if DATA_SIZE <= 2
static inline int glue(glue(lds, SUFFIX), MEMSUFFIX)(target_ulong ptr)
{
    int res, index;
    target_ulong addr;
    unsigned long physaddr;
    int is_user;

    addr = ptr;
    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    is_user = CPU_MEM_INDEX;
    if (__builtin_expect(env->tlb_table[is_user][index].ADDR_READ !=
                         (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))), 0)) {
        res = (DATA_STYPE)glue(glue(__ld, SUFFIX), MMUSUFFIX)(addr, is_user);
    } else {
        physaddr = addr + env->tlb_table[is_user][index].addend;
        res = glue(glue(lds, SUFFIX), _raw)((uint8_t *)physaddr);
    }
    return res;
}
#endif

#if ACCESS_TYPE != 3

/* generic store macro */

static inline void glue(glue(st, SUFFIX), MEMSUFFIX)(target_ulong ptr, RES_TYPE v)
{
    int index;
    target_ulong addr;
    unsigned long physaddr;
    int is_user;

    addr = ptr;
    index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    is_user = CPU_MEM_INDEX;
    if (__builtin_expect(env->tlb_table[is_user][index].addr_write !=
                         (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))), 0)) {
        glue(glue(__st, SUFFIX), MMUSUFFIX)(addr, v, is_user);
    } else {
        physaddr = addr + env->tlb_table[is_user][index].addend;
        glue(glue(st, SUFFIX), _raw)((uint8_t *)physaddr, v);
    }
}

#endif /* ACCESS_TYPE != 3 */

#endif /* !asm */

#if ACCESS_TYPE != 3

#if DATA_SIZE == 8
static inline float64 glue(ldfq, MEMSUFFIX)(target_ulong ptr)
{
    union {
        float64 d;
        uint64_t i;
    } u;
    u.i = glue(ldq, MEMSUFFIX)(ptr);
    return u.d;
}

static inline void glue(stfq, MEMSUFFIX)(target_ulong ptr, float64 v)
{
    union {
        float64 d;
        uint64_t i;
    } u;
    u.d = v;
    glue(stq, MEMSUFFIX)(ptr, u.i);
}
#endif /* DATA_SIZE == 8 */

#if DATA_SIZE == 4
static inline float32 glue(ldfl, MEMSUFFIX)(target_ulong ptr)
{
    union {
        float32 f;
        uint32_t i;
    } u;
    u.i = glue(ldl, MEMSUFFIX)(ptr);
    return u.f;
}

static inline void glue(stfl, MEMSUFFIX)(target_ulong ptr, float32 v)
{
    union {
        float32 f;
        uint32_t i;
    } u;
    u.f = v;
    glue(stl, MEMSUFFIX)(ptr, u.i);
}
#endif /* DATA_SIZE == 4 */

#endif /* ACCESS_TYPE != 3 */

#undef RES_TYPE
#undef DATA_TYPE
#undef DATA_STYPE
#undef SUFFIX
#undef USUFFIX
#undef DATA_SIZE
#undef CPU_MEM_INDEX
#undef MMUSUFFIX
#undef ADDR_READ
Commit	Line	Data
b92e5a22 FB	1	/*
b92e5a22 FB	2	* Software MMU support
5fafdf24	3	*
b92e5a22 FB	4	* Copyright (c) 2003 Fabrice Bellard
	5	*
	6	* This library is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2 of the License, or (at your option) any later version.
	10	*
	11	* This library is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with this library; if not, write to the Free Software
	18	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	19	*/
	20	#if DATA_SIZE == 8
	21	#define SUFFIX q
61382a50	22	#define USUFFIX q
b92e5a22 FB	23	#define DATA_TYPE uint64_t
	24	#elif DATA_SIZE == 4
	25	#define SUFFIX l
61382a50	26	#define USUFFIX l
b92e5a22 FB	27	#define DATA_TYPE uint32_t
	28	#elif DATA_SIZE == 2
	29	#define SUFFIX w
61382a50	30	#define USUFFIX uw
b92e5a22 FB	31	#define DATA_TYPE uint16_t
	32	#define DATA_STYPE int16_t
	33	#elif DATA_SIZE == 1
	34	#define SUFFIX b
61382a50	35	#define USUFFIX ub
b92e5a22 FB	36	#define DATA_TYPE uint8_t
	37	#define DATA_STYPE int8_t
	38	#else
	39	#error unsupported data size
	40	#endif
	41
61382a50 FB	42	#if ACCESS_TYPE == 0
	43
	44	#define CPU_MEM_INDEX 0
	45	#define MMUSUFFIX _mmu
	46
	47	#elif ACCESS_TYPE == 1
	48
	49	#define CPU_MEM_INDEX 1
	50	#define MMUSUFFIX _mmu
	51
	52	#elif ACCESS_TYPE == 2
	53
2d603d22	54	#ifdef TARGET_I386
61382a50	55	#define CPU_MEM_INDEX ((env->hflags & HF_CPL_MASK) == 3)
2d603d22 FB	56	#elif defined (TARGET_PPC)
2d603d22 FB	57	#define CPU_MEM_INDEX (msr_pr)
6af0bf9c FB	58	#elif defined (TARGET_MIPS)
6af0bf9c FB	59	#define CPU_MEM_INDEX ((env->hflags & MIPS_HFLAG_MODE) == MIPS_HFLAG_UM)
e95c8d51 FB	60	#elif defined (TARGET_SPARC)
e95c8d51 FB	61	#define CPU_MEM_INDEX ((env->psrs) == 0)
b5ff1b31 FB	62	#elif defined (TARGET_ARM)
b5ff1b31 FB	63	#define CPU_MEM_INDEX ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR)
fdf9b3e8 FB	64	#elif defined (TARGET_SH4)
fdf9b3e8 FB	65	#define CPU_MEM_INDEX ((env->sr & SR_MD) == 0)
eddf68a6 JM	66	#elif defined (TARGET_ALPHA)
eddf68a6 JM	67	#define CPU_MEM_INDEX ((env->ps >> 3) & 3)
0633879f PB	68	#elif defined (TARGET_M68K)
0633879f PB	69	#define CPU_MEM_INDEX ((env->sr & SR_S) == 0)
b5ff1b31 FB	70	#else
b5ff1b31 FB	71	#error unsupported CPU
2d603d22	72	#endif
61382a50 FB	73	#define MMUSUFFIX _mmu
	74
	75	#elif ACCESS_TYPE == 3
	76
2d603d22	77	#ifdef TARGET_I386
61382a50	78	#define CPU_MEM_INDEX ((env->hflags & HF_CPL_MASK) == 3)
2d603d22 FB	79	#elif defined (TARGET_PPC)
2d603d22 FB	80	#define CPU_MEM_INDEX (msr_pr)
6af0bf9c FB	81	#elif defined (TARGET_MIPS)
6af0bf9c FB	82	#define CPU_MEM_INDEX ((env->hflags & MIPS_HFLAG_MODE) == MIPS_HFLAG_UM)
e95c8d51 FB	83	#elif defined (TARGET_SPARC)
e95c8d51 FB	84	#define CPU_MEM_INDEX ((env->psrs) == 0)
b5ff1b31 FB	85	#elif defined (TARGET_ARM)
b5ff1b31 FB	86	#define CPU_MEM_INDEX ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR)
fdf9b3e8 FB	87	#elif defined (TARGET_SH4)
fdf9b3e8 FB	88	#define CPU_MEM_INDEX ((env->sr & SR_MD) == 0)
eddf68a6 JM	89	#elif defined (TARGET_ALPHA)
eddf68a6 JM	90	#define CPU_MEM_INDEX ((env->ps >> 3) & 3)
0633879f PB	91	#elif defined (TARGET_M68K)
0633879f PB	92	#define CPU_MEM_INDEX ((env->sr & SR_S) == 0)
b5ff1b31 FB	93	#else
b5ff1b31 FB	94	#error unsupported CPU
2d603d22	95	#endif
61382a50 FB	96	#define MMUSUFFIX _cmmu
61382a50 FB	97
b92e5a22	98	#else
61382a50	99	#error invalid ACCESS_TYPE
b92e5a22 FB	100	#endif
	101
	102	#if DATA_SIZE == 8
	103	#define RES_TYPE uint64_t
	104	#else
	105	#define RES_TYPE int
	106	#endif
	107
84b7b8e7 FB	108	#if ACCESS_TYPE == 3
	109	#define ADDR_READ addr_code
	110	#else
	111	#define ADDR_READ addr_read
	112	#endif
b92e5a22	113
c27004ec	114	DATA_TYPE REGPARM(1) glue(glue(__ld, SUFFIX), MMUSUFFIX)(target_ulong addr,
61382a50	115	int is_user);
c27004ec	116	void REGPARM(2) glue(glue(__st, SUFFIX), MMUSUFFIX)(target_ulong addr, DATA_TYPE v, int is_user);
b92e5a22	117
c27004ec FB	118	#if (DATA_SIZE <= 4) && (TARGET_LONG_BITS == 32) && defined(__i386__) && \
c27004ec FB	119	(ACCESS_TYPE <= 1) && defined(ASM_SOFTMMU)
e16c53fa	120
84b7b8e7 FB	121	#define CPU_TLB_ENTRY_BITS 4
84b7b8e7 FB	122
c27004ec	123	static inline RES_TYPE glue(glue(ld, USUFFIX), MEMSUFFIX)(target_ulong ptr)
e16c53fa FB	124	{
	125	int res;
	126
	127	asm volatile ("movl %1, %%edx\n"
	128	"movl %1, %%eax\n"
	129	"shrl %3, %%edx\n"
	130	"andl %4, %%eax\n"
	131	"andl %2, %%edx\n"
	132	"leal %5(%%edx, %%ebp), %%edx\n"
	133	"cmpl (%%edx), %%eax\n"
	134	"movl %1, %%eax\n"
	135	"je 1f\n"
	136	"pushl %6\n"
	137	"call %7\n"
	138	"popl %%edx\n"
	139	"movl %%eax, %0\n"
	140	"jmp 2f\n"
	141	"1:\n"
84b7b8e7	142	"addl 12(%%edx), %%eax\n"
e16c53fa FB	143	#if DATA_SIZE == 1
	144	"movzbl (%%eax), %0\n"
	145	#elif DATA_SIZE == 2
	146	"movzwl (%%eax), %0\n"
	147	#elif DATA_SIZE == 4
	148	"movl (%%eax), %0\n"
	149	#else
	150	#error unsupported size
	151	#endif
	152	"2:\n"
	153	: "=r" (res)
5fafdf24 TS	154	: "r" (ptr),
	155	"i" ((CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS),
	156	"i" (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
e16c53fa	157	"i" (TARGET_PAGE_MASK \| (DATA_SIZE - 1)),
84b7b8e7	158	"m" ((uint32_t )offsetof(CPUState, tlb_table[CPU_MEM_INDEX][0].addr_read)),
e16c53fa FB	159	"i" (CPU_MEM_INDEX),
	160	"m" ((uint8_t )&glue(glue(__ld, SUFFIX), MMUSUFFIX))
	161	: "%eax", "%ecx", "%edx", "memory", "cc");
	162	return res;
	163	}
	164
	165	#if DATA_SIZE <= 2
c27004ec	166	static inline int glue(glue(lds, SUFFIX), MEMSUFFIX)(target_ulong ptr)
e16c53fa FB	167	{
	168	int res;
	169
	170	asm volatile ("movl %1, %%edx\n"
	171	"movl %1, %%eax\n"
	172	"shrl %3, %%edx\n"
	173	"andl %4, %%eax\n"
	174	"andl %2, %%edx\n"
	175	"leal %5(%%edx, %%ebp), %%edx\n"
	176	"cmpl (%%edx), %%eax\n"
	177	"movl %1, %%eax\n"
	178	"je 1f\n"
	179	"pushl %6\n"
	180	"call %7\n"
	181	"popl %%edx\n"
	182	#if DATA_SIZE == 1
	183	"movsbl %%al, %0\n"
	184	#elif DATA_SIZE == 2
	185	"movswl %%ax, %0\n"
	186	#else
	187	#error unsupported size
	188	#endif
	189	"jmp 2f\n"
	190	"1:\n"
84b7b8e7	191	"addl 12(%%edx), %%eax\n"
e16c53fa FB	192	#if DATA_SIZE == 1
	193	"movsbl (%%eax), %0\n"
	194	#elif DATA_SIZE == 2
	195	"movswl (%%eax), %0\n"
	196	#else
	197	#error unsupported size
	198	#endif
	199	"2:\n"
	200	: "=r" (res)
5fafdf24 TS	201	: "r" (ptr),
	202	"i" ((CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS),
	203	"i" (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
e16c53fa	204	"i" (TARGET_PAGE_MASK \| (DATA_SIZE - 1)),
84b7b8e7	205	"m" ((uint32_t )offsetof(CPUState, tlb_table[CPU_MEM_INDEX][0].addr_read)),
e16c53fa FB	206	"i" (CPU_MEM_INDEX),
	207	"m" ((uint8_t )&glue(glue(__ld, SUFFIX), MMUSUFFIX))
	208	: "%eax", "%ecx", "%edx", "memory", "cc");
	209	return res;
	210	}
	211	#endif
	212
c27004ec	213	static inline void glue(glue(st, SUFFIX), MEMSUFFIX)(target_ulong ptr, RES_TYPE v)
e16c53fa FB	214	{
	215	asm volatile ("movl %0, %%edx\n"
	216	"movl %0, %%eax\n"
	217	"shrl %3, %%edx\n"
	218	"andl %4, %%eax\n"
	219	"andl %2, %%edx\n"
	220	"leal %5(%%edx, %%ebp), %%edx\n"
	221	"cmpl (%%edx), %%eax\n"
	222	"movl %0, %%eax\n"
	223	"je 1f\n"
	224	#if DATA_SIZE == 1
	225	"movzbl %b1, %%edx\n"
	226	#elif DATA_SIZE == 2
	227	"movzwl %w1, %%edx\n"
	228	#elif DATA_SIZE == 4
	229	"movl %1, %%edx\n"
	230	#else
	231	#error unsupported size
	232	#endif
	233	"pushl %6\n"
	234	"call %7\n"
	235	"popl %%eax\n"
	236	"jmp 2f\n"
	237	"1:\n"
84b7b8e7	238	"addl 8(%%edx), %%eax\n"
e16c53fa FB	239	#if DATA_SIZE == 1
	240	"movb %b1, (%%eax)\n"
	241	#elif DATA_SIZE == 2
	242	"movw %w1, (%%eax)\n"
	243	#elif DATA_SIZE == 4
	244	"movl %1, (%%eax)\n"
	245	#else
	246	#error unsupported size
	247	#endif
	248	"2:\n"
5fafdf24 TS	249	:
5fafdf24 TS	250	: "r" (ptr),
e16c53fa FB	251	/* NOTE: 'q' would be needed as constraint, but we could not use it
e16c53fa FB	252	with T1 ! */
5fafdf24 TS	253	"r" (v),
	254	"i" ((CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS),
	255	"i" (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
e16c53fa	256	"i" (TARGET_PAGE_MASK \| (DATA_SIZE - 1)),
84b7b8e7	257	"m" ((uint32_t )offsetof(CPUState, tlb_table[CPU_MEM_INDEX][0].addr_write)),
e16c53fa FB	258	"i" (CPU_MEM_INDEX),
	259	"m" ((uint8_t )&glue(glue(__st, SUFFIX), MMUSUFFIX))
	260	: "%eax", "%ecx", "%edx", "memory", "cc");
	261	}
	262
	263	#else
	264
	265	/* generic load/store macros */
	266
c27004ec	267	static inline RES_TYPE glue(glue(ld, USUFFIX), MEMSUFFIX)(target_ulong ptr)
b92e5a22 FB	268	{
	269	int index;
	270	RES_TYPE res;
c27004ec FB	271	target_ulong addr;
c27004ec FB	272	unsigned long physaddr;
61382a50 FB	273	int is_user;
61382a50 FB	274
c27004ec	275	addr = ptr;
b92e5a22	276	index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
61382a50	277	is_user = CPU_MEM_INDEX;
5fafdf24	278	if (__builtin_expect(env->tlb_table[is_user][index].ADDR_READ !=
b92e5a22	279	(addr & (TARGET_PAGE_MASK \| (DATA_SIZE - 1))), 0)) {
61382a50	280	res = glue(glue(__ld, SUFFIX), MMUSUFFIX)(addr, is_user);
b92e5a22	281	} else {
84b7b8e7	282	physaddr = addr + env->tlb_table[is_user][index].addend;
61382a50	283	res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)physaddr);
b92e5a22 FB	284	}
	285	return res;
	286	}
	287
	288	#if DATA_SIZE <= 2
c27004ec	289	static inline int glue(glue(lds, SUFFIX), MEMSUFFIX)(target_ulong ptr)
b92e5a22 FB	290	{
b92e5a22 FB	291	int res, index;
c27004ec FB	292	target_ulong addr;
c27004ec FB	293	unsigned long physaddr;
61382a50 FB	294	int is_user;
61382a50 FB	295
c27004ec	296	addr = ptr;
b92e5a22	297	index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
61382a50	298	is_user = CPU_MEM_INDEX;
5fafdf24	299	if (__builtin_expect(env->tlb_table[is_user][index].ADDR_READ !=
b92e5a22	300	(addr & (TARGET_PAGE_MASK \| (DATA_SIZE - 1))), 0)) {
61382a50	301	res = (DATA_STYPE)glue(glue(__ld, SUFFIX), MMUSUFFIX)(addr, is_user);
b92e5a22	302	} else {
84b7b8e7	303	physaddr = addr + env->tlb_table[is_user][index].addend;
b92e5a22 FB	304	res = glue(glue(lds, SUFFIX), _raw)((uint8_t *)physaddr);
	305	}
	306	return res;
	307	}
	308	#endif
	309
84b7b8e7 FB	310	#if ACCESS_TYPE != 3
84b7b8e7 FB	311
e16c53fa FB	312	/* generic store macro */
e16c53fa FB	313
c27004ec	314	static inline void glue(glue(st, SUFFIX), MEMSUFFIX)(target_ulong ptr, RES_TYPE v)
b92e5a22 FB	315	{
b92e5a22 FB	316	int index;
c27004ec FB	317	target_ulong addr;
c27004ec FB	318	unsigned long physaddr;
61382a50 FB	319	int is_user;
61382a50 FB	320
c27004ec	321	addr = ptr;
b92e5a22	322	index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
61382a50	323	is_user = CPU_MEM_INDEX;
5fafdf24	324	if (__builtin_expect(env->tlb_table[is_user][index].addr_write !=
b92e5a22	325	(addr & (TARGET_PAGE_MASK \| (DATA_SIZE - 1))), 0)) {
61382a50	326	glue(glue(__st, SUFFIX), MMUSUFFIX)(addr, v, is_user);
b92e5a22	327	} else {
84b7b8e7	328	physaddr = addr + env->tlb_table[is_user][index].addend;
b92e5a22 FB	329	glue(glue(st, SUFFIX), _raw)((uint8_t *)physaddr, v);
	330	}
	331	}
	332
84b7b8e7 FB	333	#endif /* ACCESS_TYPE != 3 */
	334
	335	#endif /* !asm */
	336
	337	#if ACCESS_TYPE != 3
e16c53fa	338
2d603d22	339	#if DATA_SIZE == 8
3f87bf69	340	static inline float64 glue(ldfq, MEMSUFFIX)(target_ulong ptr)
2d603d22 FB	341	{
2d603d22 FB	342	union {
3f87bf69	343	float64 d;
2d603d22 FB	344	uint64_t i;
	345	} u;
	346	u.i = glue(ldq, MEMSUFFIX)(ptr);
	347	return u.d;
	348	}
	349
3f87bf69	350	static inline void glue(stfq, MEMSUFFIX)(target_ulong ptr, float64 v)
2d603d22 FB	351	{
2d603d22 FB	352	union {
3f87bf69	353	float64 d;
2d603d22 FB	354	uint64_t i;
	355	} u;
	356	u.d = v;
	357	glue(stq, MEMSUFFIX)(ptr, u.i);
	358	}
	359	#endif /* DATA_SIZE == 8 */
	360
	361	#if DATA_SIZE == 4
3f87bf69	362	static inline float32 glue(ldfl, MEMSUFFIX)(target_ulong ptr)
2d603d22 FB	363	{
2d603d22 FB	364	union {
3f87bf69	365	float32 f;
2d603d22 FB	366	uint32_t i;
	367	} u;
	368	u.i = glue(ldl, MEMSUFFIX)(ptr);
	369	return u.f;
	370	}
	371
3f87bf69	372	static inline void glue(stfl, MEMSUFFIX)(target_ulong ptr, float32 v)
2d603d22 FB	373	{
2d603d22 FB	374	union {
3f87bf69	375	float32 f;
2d603d22 FB	376	uint32_t i;
	377	} u;
	378	u.f = v;
	379	glue(stl, MEMSUFFIX)(ptr, u.i);
	380	}
	381	#endif /* DATA_SIZE == 4 */
	382
84b7b8e7 FB	383	#endif /* ACCESS_TYPE != 3 */
84b7b8e7 FB	384
b92e5a22 FB	385	#undef RES_TYPE
	386	#undef DATA_TYPE
	387	#undef DATA_STYPE
	388	#undef SUFFIX
61382a50	389	#undef USUFFIX
b92e5a22	390	#undef DATA_SIZE
61382a50 FB	391	#undef CPU_MEM_INDEX
61382a50 FB	392	#undef MMUSUFFIX
84b7b8e7	393	#undef ADDR_READ