[qemu.git] / cputlb.c

/*
 *  Common CPU TLB handling
 *
 *  Copyright (c) 2003 Fabrice Bellard
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */

#include "qemu/osdep.h"
#include "cpu.h"
#include "exec/exec-all.h"
#include "exec/memory.h"
#include "exec/address-spaces.h"
#include "exec/cpu_ldst.h"

#include "exec/cputlb.h"

#include "exec/memory-internal.h"
#include "exec/ram_addr.h"
#include "exec/exec-all.h"
#include "tcg/tcg.h"
#include "qemu/error-report.h"
#include "exec/log.h"

/* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
/* #define DEBUG_TLB */
/* #define DEBUG_TLB_LOG */

#ifdef DEBUG_TLB
# define DEBUG_TLB_GATE 1
# ifdef DEBUG_TLB_LOG
#  define DEBUG_TLB_LOG_GATE 1
# else
#  define DEBUG_TLB_LOG_GATE 0
# endif
#else
# define DEBUG_TLB_GATE 0
# define DEBUG_TLB_LOG_GATE 0
#endif

#define tlb_debug(fmt, ...) do { \
    if (DEBUG_TLB_LOG_GATE) { \
        qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
                      ## __VA_ARGS__); \
    } else if (DEBUG_TLB_GATE) { \
        fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
    } \
} while (0)

/* statistics */
int tlb_flush_count;

/* NOTE:
 * If flush_global is true (the usual case), flush all tlb entries.
 * If flush_global is false, flush (at least) all tlb entries not
 * marked global.
 *
 * Since QEMU doesn't currently implement a global/not-global flag
 * for tlb entries, at the moment tlb_flush() will also flush all
 * tlb entries in the flush_global == false case. This is OK because
 * CPU architectures generally permit an implementation to drop
 * entries from the TLB at any time, so flushing more entries than
 * required is only an efficiency issue, not a correctness issue.
 */
void tlb_flush(CPUState *cpu, int flush_global)
{
    CPUArchState *env = cpu->env_ptr;

    tlb_debug("(%d)\n", flush_global);

    memset(env->tlb_table, -1, sizeof(env->tlb_table));
    memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
    memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));

    env->vtlb_index = 0;
    env->tlb_flush_addr = -1;
    env->tlb_flush_mask = 0;
    tlb_flush_count++;
}

static inline void v_tlb_flush_by_mmuidx(CPUState *cpu, va_list argp)
{
    CPUArchState *env = cpu->env_ptr;

    tlb_debug("start\n");

    for (;;) {
        int mmu_idx = va_arg(argp, int);

        if (mmu_idx < 0) {
            break;
        }

        tlb_debug("%d\n", mmu_idx);

        memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
        memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
    }

    memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
}

void tlb_flush_by_mmuidx(CPUState *cpu, ...)
{
    va_list argp;
    va_start(argp, cpu);
    v_tlb_flush_by_mmuidx(cpu, argp);
    va_end(argp);
}

static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
{
    if (addr == (tlb_entry->addr_read &
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
        addr == (tlb_entry->addr_write &
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
        addr == (tlb_entry->addr_code &
                 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
        memset(tlb_entry, -1, sizeof(*tlb_entry));
    }
}

void tlb_flush_page(CPUState *cpu, target_ulong addr)
{
    CPUArchState *env = cpu->env_ptr;
    int i;
    int mmu_idx;

    tlb_debug("page :" TARGET_FMT_lx "\n", addr);

    /* Check if we need to flush due to large pages.  */
    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
        tlb_debug("forcing full flush ("
                  TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
                  env->tlb_flush_addr, env->tlb_flush_mask);

        tlb_flush(cpu, 1);
        return;
    }

    addr &= TARGET_PAGE_MASK;
    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
    }

    /* check whether there are entries that need to be flushed in the vtlb */
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
        int k;
        for (k = 0; k < CPU_VTLB_SIZE; k++) {
            tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
        }
    }

    tb_flush_jmp_cache(cpu, addr);
}

void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...)
{
    CPUArchState *env = cpu->env_ptr;
    int i, k;
    va_list argp;

    va_start(argp, addr);

    tlb_debug("addr "TARGET_FMT_lx"\n", addr);

    /* Check if we need to flush due to large pages.  */
    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
        tlb_debug("forced full flush ("
                  TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
                  env->tlb_flush_addr, env->tlb_flush_mask);

        v_tlb_flush_by_mmuidx(cpu, argp);
        va_end(argp);
        return;
    }

    addr &= TARGET_PAGE_MASK;
    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);

    for (;;) {
        int mmu_idx = va_arg(argp, int);

        if (mmu_idx < 0) {
            break;
        }

        tlb_debug("idx %d\n", mmu_idx);

        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);

        /* check whether there are vltb entries that need to be flushed */
        for (k = 0; k < CPU_VTLB_SIZE; k++) {
            tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
        }
    }
    va_end(argp);

    tb_flush_jmp_cache(cpu, addr);
}

/* update the TLBs so that writes to code in the virtual page 'addr'
   can be detected */
void tlb_protect_code(ram_addr_t ram_addr)
{
    cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE,
                                             DIRTY_MEMORY_CODE);
}

/* update the TLB so that writes in physical page 'phys_addr' are no longer
   tested for self modifying code */
void tlb_unprotect_code(ram_addr_t ram_addr)
{
    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
}

static bool tlb_is_dirty_ram(CPUTLBEntry *tlbe)
{
    return (tlbe->addr_write & (TLB_INVALID_MASK|TLB_MMIO|TLB_NOTDIRTY)) == 0;
}

void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
                           uintptr_t length)
{
    uintptr_t addr;

    if (tlb_is_dirty_ram(tlb_entry)) {
        addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
        if ((addr - start) < length) {
            tlb_entry->addr_write |= TLB_NOTDIRTY;
        }
    }
}

static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
{
    ram_addr_t ram_addr;

    ram_addr = qemu_ram_addr_from_host(ptr);
    if (ram_addr == RAM_ADDR_INVALID) {
        fprintf(stderr, "Bad ram pointer %p\n", ptr);
        abort();
    }
    return ram_addr;
}

void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
{
    CPUArchState *env;

    int mmu_idx;

    env = cpu->env_ptr;
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
        unsigned int i;

        for (i = 0; i < CPU_TLB_SIZE; i++) {
            tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
                                  start1, length);
        }

        for (i = 0; i < CPU_VTLB_SIZE; i++) {
            tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
                                  start1, length);
        }
    }
}

static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
{
    if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
        tlb_entry->addr_write = vaddr;
    }
}

/* update the TLB corresponding to virtual page vaddr
   so that it is no longer dirty */
void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
{
    CPUArchState *env = cpu->env_ptr;
    int i;
    int mmu_idx;

    vaddr &= TARGET_PAGE_MASK;
    i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
        tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
    }

    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
        int k;
        for (k = 0; k < CPU_VTLB_SIZE; k++) {
            tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
        }
    }
}

/* Our TLB does not support large pages, so remember the area covered by
   large pages and trigger a full TLB flush if these are invalidated.  */
static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr,
                               target_ulong size)
{
    target_ulong mask = ~(size - 1);

    if (env->tlb_flush_addr == (target_ulong)-1) {
        env->tlb_flush_addr = vaddr & mask;
        env->tlb_flush_mask = mask;
        return;
    }
    /* Extend the existing region to include the new page.
       This is a compromise between unnecessary flushes and the cost
       of maintaining a full variable size TLB.  */
    mask &= env->tlb_flush_mask;
    while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
        mask <<= 1;
    }
    env->tlb_flush_addr &= mask;
    env->tlb_flush_mask = mask;
}

/* Add a new TLB entry. At most one entry for a given virtual address
 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
 * supplied size is only used by tlb_flush_page.
 *
 * Called from TCG-generated code, which is under an RCU read-side
 * critical section.
 */
void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
                             hwaddr paddr, MemTxAttrs attrs, int prot,
                             int mmu_idx, target_ulong size)
{
    CPUArchState *env = cpu->env_ptr;
    MemoryRegionSection *section;
    unsigned int index;
    target_ulong address;
    target_ulong code_address;
    uintptr_t addend;
    CPUTLBEntry *te;
    hwaddr iotlb, xlat, sz;
    unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
    int asidx = cpu_asidx_from_attrs(cpu, attrs);

    assert(size >= TARGET_PAGE_SIZE);
    if (size != TARGET_PAGE_SIZE) {
        tlb_add_large_page(env, vaddr, size);
    }

    sz = size;
    section = address_space_translate_for_iotlb(cpu, asidx, paddr, &xlat, &sz);
    assert(sz >= TARGET_PAGE_SIZE);

    tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
              " prot=%x idx=%d\n",
              vaddr, paddr, prot, mmu_idx);

    address = vaddr;
    if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) {
        /* IO memory case */
        address |= TLB_MMIO;
        addend = 0;
    } else {
        /* TLB_MMIO for rom/romd handled below */
        addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
    }

    code_address = address;
    iotlb = memory_region_section_get_iotlb(cpu, section, vaddr, paddr, xlat,
                                            prot, &address);

    index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    te = &env->tlb_table[mmu_idx][index];

    /* do not discard the translation in te, evict it into a victim tlb */
    env->tlb_v_table[mmu_idx][vidx] = *te;
    env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];

    /* refill the tlb */
    env->iotlb[mmu_idx][index].addr = iotlb - vaddr;
    env->iotlb[mmu_idx][index].attrs = attrs;
    te->addend = addend - vaddr;
    if (prot & PAGE_READ) {
        te->addr_read = address;
    } else {
        te->addr_read = -1;
    }

    if (prot & PAGE_EXEC) {
        te->addr_code = code_address;
    } else {
        te->addr_code = -1;
    }
    if (prot & PAGE_WRITE) {
        if ((memory_region_is_ram(section->mr) && section->readonly)
            || memory_region_is_romd(section->mr)) {
            /* Write access calls the I/O callback.  */
            te->addr_write = address | TLB_MMIO;
        } else if (memory_region_is_ram(section->mr)
                   && cpu_physical_memory_is_clean(
                        memory_region_get_ram_addr(section->mr) + xlat)) {
            te->addr_write = address | TLB_NOTDIRTY;
        } else {
            te->addr_write = address;
        }
    } else {
        te->addr_write = -1;
    }
}

/* Add a new TLB entry, but without specifying the memory
 * transaction attributes to be used.
 */
void tlb_set_page(CPUState *cpu, target_ulong vaddr,
                  hwaddr paddr, int prot,
                  int mmu_idx, target_ulong size)
{
    tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED,
                            prot, mmu_idx, size);
}

static void report_bad_exec(CPUState *cpu, target_ulong addr)
{
    /* Accidentally executing outside RAM or ROM is quite common for
     * several user-error situations, so report it in a way that
     * makes it clear that this isn't a QEMU bug and provide suggestions
     * about what a user could do to fix things.
     */
    error_report("Trying to execute code outside RAM or ROM at 0x"
                 TARGET_FMT_lx, addr);
    error_printf("This usually means one of the following happened:\n\n"
                 "(1) You told QEMU to execute a kernel for the wrong machine "
                 "type, and it crashed on startup (eg trying to run a "
                 "raspberry pi kernel on a versatilepb QEMU machine)\n"
                 "(2) You didn't give QEMU a kernel or BIOS filename at all, "
                 "and QEMU executed a ROM full of no-op instructions until "
                 "it fell off the end\n"
                 "(3) Your guest kernel has a bug and crashed by jumping "
                 "off into nowhere\n\n"
                 "This is almost always one of the first two, so check your "
                 "command line and that you are using the right type of kernel "
                 "for this machine.\n"
                 "If you think option (3) is likely then you can try debugging "
                 "your guest with the -d debug options; in particular "
                 "-d guest_errors will cause the log to include a dump of the "
                 "guest register state at this point.\n\n"
                 "Execution cannot continue; stopping here.\n\n");

    /* Report also to the logs, with more detail including register dump */
    qemu_log_mask(LOG_GUEST_ERROR, "qemu: fatal: Trying to execute code "
                  "outside RAM or ROM at 0x" TARGET_FMT_lx "\n", addr);
    log_cpu_state_mask(LOG_GUEST_ERROR, cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
}

/* NOTE: this function can trigger an exception */
/* NOTE2: the returned address is not exactly the physical address: it
 * is actually a ram_addr_t (in system mode; the user mode emulation
 * version of this function returns a guest virtual address).
 */
tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
{
    int mmu_idx, page_index, pd;
    void *p;
    MemoryRegion *mr;
    CPUState *cpu = ENV_GET_CPU(env1);
    CPUIOTLBEntry *iotlbentry;

    page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
    mmu_idx = cpu_mmu_index(env1, true);
    if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
                 (addr & TARGET_PAGE_MASK))) {
        cpu_ldub_code(env1, addr);
    }
    iotlbentry = &env1->iotlb[mmu_idx][page_index];
    pd = iotlbentry->addr & ~TARGET_PAGE_MASK;
    mr = iotlb_to_region(cpu, pd, iotlbentry->attrs);
    if (memory_region_is_unassigned(mr)) {
        CPUClass *cc = CPU_GET_CLASS(cpu);

        if (cc->do_unassigned_access) {
            cc->do_unassigned_access(cpu, addr, false, true, 0, 4);
        } else {
            report_bad_exec(cpu, addr);
            exit(1);
        }
    }
    p = (void *)((uintptr_t)addr + env1->tlb_table[mmu_idx][page_index].addend);
    return qemu_ram_addr_from_host_nofail(p);
}

/* Return true if ADDR is present in the victim tlb, and has been copied
   back to the main tlb.  */
static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
                           size_t elt_ofs, target_ulong page)
{
    size_t vidx;
    for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
        CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
        target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);

        if (cmp == page) {
            /* Found entry in victim tlb, swap tlb and iotlb.  */
            CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];
            CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];
            CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];

            tmptlb = *tlb; *tlb = *vtlb; *vtlb = tmptlb;
            tmpio = *io; *io = *vio; *vio = tmpio;
            return true;
        }
    }
    return false;
}

/* Macro to call the above, with local variables from the use context.  */
#define VICTIM_TLB_HIT(TY, ADDR) \
  victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
                 (ADDR) & TARGET_PAGE_MASK)

#define MMUSUFFIX _mmu

#define SHIFT 0
#include "softmmu_template.h"

#define SHIFT 1
#include "softmmu_template.h"

#define SHIFT 2
#include "softmmu_template.h"

#define SHIFT 3
#include "softmmu_template.h"
#undef MMUSUFFIX

#define MMUSUFFIX _cmmu
#undef GETPC
#define GETPC() ((uintptr_t)0)
#define SOFTMMU_CODE_ACCESS

#define SHIFT 0
#include "softmmu_template.h"

#define SHIFT 1
#include "softmmu_template.h"

#define SHIFT 2
#include "softmmu_template.h"

#define SHIFT 3
#include "softmmu_template.h"
Commit	Line	Data
0cac1b66 BS	1	/*
	2	* Common CPU TLB handling
	3	*
	4	* Copyright (c) 2003 Fabrice Bellard
	5	*
	6	* This library is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU Lesser General Public
	8	* License as published by the Free Software Foundation; either
	9	* version 2 of the License, or (at your option) any later version.
	10	*
	11	* This library is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* Lesser General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU Lesser General Public
	17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
	18	*/
	19
7b31bbc2	20	#include "qemu/osdep.h"
0cac1b66	21	#include "cpu.h"
022c62cb PB	22	#include "exec/exec-all.h"
	23	#include "exec/memory.h"
	24	#include "exec/address-spaces.h"
f08b6170	25	#include "exec/cpu_ldst.h"
0cac1b66	26
022c62cb	27	#include "exec/cputlb.h"
0cac1b66	28
022c62cb	29	#include "exec/memory-internal.h"
220c3ebd	30	#include "exec/ram_addr.h"
63c91552	31	#include "exec/exec-all.h"
0f590e74	32	#include "tcg/tcg.h"
d7f30403 PM	33	#include "qemu/error-report.h"
d7f30403 PM	34	#include "exec/log.h"
0cac1b66	35
8526e1f4 AB	36	/* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
	37	/* #define DEBUG_TLB */
	38	/* #define DEBUG_TLB_LOG */
	39
	40	#ifdef DEBUG_TLB
	41	# define DEBUG_TLB_GATE 1
	42	# ifdef DEBUG_TLB_LOG
	43	# define DEBUG_TLB_LOG_GATE 1
	44	# else
	45	# define DEBUG_TLB_LOG_GATE 0
	46	# endif
	47	#else
	48	# define DEBUG_TLB_GATE 0
	49	# define DEBUG_TLB_LOG_GATE 0
	50	#endif
	51
	52	#define tlb_debug(fmt, ...) do { \
	53	if (DEBUG_TLB_LOG_GATE) { \
	54	qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
	55	## __VA_ARGS__); \
	56	} else if (DEBUG_TLB_GATE) { \
	57	fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
	58	} \
	59	} while (0)
0cac1b66 BS	60
	61	/* statistics */
	62	int tlb_flush_count;
	63
0cac1b66 BS	64	/* NOTE:
	65	* If flush_global is true (the usual case), flush all tlb entries.
	66	* If flush_global is false, flush (at least) all tlb entries not
	67	* marked global.
	68	*
	69	* Since QEMU doesn't currently implement a global/not-global flag
	70	* for tlb entries, at the moment tlb_flush() will also flush all
	71	* tlb entries in the flush_global == false case. This is OK because
	72	* CPU architectures generally permit an implementation to drop
	73	* entries from the TLB at any time, so flushing more entries than
	74	* required is only an efficiency issue, not a correctness issue.
	75	*/
00c8cb0a	76	void tlb_flush(CPUState *cpu, int flush_global)
0cac1b66	77	{
00c8cb0a	78	CPUArchState *env = cpu->env_ptr;
0cac1b66	79
8526e1f4 AB	80	tlb_debug("(%d)\n", flush_global);
8526e1f4 AB	81
4fadb3bb	82	memset(env->tlb_table, -1, sizeof(env->tlb_table));
88e89a57	83	memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
8cd70437	84	memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
0cac1b66	85
88e89a57	86	env->vtlb_index = 0;
0cac1b66 BS	87	env->tlb_flush_addr = -1;
	88	env->tlb_flush_mask = 0;
	89	tlb_flush_count++;
	90	}
	91
d7a74a9d PM	92	static inline void v_tlb_flush_by_mmuidx(CPUState *cpu, va_list argp)
	93	{
	94	CPUArchState *env = cpu->env_ptr;
	95
8526e1f4	96	tlb_debug("start\n");
d7a74a9d PM	97
	98	for (;;) {
	99	int mmu_idx = va_arg(argp, int);
	100
	101	if (mmu_idx < 0) {
	102	break;
	103	}
	104
8526e1f4	105	tlb_debug("%d\n", mmu_idx);
d7a74a9d PM	106
	107	memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
	108	memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
	109	}
	110
d7a74a9d PM	111	memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
	112	}
	113
	114	void tlb_flush_by_mmuidx(CPUState *cpu, ...)
	115	{
	116	va_list argp;
	117	va_start(argp, cpu);
	118	v_tlb_flush_by_mmuidx(cpu, argp);
	119	va_end(argp);
	120	}
	121
0cac1b66 BS	122	static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
	123	{
	124	if (addr == (tlb_entry->addr_read &
	125	(TARGET_PAGE_MASK \| TLB_INVALID_MASK)) \|\|
	126	addr == (tlb_entry->addr_write &
	127	(TARGET_PAGE_MASK \| TLB_INVALID_MASK)) \|\|
	128	addr == (tlb_entry->addr_code &
	129	(TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
4fadb3bb	130	memset(tlb_entry, -1, sizeof(*tlb_entry));
0cac1b66 BS	131	}
	132	}
	133
31b030d4	134	void tlb_flush_page(CPUState *cpu, target_ulong addr)
0cac1b66	135	{
31b030d4	136	CPUArchState *env = cpu->env_ptr;
0cac1b66 BS	137	int i;
	138	int mmu_idx;
	139
8526e1f4 AB	140	tlb_debug("page :" TARGET_FMT_lx "\n", addr);
8526e1f4 AB	141
0cac1b66 BS	142	/* Check if we need to flush due to large pages. */
0cac1b66 BS	143	if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
8526e1f4 AB	144	tlb_debug("forcing full flush ("
	145	TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
	146	env->tlb_flush_addr, env->tlb_flush_mask);
	147
00c8cb0a	148	tlb_flush(cpu, 1);
0cac1b66 BS	149	return;
0cac1b66 BS	150	}
0cac1b66 BS	151
	152	addr &= TARGET_PAGE_MASK;
	153	i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
	154	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
	155	tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
	156	}
	157
88e89a57 XT	158	/* check whether there are entries that need to be flushed in the vtlb */
	159	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
	160	int k;
	161	for (k = 0; k < CPU_VTLB_SIZE; k++) {
	162	tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
	163	}
	164	}
	165
611d4f99	166	tb_flush_jmp_cache(cpu, addr);
0cac1b66 BS	167	}
0cac1b66 BS	168
d7a74a9d PM	169	void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...)
	170	{
	171	CPUArchState *env = cpu->env_ptr;
	172	int i, k;
	173	va_list argp;
	174
	175	va_start(argp, addr);
	176
8526e1f4 AB	177	tlb_debug("addr "TARGET_FMT_lx"\n", addr);
8526e1f4 AB	178
d7a74a9d PM	179	/* Check if we need to flush due to large pages. */
d7a74a9d PM	180	if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
8526e1f4 AB	181	tlb_debug("forced full flush ("
	182	TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
	183	env->tlb_flush_addr, env->tlb_flush_mask);
	184
d7a74a9d PM	185	v_tlb_flush_by_mmuidx(cpu, argp);
	186	va_end(argp);
	187	return;
	188	}
d7a74a9d PM	189
	190	addr &= TARGET_PAGE_MASK;
	191	i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
	192
	193	for (;;) {
	194	int mmu_idx = va_arg(argp, int);
	195
	196	if (mmu_idx < 0) {
	197	break;
	198	}
	199
8526e1f4	200	tlb_debug("idx %d\n", mmu_idx);
d7a74a9d PM	201
	202	tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
	203
	204	/* check whether there are vltb entries that need to be flushed */
	205	for (k = 0; k < CPU_VTLB_SIZE; k++) {
	206	tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
	207	}
	208	}
	209	va_end(argp);
	210
d7a74a9d PM	211	tb_flush_jmp_cache(cpu, addr);
	212	}
	213
0cac1b66 BS	214	/* update the TLBs so that writes to code in the virtual page 'addr'
	215	can be detected */
	216	void tlb_protect_code(ram_addr_t ram_addr)
	217	{
03eebc9e SH	218	cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE,
03eebc9e SH	219	DIRTY_MEMORY_CODE);
0cac1b66 BS	220	}
	221
	222	/* update the TLB so that writes in physical page 'phys_addr' are no longer
	223	tested for self modifying code */
9564f52d	224	void tlb_unprotect_code(ram_addr_t ram_addr)
0cac1b66	225	{
52159192	226	cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
0cac1b66 BS	227	}
	228
	229	static bool tlb_is_dirty_ram(CPUTLBEntry *tlbe)
	230	{
	231	return (tlbe->addr_write & (TLB_INVALID_MASK\|TLB_MMIO\|TLB_NOTDIRTY)) == 0;
	232	}
	233
	234	void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
	235	uintptr_t length)
	236	{
	237	uintptr_t addr;
	238
	239	if (tlb_is_dirty_ram(tlb_entry)) {
	240	addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
	241	if ((addr - start) < length) {
	242	tlb_entry->addr_write \|= TLB_NOTDIRTY;
	243	}
	244	}
	245	}
	246
7443b437 PB	247	static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
	248	{
	249	ram_addr_t ram_addr;
	250
07bdaa41 PB	251	ram_addr = qemu_ram_addr_from_host(ptr);
07bdaa41 PB	252	if (ram_addr == RAM_ADDR_INVALID) {
7443b437 PB	253	fprintf(stderr, "Bad ram pointer %p\n", ptr);
	254	abort();
	255	}
	256	return ram_addr;
	257	}
	258
9a13565d	259	void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
0cac1b66 BS	260	{
	261	CPUArchState *env;
	262
9a13565d	263	int mmu_idx;
0cac1b66	264
9a13565d PC	265	env = cpu->env_ptr;
	266	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
	267	unsigned int i;
0cac1b66	268
9a13565d PC	269	for (i = 0; i < CPU_TLB_SIZE; i++) {
	270	tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
	271	start1, length);
	272	}
88e89a57	273
9a13565d PC	274	for (i = 0; i < CPU_VTLB_SIZE; i++) {
	275	tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
	276	start1, length);
0cac1b66 BS	277	}
	278	}
	279	}
	280
	281	static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
	282	{
	283	if (tlb_entry->addr_write == (vaddr \| TLB_NOTDIRTY)) {
	284	tlb_entry->addr_write = vaddr;
	285	}
	286	}
	287
	288	/* update the TLB corresponding to virtual page vaddr
	289	so that it is no longer dirty */
bcae01e4	290	void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
0cac1b66	291	{
bcae01e4	292	CPUArchState *env = cpu->env_ptr;
0cac1b66 BS	293	int i;
	294	int mmu_idx;
	295
	296	vaddr &= TARGET_PAGE_MASK;
	297	i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
	298	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
	299	tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
	300	}
88e89a57 XT	301
	302	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
	303	int k;
	304	for (k = 0; k < CPU_VTLB_SIZE; k++) {
	305	tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
	306	}
	307	}
0cac1b66 BS	308	}
	309
	310	/* Our TLB does not support large pages, so remember the area covered by
	311	large pages and trigger a full TLB flush if these are invalidated. */
	312	static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr,
	313	target_ulong size)
	314	{
	315	target_ulong mask = ~(size - 1);
	316
	317	if (env->tlb_flush_addr == (target_ulong)-1) {
	318	env->tlb_flush_addr = vaddr & mask;
	319	env->tlb_flush_mask = mask;
	320	return;
	321	}
	322	/* Extend the existing region to include the new page.
	323	This is a compromise between unnecessary flushes and the cost
	324	of maintaining a full variable size TLB. */
	325	mask &= env->tlb_flush_mask;
	326	while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
	327	mask <<= 1;
	328	}
	329	env->tlb_flush_addr &= mask;
	330	env->tlb_flush_mask = mask;
	331	}
	332
	333	/* Add a new TLB entry. At most one entry for a given virtual address
79e2b9ae PB	334	* is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
	335	* supplied size is only used by tlb_flush_page.
	336	*
	337	* Called from TCG-generated code, which is under an RCU read-side
	338	* critical section.
	339	*/
fadc1cbe PM	340	void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
	341	hwaddr paddr, MemTxAttrs attrs, int prot,
	342	int mmu_idx, target_ulong size)
0cac1b66	343	{
0c591eb0	344	CPUArchState *env = cpu->env_ptr;
0cac1b66 BS	345	MemoryRegionSection *section;
	346	unsigned int index;
	347	target_ulong address;
	348	target_ulong code_address;
	349	uintptr_t addend;
	350	CPUTLBEntry *te;
149f54b5	351	hwaddr iotlb, xlat, sz;
88e89a57	352	unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
d7898cda	353	int asidx = cpu_asidx_from_attrs(cpu, attrs);
0cac1b66 BS	354
	355	assert(size >= TARGET_PAGE_SIZE);
	356	if (size != TARGET_PAGE_SIZE) {
	357	tlb_add_large_page(env, vaddr, size);
	358	}
149f54b5 PB	359
149f54b5 PB	360	sz = size;
d7898cda	361	section = address_space_translate_for_iotlb(cpu, asidx, paddr, &xlat, &sz);
149f54b5 PB	362	assert(sz >= TARGET_PAGE_SIZE);
149f54b5 PB	363
8526e1f4 AB	364	tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
	365	" prot=%x idx=%d\n",
	366	vaddr, paddr, prot, mmu_idx);
0cac1b66 BS	367
0cac1b66 BS	368	address = vaddr;
8f3e03cb PB	369	if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) {
8f3e03cb PB	370	/* IO memory case */
0cac1b66	371	address \|= TLB_MMIO;
8f3e03cb PB	372	addend = 0;
	373	} else {
	374	/* TLB_MMIO for rom/romd handled below */
149f54b5	375	addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
0cac1b66	376	}
0cac1b66 BS	377
0cac1b66 BS	378	code_address = address;
bb0e627a	379	iotlb = memory_region_section_get_iotlb(cpu, section, vaddr, paddr, xlat,
149f54b5	380	prot, &address);
0cac1b66 BS	381
0cac1b66 BS	382	index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
0cac1b66	383	te = &env->tlb_table[mmu_idx][index];
88e89a57 XT	384
	385	/* do not discard the translation in te, evict it into a victim tlb */
	386	env->tlb_v_table[mmu_idx][vidx] = *te;
	387	env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
	388
	389	/* refill the tlb */
e469b22f	390	env->iotlb[mmu_idx][index].addr = iotlb - vaddr;
fadc1cbe	391	env->iotlb[mmu_idx][index].attrs = attrs;
0cac1b66 BS	392	te->addend = addend - vaddr;
	393	if (prot & PAGE_READ) {
	394	te->addr_read = address;
	395	} else {
	396	te->addr_read = -1;
	397	}
	398
	399	if (prot & PAGE_EXEC) {
	400	te->addr_code = code_address;
	401	} else {
	402	te->addr_code = -1;
	403	}
	404	if (prot & PAGE_WRITE) {
	405	if ((memory_region_is_ram(section->mr) && section->readonly)
cc5bea60	406	\|\| memory_region_is_romd(section->mr)) {
0cac1b66 BS	407	/* Write access calls the I/O callback. */
	408	te->addr_write = address \| TLB_MMIO;
	409	} else if (memory_region_is_ram(section->mr)
8e41fb63 FZ	410	&& cpu_physical_memory_is_clean(
8e41fb63 FZ	411	memory_region_get_ram_addr(section->mr) + xlat)) {
0cac1b66 BS	412	te->addr_write = address \| TLB_NOTDIRTY;
	413	} else {
	414	te->addr_write = address;
	415	}
	416	} else {
	417	te->addr_write = -1;
	418	}
	419	}
	420
fadc1cbe PM	421	/* Add a new TLB entry, but without specifying the memory
	422	* transaction attributes to be used.
	423	*/
	424	void tlb_set_page(CPUState *cpu, target_ulong vaddr,
	425	hwaddr paddr, int prot,
	426	int mmu_idx, target_ulong size)
	427	{
	428	tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED,
	429	prot, mmu_idx, size);
	430	}
	431
d7f30403 PM	432	static void report_bad_exec(CPUState *cpu, target_ulong addr)
	433	{
	434	/* Accidentally executing outside RAM or ROM is quite common for
	435	* several user-error situations, so report it in a way that
	436	* makes it clear that this isn't a QEMU bug and provide suggestions
	437	* about what a user could do to fix things.
	438	*/
	439	error_report("Trying to execute code outside RAM or ROM at 0x"
	440	TARGET_FMT_lx, addr);
	441	error_printf("This usually means one of the following happened:\n\n"
	442	"(1) You told QEMU to execute a kernel for the wrong machine "
	443	"type, and it crashed on startup (eg trying to run a "
	444	"raspberry pi kernel on a versatilepb QEMU machine)\n"
	445	"(2) You didn't give QEMU a kernel or BIOS filename at all, "
	446	"and QEMU executed a ROM full of no-op instructions until "
	447	"it fell off the end\n"
	448	"(3) Your guest kernel has a bug and crashed by jumping "
	449	"off into nowhere\n\n"
	450	"This is almost always one of the first two, so check your "
	451	"command line and that you are using the right type of kernel "
	452	"for this machine.\n"
	453	"If you think option (3) is likely then you can try debugging "
	454	"your guest with the -d debug options; in particular "
	455	"-d guest_errors will cause the log to include a dump of the "
	456	"guest register state at this point.\n\n"
	457	"Execution cannot continue; stopping here.\n\n");
	458
	459	/* Report also to the logs, with more detail including register dump */
	460	qemu_log_mask(LOG_GUEST_ERROR, "qemu: fatal: Trying to execute code "
	461	"outside RAM or ROM at 0x" TARGET_FMT_lx "\n", addr);
	462	log_cpu_state_mask(LOG_GUEST_ERROR, cpu, CPU_DUMP_FPU \| CPU_DUMP_CCOP);
	463	}
	464
0cac1b66 BS	465	/* NOTE: this function can trigger an exception */
0cac1b66 BS	466	/* NOTE2: the returned address is not exactly the physical address: it
116aae36 PM	467	* is actually a ram_addr_t (in system mode; the user mode emulation
	468	* version of this function returns a guest virtual address).
	469	*/
0cac1b66 BS	470	tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
	471	{
	472	int mmu_idx, page_index, pd;
	473	void *p;
	474	MemoryRegion *mr;
09daed84	475	CPUState *cpu = ENV_GET_CPU(env1);
a54c87b6	476	CPUIOTLBEntry *iotlbentry;
0cac1b66 BS	477
0cac1b66 BS	478	page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
97ed5ccd	479	mmu_idx = cpu_mmu_index(env1, true);
0cac1b66 BS	480	if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
0cac1b66 BS	481	(addr & TARGET_PAGE_MASK))) {
0cac1b66	482	cpu_ldub_code(env1, addr);
0cac1b66	483	}
a54c87b6 PM	484	iotlbentry = &env1->iotlb[mmu_idx][page_index];
	485	pd = iotlbentry->addr & ~TARGET_PAGE_MASK;
	486	mr = iotlb_to_region(cpu, pd, iotlbentry->attrs);
0cac1b66	487	if (memory_region_is_unassigned(mr)) {
c658b94f AF	488	CPUClass *cc = CPU_GET_CLASS(cpu);
	489
	490	if (cc->do_unassigned_access) {
	491	cc->do_unassigned_access(cpu, addr, false, true, 0, 4);
	492	} else {
d7f30403 PM	493	report_bad_exec(cpu, addr);
d7f30403 PM	494	exit(1);
c658b94f	495	}
0cac1b66 BS	496	}
	497	p = (void *)((uintptr_t)addr + env1->tlb_table[mmu_idx][page_index].addend);
	498	return qemu_ram_addr_from_host_nofail(p);
	499	}
	500
7e9a7c50 RH	501	/* Return true if ADDR is present in the victim tlb, and has been copied
	502	back to the main tlb. */
	503	static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
	504	size_t elt_ofs, target_ulong page)
	505	{
	506	size_t vidx;
	507	for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
	508	CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
	509	target_ulong cmp = (target_ulong )((uintptr_t)vtlb + elt_ofs);
	510
	511	if (cmp == page) {
	512	/* Found entry in victim tlb, swap tlb and iotlb. */
	513	CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];
	514	CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];
	515	CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];
	516
	517	tmptlb = tlb; tlb = vtlb; vtlb = tmptlb;
	518	tmpio = io; io = vio; vio = tmpio;
	519	return true;
	520	}
	521	}
	522	return false;
	523	}
	524
	525	/* Macro to call the above, with local variables from the use context. */
a390284b	526	#define VICTIM_TLB_HIT(TY, ADDR) \
7e9a7c50	527	victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
a390284b	528	(ADDR) & TARGET_PAGE_MASK)
7e9a7c50	529
0f590e74 PB	530	#define MMUSUFFIX _mmu
	531
	532	#define SHIFT 0
58ed270d	533	#include "softmmu_template.h"
0f590e74 PB	534
0f590e74 PB	535	#define SHIFT 1
58ed270d	536	#include "softmmu_template.h"
0f590e74 PB	537
0f590e74 PB	538	#define SHIFT 2
58ed270d	539	#include "softmmu_template.h"
0f590e74 PB	540
0f590e74 PB	541	#define SHIFT 3
58ed270d	542	#include "softmmu_template.h"
0f590e74 PB	543	#undef MMUSUFFIX
0f590e74 PB	544
0cac1b66	545	#define MMUSUFFIX _cmmu
01ecaf43 RH	546	#undef GETPC
01ecaf43 RH	547	#define GETPC() ((uintptr_t)0)
0cac1b66 BS	548	#define SOFTMMU_CODE_ACCESS
	549
	550	#define SHIFT 0
58ed270d	551	#include "softmmu_template.h"
0cac1b66 BS	552
0cac1b66 BS	553	#define SHIFT 1
58ed270d	554	#include "softmmu_template.h"
0cac1b66 BS	555
0cac1b66 BS	556	#define SHIFT 2
58ed270d	557	#include "softmmu_template.h"
0cac1b66 BS	558
0cac1b66 BS	559	#define SHIFT 3
58ed270d	560	#include "softmmu_template.h"