X-Git-Url: https://repo.jachan.dev/qemu.git/blobdiff_plain/4cddc7f44ff83ff3545059f1b8648b8bd7fe0e72..54c54f8b56047d3c2420e1ae06a6a8890c220ac4:/cputlb.c diff --git a/cputlb.c b/cputlb.c index 9270055b83..bf1d50adde 100644 --- a/cputlb.c +++ b/cputlb.c @@ -22,10 +22,13 @@ #include "exec/exec-all.h" #include "exec/memory.h" #include "exec/address-spaces.h" +#include "exec/cpu_ldst.h" #include "exec/cputlb.h" #include "exec/memory-internal.h" +#include "exec/ram_addr.h" +#include "tcg/tcg.h" //#define DEBUG_TLB //#define DEBUG_TLB_CHECK @@ -45,9 +48,9 @@ int tlb_flush_count; * entries from the TLB at any time, so flushing more entries than * required is only an efficiency issue, not a correctness issue. */ -void tlb_flush(CPUArchState *env, int flush_global) +void tlb_flush(CPUState *cpu, int flush_global) { - CPUState *cpu = ENV_GET_CPU(env); + CPUArchState *env = cpu->env_ptr; #if defined(DEBUG_TLB) printf("tlb_flush:\n"); @@ -57,13 +60,56 @@ void tlb_flush(CPUArchState *env, int flush_global) cpu->current_tb = NULL; memset(env->tlb_table, -1, sizeof(env->tlb_table)); - memset(env->tb_jmp_cache, 0, sizeof(env->tb_jmp_cache)); + memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table)); + memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache)); + env->vtlb_index = 0; env->tlb_flush_addr = -1; env->tlb_flush_mask = 0; tlb_flush_count++; } +static inline void v_tlb_flush_by_mmuidx(CPUState *cpu, va_list argp) +{ + CPUArchState *env = cpu->env_ptr; + +#if defined(DEBUG_TLB) + printf("tlb_flush_by_mmuidx:"); +#endif + /* must reset current TB so that interrupts cannot modify the + links while we are modifying them */ + cpu->current_tb = NULL; + + for (;;) { + int mmu_idx = va_arg(argp, int); + + if (mmu_idx < 0) { + break; + } + +#if defined(DEBUG_TLB) + printf(" %d", mmu_idx); +#endif + + memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0])); + memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0])); + } + +#if defined(DEBUG_TLB) + printf("\n"); +#endif + + memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache)); +} + +void tlb_flush_by_mmuidx(CPUState *cpu, ...) +{ + va_list argp; + va_start(argp, cpu); + v_tlb_flush_by_mmuidx(cpu, argp); + va_end(argp); +} + static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr) { if (addr == (tlb_entry->addr_read & @@ -76,9 +122,9 @@ static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr) } } -void tlb_flush_page(CPUArchState *env, target_ulong addr) +void tlb_flush_page(CPUState *cpu, target_ulong addr) { - CPUState *cpu = ENV_GET_CPU(env); + CPUArchState *env = cpu->env_ptr; int i; int mmu_idx; @@ -92,7 +138,7 @@ void tlb_flush_page(CPUArchState *env, target_ulong addr) TARGET_FMT_lx "/" TARGET_FMT_lx ")\n", env->tlb_flush_addr, env->tlb_flush_mask); #endif - tlb_flush(env, 1); + tlb_flush(cpu, 1); return; } /* must reset current TB so that interrupts cannot modify the @@ -105,24 +151,86 @@ void tlb_flush_page(CPUArchState *env, target_ulong addr) tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr); } - tb_flush_jmp_cache(env, addr); + /* check whether there are entries that need to be flushed in the vtlb */ + for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { + int k; + for (k = 0; k < CPU_VTLB_SIZE; k++) { + tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr); + } + } + + tb_flush_jmp_cache(cpu, addr); +} + +void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...) +{ + CPUArchState *env = cpu->env_ptr; + int i, k; + va_list argp; + + va_start(argp, addr); + +#if defined(DEBUG_TLB) + printf("tlb_flush_page_by_mmu_idx: " TARGET_FMT_lx, addr); +#endif + /* Check if we need to flush due to large pages. */ + if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) { +#if defined(DEBUG_TLB) + printf(" forced full flush (" + TARGET_FMT_lx "/" TARGET_FMT_lx ")\n", + env->tlb_flush_addr, env->tlb_flush_mask); +#endif + v_tlb_flush_by_mmuidx(cpu, argp); + va_end(argp); + return; + } + /* must reset current TB so that interrupts cannot modify the + links while we are modifying them */ + cpu->current_tb = NULL; + + addr &= TARGET_PAGE_MASK; + i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); + + for (;;) { + int mmu_idx = va_arg(argp, int); + + if (mmu_idx < 0) { + break; + } + +#if defined(DEBUG_TLB) + printf(" %d", mmu_idx); +#endif + + tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr); + + /* check whether there are vltb entries that need to be flushed */ + for (k = 0; k < CPU_VTLB_SIZE; k++) { + tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr); + } + } + va_end(argp); + +#if defined(DEBUG_TLB) + printf("\n"); +#endif + + tb_flush_jmp_cache(cpu, addr); } /* update the TLBs so that writes to code in the virtual page 'addr' can be detected */ void tlb_protect_code(ram_addr_t ram_addr) { - cpu_physical_memory_reset_dirty(ram_addr, - ram_addr + TARGET_PAGE_SIZE, - CODE_DIRTY_FLAG); + cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE, + DIRTY_MEMORY_CODE); } /* update the TLB so that writes in physical page 'phys_addr' are no longer tested for self modifying code */ -void tlb_unprotect_code_phys(CPUArchState *env, ram_addr_t ram_addr, - target_ulong vaddr) +void tlb_unprotect_code(ram_addr_t ram_addr) { - cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG); + cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE); } static bool tlb_is_dirty_ram(CPUTLBEntry *tlbe) @@ -154,22 +262,24 @@ static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr) return ram_addr; } -void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length) +void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length) { - CPUState *cpu; CPUArchState *env; - CPU_FOREACH(cpu) { - int mmu_idx; + int mmu_idx; + + env = cpu->env_ptr; + for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { + unsigned int i; - env = cpu->env_ptr; - for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { - unsigned int i; + for (i = 0; i < CPU_TLB_SIZE; i++) { + tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i], + start1, length); + } - for (i = 0; i < CPU_TLB_SIZE; i++) { - tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i], - start1, length); - } + for (i = 0; i < CPU_VTLB_SIZE; i++) { + tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i], + start1, length); } } } @@ -183,8 +293,9 @@ static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr) /* update the TLB corresponding to virtual page vaddr so that it is no longer dirty */ -void tlb_set_dirty(CPUArchState *env, target_ulong vaddr) +void tlb_set_dirty(CPUState *cpu, target_ulong vaddr) { + CPUArchState *env = cpu->env_ptr; int i; int mmu_idx; @@ -193,6 +304,13 @@ void tlb_set_dirty(CPUArchState *env, target_ulong vaddr) for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr); } + + for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) { + int k; + for (k = 0; k < CPU_VTLB_SIZE; k++) { + tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr); + } + } } /* Our TLB does not support large pages, so remember the area covered by @@ -219,12 +337,17 @@ static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr, } /* Add a new TLB entry. At most one entry for a given virtual address - is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the - supplied size is only used by tlb_flush_page. */ -void tlb_set_page(CPUArchState *env, target_ulong vaddr, - hwaddr paddr, int prot, - int mmu_idx, target_ulong size) + * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the + * supplied size is only used by tlb_flush_page. + * + * Called from TCG-generated code, which is under an RCU read-side + * critical section. + */ +void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr, + hwaddr paddr, MemTxAttrs attrs, int prot, + int mmu_idx, target_ulong size) { + CPUArchState *env = cpu->env_ptr; MemoryRegionSection *section; unsigned int index; target_ulong address; @@ -232,6 +355,7 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr, uintptr_t addend; CPUTLBEntry *te; hwaddr iotlb, xlat, sz; + unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE; assert(size >= TARGET_PAGE_SIZE); if (size != TARGET_PAGE_SIZE) { @@ -239,12 +363,12 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr, } sz = size; - section = address_space_translate_for_iotlb(&address_space_memory, paddr, - &xlat, &sz); + section = address_space_translate_for_iotlb(cpu, paddr, &xlat, &sz); assert(sz >= TARGET_PAGE_SIZE); #if defined(DEBUG_TLB) - printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx + qemu_log_mask(CPU_LOG_MMU, + "tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx " prot=%x idx=%d\n", vaddr, paddr, prot, mmu_idx); #endif @@ -260,12 +384,19 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr, } code_address = address; - iotlb = memory_region_section_get_iotlb(env, section, vaddr, paddr, xlat, + iotlb = memory_region_section_get_iotlb(cpu, section, vaddr, paddr, xlat, prot, &address); index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - env->iotlb[mmu_idx][index] = iotlb - vaddr; te = &env->tlb_table[mmu_idx][index]; + + /* do not discard the translation in te, evict it into a victim tlb */ + env->tlb_v_table[mmu_idx][vidx] = *te; + env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index]; + + /* refill the tlb */ + env->iotlb[mmu_idx][index].addr = iotlb - vaddr; + env->iotlb[mmu_idx][index].attrs = attrs; te->addend = addend - vaddr; if (prot & PAGE_READ) { te->addr_read = address; @@ -284,7 +415,8 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr, /* Write access calls the I/O callback. */ te->addr_write = address | TLB_MMIO; } else if (memory_region_is_ram(section->mr) - && !cpu_physical_memory_is_dirty(section->mr->ram_addr + xlat)) { + && cpu_physical_memory_is_clean(section->mr->ram_addr + + xlat)) { te->addr_write = address | TLB_NOTDIRTY; } else { te->addr_write = address; @@ -294,6 +426,17 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr, } } +/* Add a new TLB entry, but without specifying the memory + * transaction attributes to be used. + */ +void tlb_set_page(CPUState *cpu, target_ulong vaddr, + hwaddr paddr, int prot, + int mmu_idx, target_ulong size) +{ + tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED, + prot, mmu_idx, size); +} + /* NOTE: this function can trigger an exception */ /* NOTE2: the returned address is not exactly the physical address: it * is actually a ram_addr_t (in system mode; the user mode emulation @@ -304,23 +447,23 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr) int mmu_idx, page_index, pd; void *p; MemoryRegion *mr; + CPUState *cpu = ENV_GET_CPU(env1); page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1); - mmu_idx = cpu_mmu_index(env1); + mmu_idx = cpu_mmu_index(env1, true); if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code != (addr & TARGET_PAGE_MASK))) { cpu_ldub_code(env1, addr); } - pd = env1->iotlb[mmu_idx][page_index] & ~TARGET_PAGE_MASK; - mr = iotlb_to_region(pd); + pd = env1->iotlb[mmu_idx][page_index].addr & ~TARGET_PAGE_MASK; + mr = iotlb_to_region(cpu, pd); if (memory_region_is_unassigned(mr)) { - CPUState *cpu = ENV_GET_CPU(env1); CPUClass *cc = CPU_GET_CLASS(cpu); if (cc->do_unassigned_access) { cc->do_unassigned_access(cpu, addr, false, true, 0, 4); } else { - cpu_abort(env1, "Trying to execute code outside RAM or ROM at 0x" + cpu_abort(cpu, "Trying to execute code outside RAM or ROM at 0x" TARGET_FMT_lx "\n", addr); } } @@ -328,21 +471,36 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr) return qemu_ram_addr_from_host_nofail(p); } +#define MMUSUFFIX _mmu + +#define SHIFT 0 +#include "softmmu_template.h" + +#define SHIFT 1 +#include "softmmu_template.h" + +#define SHIFT 2 +#include "softmmu_template.h" + +#define SHIFT 3 +#include "softmmu_template.h" +#undef MMUSUFFIX + #define MMUSUFFIX _cmmu -#undef GETPC -#define GETPC() ((uintptr_t)0) +#undef GETPC_ADJ +#define GETPC_ADJ 0 +#undef GETRA +#define GETRA() ((uintptr_t)0) #define SOFTMMU_CODE_ACCESS #define SHIFT 0 -#include "exec/softmmu_template.h" +#include "softmmu_template.h" #define SHIFT 1 -#include "exec/softmmu_template.h" +#include "softmmu_template.h" #define SHIFT 2 -#include "exec/softmmu_template.h" +#include "softmmu_template.h" #define SHIFT 3 -#include "exec/softmmu_template.h" - -#undef env +#include "softmmu_template.h"