X-Git-Url: https://repo.jachan.dev/qemu.git/blobdiff_plain/0b0d3320db74cde233ee7855ad32a9c121d20eb4..4a136e0a6b4ceac177bc2ab29502161553e25ae2:/translate-all.c diff --git a/translate-all.c b/translate-all.c index d666562a44..df7c697692 100644 --- a/translate-all.c +++ b/translate-all.c @@ -35,9 +35,6 @@ #include "cpu.h" #include "disas/disas.h" #include "tcg.h" -#include "qemu/timer.h" -#include "exec/memory.h" -#include "exec/address-spaces.h" #if defined(CONFIG_USER_ONLY) #include "qemu.h" #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) @@ -55,10 +52,13 @@ #include #endif #endif +#else +#include "exec/address-spaces.h" #endif #include "exec/cputlb.h" #include "translate-all.h" +#include "qemu/timer.h" //#define DEBUG_TB_INVALIDATE //#define DEBUG_FLUSH @@ -72,13 +72,6 @@ #define SMC_BITMAP_USE_THRESHOLD 10 -/* Translation blocks */ -static TranslationBlock *tbs; -TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE]; -static int nb_tbs; -/* any access to the tbs or the page table must use this lock */ -spinlock_t tb_lock = SPIN_LOCK_UNLOCKED; - typedef struct PageDesc { /* list of TBs intersecting this ram page */ TranslationBlock *first_tb; @@ -125,10 +118,6 @@ uintptr_t qemu_host_page_mask; The bottom level has pointers to PageDesc. */ static void *l1_map[V_L1_SIZE]; -/* statistics */ -static int tb_flush_count; -static int tb_phys_invalidate_count; - /* code generation context */ TCGContext tcg_ctx; @@ -471,6 +460,8 @@ static inline PageDesc *page_find(tb_page_addr_t index) # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) #elif defined(__sparc__) # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024) +#elif defined(__aarch64__) +# define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024) #elif defined(__arm__) # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024) #elif defined(__s390x__) @@ -589,7 +580,8 @@ static inline void code_gen_alloc(size_t tb_size) (TCG_MAX_OP_SIZE * OPC_BUF_SIZE); tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE; - tbs = g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock)); + tcg_ctx.tb_ctx.tbs = + g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock)); } /* Must be called before using the QEMU cpus. 'tb_size' is the size @@ -620,12 +612,12 @@ static TranslationBlock *tb_alloc(target_ulong pc) { TranslationBlock *tb; - if (nb_tbs >= tcg_ctx.code_gen_max_blocks || + if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks || (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >= tcg_ctx.code_gen_buffer_max_size) { return NULL; } - tb = &tbs[nb_tbs++]; + tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++]; tb->pc = pc; tb->cflags = 0; return tb; @@ -636,9 +628,10 @@ void tb_free(TranslationBlock *tb) /* In practice this is mostly used for single use temporary TB Ignore the hard cases and just back up if this TB happens to be the last one generated. */ - if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) { + if (tcg_ctx.tb_ctx.nb_tbs > 0 && + tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) { tcg_ctx.code_gen_ptr = tb->tc_ptr; - nb_tbs--; + tcg_ctx.tb_ctx.nb_tbs--; } } @@ -693,27 +686,28 @@ void tb_flush(CPUArchState *env1) #if defined(DEBUG_FLUSH) printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n", (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer), - nb_tbs, nb_tbs > 0 ? + tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ? ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) / - nb_tbs : 0); + tcg_ctx.tb_ctx.nb_tbs : 0); #endif if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) > tcg_ctx.code_gen_buffer_size) { cpu_abort(env1, "Internal error: code buffer overflow\n"); } - nb_tbs = 0; + tcg_ctx.tb_ctx.nb_tbs = 0; for (env = first_cpu; env != NULL; env = env->next_cpu) { memset(env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof(void *)); } - memset(tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof(void *)); + memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, + CODE_GEN_PHYS_HASH_SIZE * sizeof(void *)); page_flush_tb(); tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer; /* XXX: flush processor icache at this point if cache flush is expensive */ - tb_flush_count++; + tcg_ctx.tb_ctx.tb_flush_count++; } #ifdef DEBUG_TB_CHECK @@ -725,7 +719,7 @@ static void tb_invalidate_check(target_ulong address) address &= TARGET_PAGE_MASK; for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) { - for (tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) { + for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) { if (!(address + TARGET_PAGE_SIZE <= tb->pc || address >= tb->pc + tb->size)) { printf("ERROR invalidate: address=" TARGET_FMT_lx @@ -743,7 +737,8 @@ static void tb_page_check(void) int i, flags1, flags2; for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) { - for (tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) { + for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL; + tb = tb->phys_hash_next) { flags1 = page_get_flags(tb->pc); flags2 = page_get_flags(tb->pc + tb->size - 1); if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { @@ -835,7 +830,7 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) /* remove the TB from the hash list */ phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); h = tb_phys_hash_func(phys_pc); - tb_hash_remove(&tb_phys_hash[h], tb); + tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb); /* remove the TB from the page list */ if (tb->page_addr[0] != page_addr) { @@ -849,7 +844,7 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) invalidate_page_bitmap(p); } - tb_invalidated_flag = 1; + tcg_ctx.tb_ctx.tb_invalidated_flag = 1; /* remove the TB from the hash list */ h = tb_jmp_cache_hash_func(tb->pc); @@ -878,7 +873,7 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) } tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */ - tb_phys_invalidate_count++; + tcg_ctx.tb_ctx.tb_phys_invalidate_count++; } static inline void set_bits(uint8_t *tab, int start, int len) @@ -955,7 +950,7 @@ TranslationBlock *tb_gen_code(CPUArchState *env, /* cannot fail at this point */ tb = tb_alloc(pc); /* Don't forget to invalidate previous TB info. */ - tb_invalidated_flag = 1; + tcg_ctx.tb_ctx.tb_invalidated_flag = 1; } tc_ptr = tcg_ctx.code_gen_ptr; tb->tc_ptr = tc_ptr; @@ -1005,6 +1000,7 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, { TranslationBlock *tb, *tb_next, *saved_tb; CPUArchState *env = cpu_single_env; + CPUState *cpu = NULL; tb_page_addr_t tb_start, tb_end; PageDesc *p; int n; @@ -1027,6 +1023,9 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, /* build code bitmap */ build_page_bitmap(p); } + if (env != NULL) { + cpu = ENV_GET_CPU(env); + } /* we remove all the TBs in the range [start, end[ */ /* XXX: see if in some cases it could be faster to invalidate all @@ -1073,15 +1072,15 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, /* we need to do that to handle the case where a signal occurs while doing tb_phys_invalidate() */ saved_tb = NULL; - if (env) { - saved_tb = env->current_tb; - env->current_tb = NULL; + if (cpu != NULL) { + saved_tb = cpu->current_tb; + cpu->current_tb = NULL; } tb_phys_invalidate(tb, -1); - if (env) { - env->current_tb = saved_tb; - if (env->interrupt_request && env->current_tb) { - cpu_interrupt(env, env->interrupt_request); + if (cpu != NULL) { + cpu->current_tb = saved_tb; + if (cpu->interrupt_request && cpu->current_tb) { + cpu_interrupt(cpu, cpu->interrupt_request); } } } @@ -1101,7 +1100,7 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end, /* we generate a block containing just the instruction modifying the memory. It will ensure that it cannot modify itself */ - env->current_tb = NULL; + cpu->current_tb = NULL; tb_gen_code(env, current_pc, current_cs_base, current_flags, 1); cpu_resume_from_signal(env, NULL); } @@ -1149,6 +1148,7 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr, #ifdef TARGET_HAS_PRECISE_SMC TranslationBlock *current_tb = NULL; CPUArchState *env = cpu_single_env; + CPUState *cpu = NULL; int current_tb_modified = 0; target_ulong current_pc = 0; target_ulong current_cs_base = 0; @@ -1165,6 +1165,9 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr, if (tb && pc != 0) { current_tb = tb_find_pc(pc); } + if (env != NULL) { + cpu = ENV_GET_CPU(env); + } #endif while (tb != NULL) { n = (uintptr_t)tb & 3; @@ -1193,7 +1196,7 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr, /* we generate a block containing just the instruction modifying the memory. It will ensure that it cannot modify itself */ - env->current_tb = NULL; + cpu->current_tb = NULL; tb_gen_code(env, current_pc, current_cs_base, current_flags, 1); cpu_resume_from_signal(env, puc); } @@ -1273,7 +1276,7 @@ static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, mmap_lock(); /* add in the physical hash table */ h = tb_phys_hash_func(phys_pc); - ptb = &tb_phys_hash[h]; + ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h]; tb->phys_hash_next = *ptb; *ptb = tb; @@ -1307,11 +1310,11 @@ static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, /* check whether the given addr is in TCG generated code buffer or not */ bool is_tcg_gen_code(uintptr_t tc_ptr) { - /* This can be called during code generation, code_gen_buffer_max_size + /* This can be called during code generation, code_gen_buffer_size is used instead of code_gen_ptr for upper boundary checking */ return (tc_ptr >= (uintptr_t)tcg_ctx.code_gen_buffer && tc_ptr < (uintptr_t)(tcg_ctx.code_gen_buffer + - tcg_ctx.code_gen_buffer_max_size)); + tcg_ctx.code_gen_buffer_size)); } #endif @@ -1323,7 +1326,7 @@ static TranslationBlock *tb_find_pc(uintptr_t tc_ptr) uintptr_t v; TranslationBlock *tb; - if (nb_tbs <= 0) { + if (tcg_ctx.tb_ctx.nb_tbs <= 0) { return NULL; } if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer || @@ -1332,10 +1335,10 @@ static TranslationBlock *tb_find_pc(uintptr_t tc_ptr) } /* binary search (cf Knuth) */ m_min = 0; - m_max = nb_tbs - 1; + m_max = tcg_ctx.tb_ctx.nb_tbs - 1; while (m_min <= m_max) { m = (m_min + m_max) >> 1; - tb = &tbs[m]; + tb = &tcg_ctx.tb_ctx.tbs[m]; v = (uintptr_t)tb->tc_ptr; if (v == tc_ptr) { return tb; @@ -1345,56 +1348,7 @@ static TranslationBlock *tb_find_pc(uintptr_t tc_ptr) m_min = m + 1; } } - return &tbs[m_max]; -} - -static void tb_reset_jump_recursive(TranslationBlock *tb); - -static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n) -{ - TranslationBlock *tb1, *tb_next, **ptb; - unsigned int n1; - - tb1 = tb->jmp_next[n]; - if (tb1 != NULL) { - /* find head of list */ - for (;;) { - n1 = (uintptr_t)tb1 & 3; - tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); - if (n1 == 2) { - break; - } - tb1 = tb1->jmp_next[n1]; - } - /* we are now sure now that tb jumps to tb1 */ - tb_next = tb1; - - /* remove tb from the jmp_first list */ - ptb = &tb_next->jmp_first; - for (;;) { - tb1 = *ptb; - n1 = (uintptr_t)tb1 & 3; - tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3); - if (n1 == n && tb1 == tb) { - break; - } - ptb = &tb1->jmp_next[n1]; - } - *ptb = tb->jmp_next[n]; - tb->jmp_next[n] = NULL; - - /* suppress the jump to next tb in generated code */ - tb_reset_jump(tb, n); - - /* suppress jumps in the tb on which we could have jumped */ - tb_reset_jump_recursive(tb_next); - } -} - -static void tb_reset_jump_recursive(TranslationBlock *tb) -{ - tb_reset_jump_recursive2(tb, 0); - tb_reset_jump_recursive2(tb, 1); + return &tcg_ctx.tb_ctx.tbs[m_max]; } #if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY) @@ -1402,39 +1356,19 @@ void tb_invalidate_phys_addr(hwaddr addr) { ram_addr_t ram_addr; MemoryRegionSection *section; + hwaddr l = 1; - section = phys_page_find(address_space_memory.dispatch, - addr >> TARGET_PAGE_BITS); + section = address_space_translate(&address_space_memory, addr, &addr, &l, false); if (!(memory_region_is_ram(section->mr) - || (section->mr->rom_device && section->mr->readable))) { + || memory_region_is_romd(section->mr))) { return; } ram_addr = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK) - + memory_region_section_addr(section, addr); + + addr; tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); } #endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */ -void cpu_unlink_tb(CPUArchState *env) -{ - /* FIXME: TB unchaining isn't SMP safe. For now just ignore the - problem and hope the cpu will stop of its own accord. For userspace - emulation this often isn't actually as bad as it sounds. Often - signals are used primarily to interrupt blocking syscalls. */ - TranslationBlock *tb; - static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED; - - spin_lock(&interrupt_lock); - tb = env->current_tb; - /* if the cpu is currently executing code, we must unlink it and - all the potentially executing TB */ - if (tb) { - env->current_tb = NULL; - tb_reset_jump_recursive(tb); - } - spin_unlock(&interrupt_lock); -} - void tb_check_watchpoint(CPUArchState *env) { TranslationBlock *tb; @@ -1450,13 +1384,13 @@ void tb_check_watchpoint(CPUArchState *env) #ifndef CONFIG_USER_ONLY /* mask must never be zero, except for A20 change call */ -static void tcg_handle_interrupt(CPUArchState *env, int mask) +static void tcg_handle_interrupt(CPUState *cpu, int mask) { - CPUState *cpu = ENV_GET_CPU(env); + CPUArchState *env = cpu->env_ptr; int old_mask; - old_mask = env->interrupt_request; - env->interrupt_request |= mask; + old_mask = cpu->interrupt_request; + cpu->interrupt_request |= mask; /* * If called from iothread context, wake the target cpu in @@ -1474,7 +1408,7 @@ static void tcg_handle_interrupt(CPUArchState *env, int mask) cpu_abort(env, "Raised interrupt while not in I/O function"); } } else { - cpu_unlink_tb(env); + cpu->tcg_exit_req = 1; } } @@ -1566,8 +1500,8 @@ void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) cross_page = 0; direct_jmp_count = 0; direct_jmp2_count = 0; - for (i = 0; i < nb_tbs; i++) { - tb = &tbs[i]; + for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) { + tb = &tcg_ctx.tb_ctx.tbs[i]; target_code_size += tb->size; if (tb->size > max_target_code_size) { max_target_code_size = tb->size; @@ -1588,37 +1522,42 @@ void dump_exec_info(FILE *f, fprintf_function cpu_fprintf) tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_max_size); cpu_fprintf(f, "TB count %d/%d\n", - nb_tbs, tcg_ctx.code_gen_max_blocks); + tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks); cpu_fprintf(f, "TB avg target size %d max=%d bytes\n", - nb_tbs ? target_code_size / nb_tbs : 0, - max_target_code_size); + tcg_ctx.tb_ctx.nb_tbs ? target_code_size / + tcg_ctx.tb_ctx.nb_tbs : 0, + max_target_code_size); cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n", - nb_tbs ? (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) / - nb_tbs : 0, - target_code_size ? - (double) (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) / - target_code_size : 0); - cpu_fprintf(f, "cross page TB count %d (%d%%)\n", - cross_page, - nb_tbs ? (cross_page * 100) / nb_tbs : 0); + tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr - + tcg_ctx.code_gen_buffer) / + tcg_ctx.tb_ctx.nb_tbs : 0, + target_code_size ? (double) (tcg_ctx.code_gen_ptr - + tcg_ctx.code_gen_buffer) / + target_code_size : 0); + cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page, + tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) / + tcg_ctx.tb_ctx.nb_tbs : 0); cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n", direct_jmp_count, - nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0, + tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) / + tcg_ctx.tb_ctx.nb_tbs : 0, direct_jmp2_count, - nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0); + tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) / + tcg_ctx.tb_ctx.nb_tbs : 0); cpu_fprintf(f, "\nStatistics:\n"); - cpu_fprintf(f, "TB flush count %d\n", tb_flush_count); - cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count); + cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count); + cpu_fprintf(f, "TB invalidate count %d\n", + tcg_ctx.tb_ctx.tb_phys_invalidate_count); cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count); tcg_dump_info(f, cpu_fprintf); } #else /* CONFIG_USER_ONLY */ -void cpu_interrupt(CPUArchState *env, int mask) +void cpu_interrupt(CPUState *cpu, int mask) { - env->interrupt_request |= mask; - cpu_unlink_tb(env); + cpu->interrupt_request |= mask; + cpu->tcg_exit_req = 1; } /*