#endif
#include "exec/cputlb.h"
+#include "exec/tb-hash.h"
#include "translate-all.h"
+#include "qemu/bitmap.h"
#include "qemu/timer.h"
//#define DEBUG_TB_INVALIDATE
/* in order to optimize self modifying code, we count the number
of lookups we do to a given page to use a bitmap */
unsigned int code_write_count;
- uint8_t *code_bitmap;
+ unsigned long *code_bitmap;
#if defined(CONFIG_USER_ONLY)
unsigned long flags;
#endif
#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
-uintptr_t qemu_real_host_page_size;
uintptr_t qemu_host_page_size;
uintptr_t qemu_host_page_mask;
-/* This is a multi-level map on the virtual address space.
- The bottom level has pointers to PageDesc. */
+/* The bottom level has pointers to PageDesc */
static void *l1_map[V_L1_SIZE];
/* code generation context */
TCGContext tcg_ctx;
-static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
- tb_page_addr_t phys_page2);
-static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
+/* translation block context */
+#ifdef CONFIG_USER_ONLY
+__thread int have_tb_lock;
+#endif
-void cpu_gen_init(void)
+void tb_lock(void)
{
- tcg_context_init(&tcg_ctx);
+#ifdef CONFIG_USER_ONLY
+ assert(!have_tb_lock);
+ qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
+ have_tb_lock++;
+#endif
}
-/* return non zero if the very first instruction is invalid so that
- the virtual CPU can trigger an exception.
-
- '*gen_code_size_ptr' contains the size of the generated code (host
- code).
-*/
-int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
+void tb_unlock(void)
{
- TCGContext *s = &tcg_ctx;
- tcg_insn_unit *gen_code_buf;
- int gen_code_size;
-#ifdef CONFIG_PROFILER
- int64_t ti;
-#endif
-
-#ifdef CONFIG_PROFILER
- s->tb_count1++; /* includes aborted translations because of
- exceptions */
- ti = profile_getclock();
+#ifdef CONFIG_USER_ONLY
+ assert(have_tb_lock);
+ have_tb_lock--;
+ qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
#endif
- tcg_func_start(s);
-
- gen_intermediate_code(env, tb);
-
- trace_translate_block(tb, tb->pc, tb->tc_ptr);
+}
- /* generate machine code */
- gen_code_buf = tb->tc_ptr;
- tb->tb_next_offset[0] = 0xffff;
- tb->tb_next_offset[1] = 0xffff;
- s->tb_next_offset = tb->tb_next_offset;
-#ifdef USE_DIRECT_JUMP
- s->tb_jmp_offset = tb->tb_jmp_offset;
- s->tb_next = NULL;
-#else
- s->tb_jmp_offset = NULL;
- s->tb_next = tb->tb_next;
+void tb_lock_reset(void)
+{
+#ifdef CONFIG_USER_ONLY
+ if (have_tb_lock) {
+ qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
+ have_tb_lock = 0;
+ }
#endif
+}
-#ifdef CONFIG_PROFILER
- s->tb_count++;
- s->interm_time += profile_getclock() - ti;
- s->code_time -= profile_getclock();
-#endif
- gen_code_size = tcg_gen_code(s, gen_code_buf);
- *gen_code_size_ptr = gen_code_size;
-#ifdef CONFIG_PROFILER
- s->code_time += profile_getclock();
- s->code_in_len += tb->size;
- s->code_out_len += gen_code_size;
-#endif
+static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+ tb_page_addr_t phys_page2);
+static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
-#ifdef DEBUG_DISAS
- if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
- qemu_log("OUT: [size=%d]\n", gen_code_size);
- log_disas(tb->tc_ptr, gen_code_size);
- qemu_log("\n");
- qemu_log_flush();
- }
-#endif
- return 0;
+void cpu_gen_init(void)
+{
+ tcg_context_init(&tcg_ctx);
}
-/* The cpu state corresponding to 'searched_pc' is restored.
- */
+/* The cpu state corresponding to 'searched_pc' is restored. */
static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
uintptr_t searched_pc)
{
gen_intermediate_code_pc(env, tb);
- if (use_icount) {
+ if (tb->cflags & CF_USE_ICOUNT) {
+ assert(use_icount);
/* Reset the cycle counter to the start of the block. */
cpu->icount_decr.u16.low += tb->icount;
/* Clear the IO flag. */
}
cpu->icount_decr.u16.low -= s->gen_opc_icount[j];
- restore_state_to_opc(env, tb, j);
+ restore_state_to_opc(env, tb, s->gen_opc_data);
#ifdef CONFIG_PROFILER
s->restore_time += profile_getclock() - ti;
}
#ifdef _WIN32
-static inline void map_exec(void *addr, long size)
+static __attribute__((unused)) void map_exec(void *addr, long size)
{
DWORD old_protect;
VirtualProtect(addr, size,
PAGE_EXECUTE_READWRITE, &old_protect);
}
#else
-static inline void map_exec(void *addr, long size)
+static __attribute__((unused)) void map_exec(void *addr, long size)
{
unsigned long start, end, page_size;
/* NOTE: we can always suppose that qemu_host_page_size >=
TARGET_PAGE_SIZE */
qemu_real_host_page_size = getpagesize();
+ qemu_real_host_page_mask = ~(qemu_real_host_page_size - 1);
if (qemu_host_page_size == 0) {
qemu_host_page_size = qemu_real_host_page_size;
}
#endif
}
+/* If alloc=1:
+ * Called with mmap_lock held for user-mode emulation.
+ */
static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
{
PageDesc *pd;
void **lp;
int i;
-#if defined(CONFIG_USER_ONLY)
- /* We can't use g_malloc because it may recurse into a locked mutex. */
-# define ALLOC(P, SIZE) \
- do { \
- P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
- MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
- } while (0)
-#else
-# define ALLOC(P, SIZE) \
- do { P = g_malloc0(SIZE); } while (0)
-#endif
-
/* Level 1. Always allocated. */
lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
/* Level 2..N-1. */
for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
- void **p = *lp;
+ void **p = atomic_rcu_read(lp);
if (p == NULL) {
if (!alloc) {
return NULL;
}
- ALLOC(p, sizeof(void *) * V_L2_SIZE);
- *lp = p;
+ p = g_new0(void *, V_L2_SIZE);
+ atomic_rcu_set(lp, p);
}
lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
}
- pd = *lp;
+ pd = atomic_rcu_read(lp);
if (pd == NULL) {
if (!alloc) {
return NULL;
}
- ALLOC(pd, sizeof(PageDesc) * V_L2_SIZE);
- *lp = pd;
+ pd = g_new0(PageDesc, V_L2_SIZE);
+ atomic_rcu_set(lp, pd);
}
-#undef ALLOC
-
return pd + (index & (V_L2_SIZE - 1));
}
return page_find_alloc(index, 0);
}
-#if !defined(CONFIG_USER_ONLY)
-#define mmap_lock() do { } while (0)
-#define mmap_unlock() do { } while (0)
-#endif
-
#if defined(CONFIG_USER_ONLY)
/* Currently it is not recommended to allocate big chunks of data in
user mode. It will change when a dedicated libc will be used. */
#else
static inline void *alloc_code_gen_buffer(void)
{
- void *buf = g_malloc(tcg_ctx.code_gen_buffer_size);
+ void *buf = g_try_malloc(tcg_ctx.code_gen_buffer_size);
if (buf == NULL) {
return NULL;
CODE_GEN_AVG_BLOCK_SIZE;
tcg_ctx.tb_ctx.tbs =
g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
+ qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
}
/* Must be called before using the QEMU cpus. 'tb_size' is the size
tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
page_init();
-#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
+#if defined(CONFIG_SOFTMMU)
/* There's no guest base to take into account, so go ahead and
initialize the prologue now. */
tcg_prologue_init(&tcg_ctx);
static inline void invalidate_page_bitmap(PageDesc *p)
{
- if (p->code_bitmap) {
- g_free(p->code_bitmap);
- p->code_bitmap = NULL;
- }
+ g_free(p->code_bitmap);
+ p->code_bitmap = NULL;
p->code_write_count = 0;
}
/* flush all the translation blocks */
/* XXX: tb_flush is currently not thread safe */
-void tb_flush(CPUArchState *env1)
+void tb_flush(CPUState *cpu)
{
- CPUState *cpu = ENV_GET_CPU(env1);
-
#if defined(DEBUG_FLUSH)
printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
(unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
}
-static inline void set_bits(uint8_t *tab, int start, int len)
-{
- int end, mask, end1;
-
- end = start + len;
- tab += start >> 3;
- mask = 0xff << (start & 7);
- if ((start & ~7) == (end & ~7)) {
- if (start < end) {
- mask &= ~(0xff << (end & 7));
- *tab |= mask;
- }
- } else {
- *tab++ |= mask;
- start = (start + 8) & ~7;
- end1 = end & ~7;
- while (start < end1) {
- *tab++ = 0xff;
- start += 8;
- }
- if (start < end) {
- mask = ~(0xff << (end & 7));
- *tab |= mask;
- }
- }
-}
-
static void build_page_bitmap(PageDesc *p)
{
int n, tb_start, tb_end;
TranslationBlock *tb;
- p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
+ p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
tb = p->first_tb;
while (tb != NULL) {
tb_start = 0;
tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
}
- set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
+ bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
tb = tb->page_next[n];
}
}
+/* Called with mmap_lock held for user mode emulation. */
TranslationBlock *tb_gen_code(CPUState *cpu,
target_ulong pc, target_ulong cs_base,
int flags, int cflags)
TranslationBlock *tb;
tb_page_addr_t phys_pc, phys_page2;
target_ulong virt_page2;
- int code_gen_size;
+ tcg_insn_unit *gen_code_buf;
+ int gen_code_size;
+#ifdef CONFIG_PROFILER
+ int64_t ti;
+#endif
phys_pc = get_page_addr_code(env, pc);
+ if (use_icount) {
+ cflags |= CF_USE_ICOUNT;
+ }
tb = tb_alloc(pc);
if (!tb) {
/* flush must be done */
- tb_flush(env);
+ tb_flush(cpu);
/* cannot fail at this point */
tb = tb_alloc(pc);
/* Don't forget to invalidate previous TB info. */
tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
}
- tb->tc_ptr = tcg_ctx.code_gen_ptr;
+
+ gen_code_buf = tcg_ctx.code_gen_ptr;
+ tb->tc_ptr = gen_code_buf;
tb->cs_base = cs_base;
tb->flags = flags;
tb->cflags = cflags;
- cpu_gen_code(env, tb, &code_gen_size);
- tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
- code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
+
+#ifdef CONFIG_PROFILER
+ tcg_ctx.tb_count1++; /* includes aborted translations because of
+ exceptions */
+ ti = profile_getclock();
+#endif
+
+ tcg_func_start(&tcg_ctx);
+
+ gen_intermediate_code(env, tb);
+
+ trace_translate_block(tb, tb->pc, tb->tc_ptr);
+
+ /* generate machine code */
+ tb->tb_next_offset[0] = 0xffff;
+ tb->tb_next_offset[1] = 0xffff;
+ tcg_ctx.tb_next_offset = tb->tb_next_offset;
+#ifdef USE_DIRECT_JUMP
+ tcg_ctx.tb_jmp_offset = tb->tb_jmp_offset;
+ tcg_ctx.tb_next = NULL;
+#else
+ tcg_ctx.tb_jmp_offset = NULL;
+ tcg_ctx.tb_next = tb->tb_next;
+#endif
+
+#ifdef CONFIG_PROFILER
+ tcg_ctx.tb_count++;
+ tcg_ctx.interm_time += profile_getclock() - ti;
+ tcg_ctx.code_time -= profile_getclock();
+#endif
+
+ gen_code_size = tcg_gen_code(&tcg_ctx, gen_code_buf);
+
+#ifdef CONFIG_PROFILER
+ tcg_ctx.code_time += profile_getclock();
+ tcg_ctx.code_in_len += tb->size;
+ tcg_ctx.code_out_len += gen_code_size;
+#endif
+
+#ifdef DEBUG_DISAS
+ if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
+ qemu_log("OUT: [size=%d]\n", gen_code_size);
+ log_disas(tb->tc_ptr, gen_code_size);
+ qemu_log("\n");
+ qemu_log_flush();
+ }
+#endif
+
+ tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)gen_code_buf +
+ gen_code_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
/* check next page if needed */
virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
* 'is_cpu_write_access' should be true if called from a real cpu write
* access: the virtual CPU will exit the current TB if code is modified inside
* this TB.
+ *
+ * Called with mmap_lock held for user-mode emulation
*/
-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
- int is_cpu_write_access)
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
{
while (start < end) {
- tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
+ tb_invalidate_phys_page_range(start, end, 0);
start &= TARGET_PAGE_MASK;
start += TARGET_PAGE_SIZE;
}
* 'is_cpu_write_access' should be true if called from a real cpu write
* access: the virtual CPU will exit the current TB if code is modified inside
* this TB.
+ *
+ * Called with mmap_lock held for user-mode emulation
*/
void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
int is_cpu_write_access)
if (!p) {
return;
}
- if (!p->code_bitmap &&
- ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
- is_cpu_write_access) {
- /* build code bitmap */
- build_page_bitmap(p);
- }
#if defined(TARGET_HAS_PRECISE_SMC)
if (cpu != NULL) {
env = cpu->env_ptr;
/* if no code remaining, no need to continue to use slow writes */
if (!p->first_tb) {
invalidate_page_bitmap(p);
- if (is_cpu_write_access) {
- tlb_unprotect_code_phys(cpu, start, cpu->mem_io_vaddr);
- }
+ tlb_unprotect_code(start);
}
#endif
#ifdef TARGET_HAS_PRECISE_SMC
void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
{
PageDesc *p;
- int offset, b;
#if 0
if (1) {
if (!p) {
return;
}
+ if (!p->code_bitmap &&
+ ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
+ /* build code bitmap */
+ build_page_bitmap(p);
+ }
if (p->code_bitmap) {
- offset = start & ~TARGET_PAGE_MASK;
- b = p->code_bitmap[offset >> 3] >> (offset & 7);
+ unsigned int nr;
+ unsigned long b;
+
+ nr = start & ~TARGET_PAGE_MASK;
+ b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
if (b & ((1 << len) - 1)) {
goto do_invalidate;
}
}
#if !defined(CONFIG_SOFTMMU)
+/* Called with mmap_lock held. */
static void tb_invalidate_phys_page(tb_page_addr_t addr,
uintptr_t pc, void *puc,
bool locked)
}
#endif
-/* add the tb in the target page and protect it if necessary */
+/* add the tb in the target page and protect it if necessary
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
static inline void tb_alloc_page(TranslationBlock *tb,
unsigned int n, tb_page_addr_t page_addr)
{
p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
invalidate_page_bitmap(p);
-#if defined(TARGET_HAS_SMC) || 1
-
#if defined(CONFIG_USER_ONLY)
if (p->flags & PAGE_WRITE) {
target_ulong addr;
tlb_protect_code(page_addr);
}
#endif
-
-#endif /* TARGET_HAS_SMC */
}
/* add a new TB and link it to the physical page tables. phys_page2 is
- (-1) to indicate that only one page contains the TB. */
+ * (-1) to indicate that only one page contains the TB.
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
tb_page_addr_t phys_page2)
{
unsigned int h;
TranslationBlock **ptb;
- /* Grab the mmap lock to stop another thread invalidating this TB
- before we are done. */
- mmap_lock();
/* add in the physical hash table */
h = tb_phys_hash_func(phys_pc);
ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
#ifdef DEBUG_TB_CHECK
tb_page_check();
#endif
- mmap_unlock();
}
/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
return &tcg_ctx.tb_ctx.tbs[m_max];
}
-#if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
+#if !defined(CONFIG_USER_ONLY)
void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
{
ram_addr_t ram_addr;
MemoryRegion *mr;
hwaddr l = 1;
+ rcu_read_lock();
mr = address_space_translate(as, addr, &addr, &l, false);
if (!(memory_region_is_ram(mr)
|| memory_region_is_romd(mr))) {
+ rcu_read_unlock();
return;
}
ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
+ addr;
tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
+ rcu_read_unlock();
}
-#endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
+#endif /* !defined(CONFIG_USER_ONLY) */
void tb_check_watchpoint(CPUState *cpu)
{
TranslationBlock *tb;
tb = tb_find_pc(cpu->mem_io_pc);
- if (!tb) {
- cpu_abort(cpu, "check_watchpoint: could not find TB for pc=%p",
- (void *)cpu->mem_io_pc);
- }
- cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
- tb_phys_invalidate(tb, -1);
-}
-
-#ifndef CONFIG_USER_ONLY
-/* mask must never be zero, except for A20 change call */
-static void tcg_handle_interrupt(CPUState *cpu, int mask)
-{
- int old_mask;
-
- old_mask = cpu->interrupt_request;
- cpu->interrupt_request |= mask;
-
- /*
- * If called from iothread context, wake the target cpu in
- * case its halted.
- */
- if (!qemu_cpu_is_self(cpu)) {
- qemu_cpu_kick(cpu);
- return;
- }
-
- if (use_icount) {
- cpu->icount_decr.u16.high = 0xffff;
- if (!cpu_can_do_io(cpu)
- && (mask & ~old_mask) != 0) {
- cpu_abort(cpu, "Raised interrupt while not in I/O function");
- }
+ if (tb) {
+ /* We can use retranslation to find the PC. */
+ cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
+ tb_phys_invalidate(tb, -1);
} else {
- cpu->tcg_exit_req = 1;
+ /* The exception probably happened in a helper. The CPU state should
+ have been saved before calling it. Fetch the PC from there. */
+ CPUArchState *env = cpu->env_ptr;
+ target_ulong pc, cs_base;
+ tb_page_addr_t addr;
+ int flags;
+
+ cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
+ addr = get_page_addr_code(env, pc);
+ tb_invalidate_phys_range(addr, addr + 1);
}
}
-CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
-
+#ifndef CONFIG_USER_ONLY
/* in deterministic execution mode, instructions doing device I/Os
must be at the end of the TB */
void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
branch. */
#if defined(TARGET_MIPS)
if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
- env->active_tc.PC -= 4;
+ env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
cpu->icount_decr.u16.low++;
env->hflags &= ~MIPS_HFLAG_BMASK;
}
cs_base = tb->cs_base;
flags = tb->flags;
tb_phys_invalidate(tb, -1);
+ if (tb->cflags & CF_NOCACHE) {
+ if (tb->orig_tb) {
+ /* Invalidate original TB if this TB was generated in
+ * cpu_exec_nocache() */
+ tb_phys_invalidate(tb->orig_tb, -1);
+ }
+ tb_free(tb);
+ }
/* FIXME: In theory this could raise an exception. In practice
we have already translated the block once so it's probably ok. */
tb_gen_code(cpu, pc, cs_base, flags, cflags);
tcg_dump_info(f, cpu_fprintf);
}
+void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
+{
+ tcg_dump_op_count(f, cpu_fprintf);
+}
+
#else /* CONFIG_USER_ONLY */
void cpu_interrupt(CPUState *cpu, int mask)
projects / qemu.git / blobdiff