qmp-commands: move 'migrate-incoming' doc to schema

[qemu.git] / translate-all.c

diff --git a/translate-all.c b/translate-all.c
index 6b0ecb4f9f32690200cdee99962ae8207bcd4f20..20262938bb707388d5e8f9b6a6b87376fa8cc4fc 100644 (file)
--- a/translate-all.c
+++ b/translate-all.c
@@ -18,8 +18,6 @@
  */
 #ifdef _WIN32
 #include <windows.h>
  */
 #ifdef _WIN32
 #include <windows.h>

-#else
-#include <sys/mman.h>

 #endif
 #include "qemu/osdep.h"
 
 #endif
 #include "qemu/osdep.h"
 


@@ -29,9 +27,11 @@
 #include "cpu.h"
 #include "trace.h"
 #include "disas/disas.h"
 #include "cpu.h"
 #include "trace.h"
 #include "disas/disas.h"

+#include "exec/exec-all.h"

 #include "tcg.h"
 #if defined(CONFIG_USER_ONLY)
 #include "qemu.h"
 #include "tcg.h"
 #if defined(CONFIG_USER_ONLY)
 #include "qemu.h"

+#include "exec/exec-all.h"

 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 #include <sys/param.h>
 #if __FreeBSD_version >= 700104
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 #include <sys/param.h>
 #if __FreeBSD_version >= 700104


@@ -57,26 +57,50 @@
 #include "qemu/timer.h"
 #include "exec/log.h"
 
 #include "qemu/timer.h"
 #include "exec/log.h"
 

-//#define DEBUG_TB_INVALIDATE
-//#define DEBUG_FLUSH
+/* #define DEBUG_TB_INVALIDATE */
+/* #define DEBUG_TB_FLUSH */
+/* #define DEBUG_LOCKING */

 /* make various TB consistency checks */
 /* make various TB consistency checks */

-//#define DEBUG_TB_CHECK
+/* #define DEBUG_TB_CHECK */

 
 #if !defined(CONFIG_USER_ONLY)
 /* TB consistency checks only implemented for usermode emulation.  */
 #undef DEBUG_TB_CHECK
 #endif
 
 
 #if !defined(CONFIG_USER_ONLY)
 /* TB consistency checks only implemented for usermode emulation.  */
 #undef DEBUG_TB_CHECK
 #endif
 

+/* Access to the various translations structures need to be serialised via locks
+ * for consistency. This is automatic for SoftMMU based system
+ * emulation due to its single threaded nature. In user-mode emulation
+ * access to the memory related structures are protected with the
+ * mmap_lock.
+ */
+#ifdef DEBUG_LOCKING
+#define DEBUG_MEM_LOCKS 1
+#else
+#define DEBUG_MEM_LOCKS 0
+#endif
+
+#ifdef CONFIG_SOFTMMU
+#define assert_memory_lock() do { /* nothing */ } while (0)
+#else
+#define assert_memory_lock() do {               \
+        if (DEBUG_MEM_LOCKS) {                  \
+            g_assert(have_mmap_lock());         \
+        }                                       \
+    } while (0)
+#endif
+

 #define SMC_BITMAP_USE_THRESHOLD 10
 
 typedef struct PageDesc {
     /* list of TBs intersecting this ram page */
     TranslationBlock *first_tb;
 #define SMC_BITMAP_USE_THRESHOLD 10
 
 typedef struct PageDesc {
     /* list of TBs intersecting this ram page */
     TranslationBlock *first_tb;

+#ifdef CONFIG_SOFTMMU

     /* in order to optimize self modifying code, we count the number
        of lookups we do to a given page to use a bitmap */
     unsigned int code_write_count;
     unsigned long *code_bitmap;
     /* in order to optimize self modifying code, we count the number
        of lookups we do to a given page to use a bitmap */
     unsigned int code_write_count;
     unsigned long *code_bitmap;

-#if defined(CONFIG_USER_ONLY)
+#else

     unsigned long flags;
 #endif
 } PageDesc;
     unsigned long flags;
 #endif
 } PageDesc;


@@ -97,34 +121,54 @@ typedef struct PageDesc {
 #define V_L2_BITS 10
 #define V_L2_SIZE (1 << V_L2_BITS)
 
 #define V_L2_BITS 10
 #define V_L2_SIZE (1 << V_L2_BITS)
 

-/* The bits remaining after N lower levels of page tables.  */
-#define V_L1_BITS_REM \
-    ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
-
-#if V_L1_BITS_REM < 4
-#define V_L1_BITS  (V_L1_BITS_REM + V_L2_BITS)
-#else
-#define V_L1_BITS  V_L1_BITS_REM
-#endif
-
-#define V_L1_SIZE  ((target_ulong)1 << V_L1_BITS)
-
-#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
-

 uintptr_t qemu_host_page_size;
 intptr_t qemu_host_page_mask;
 
 uintptr_t qemu_host_page_size;
 intptr_t qemu_host_page_mask;
 

-/* The bottom level has pointers to PageDesc */
-static void *l1_map[V_L1_SIZE];
+/*
+ * L1 Mapping properties
+ */
+static int v_l1_size;
+static int v_l1_shift;
+static int v_l2_levels;
+
+/* The bottom level has pointers to PageDesc, and is indexed by
+ * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
+ */
+#define V_L1_MIN_BITS 4
+#define V_L1_MAX_BITS (V_L2_BITS + 3)
+#define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
+
+static void *l1_map[V_L1_MAX_SIZE];

 
 /* code generation context */
 TCGContext tcg_ctx;
 
 /* code generation context */
 TCGContext tcg_ctx;

+bool parallel_cpus;

 
 /* translation block context */
 #ifdef CONFIG_USER_ONLY
 __thread int have_tb_lock;
 #endif
 
 
 /* translation block context */
 #ifdef CONFIG_USER_ONLY
 __thread int have_tb_lock;
 #endif
 

+static void page_table_config_init(void)
+{
+    uint32_t v_l1_bits;
+
+    assert(TARGET_PAGE_BITS);
+    /* The bits remaining after N lower levels of page tables.  */
+    v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS;
+    if (v_l1_bits < V_L1_MIN_BITS) {
+        v_l1_bits += V_L2_BITS;
+    }
+
+    v_l1_size = 1 << v_l1_bits;
+    v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits;
+    v_l2_levels = v_l1_shift / V_L2_BITS - 1;
+
+    assert(v_l1_bits <= V_L1_MAX_BITS);
+    assert(v_l1_shift % V_L2_BITS == 0);
+    assert(v_l2_levels >= 0);
+}
+

 void tb_lock(void)
 {
 #ifdef CONFIG_USER_ONLY
 void tb_lock(void)
 {
 #ifdef CONFIG_USER_ONLY


@@ -153,8 +197,23 @@ void tb_lock_reset(void)
 #endif
 }
 
 #endif
 }
 

-static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
-                         tb_page_addr_t phys_page2);
+#ifdef DEBUG_LOCKING
+#define DEBUG_TB_LOCKS 1
+#else
+#define DEBUG_TB_LOCKS 0
+#endif
+
+#ifdef CONFIG_SOFTMMU
+#define assert_tb_lock() do { /* nothing */ } while (0)
+#else
+#define assert_tb_lock() do {               \
+        if (DEBUG_TB_LOCKS) {               \
+            g_assert(have_tb_lock);         \
+        }                                   \
+    } while (0)
+#endif
+
+

 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
 
 void cpu_gen_init(void)
 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
 
 void cpu_gen_init(void)


@@ -249,7 +308,9 @@ static int encode_search(TranslationBlock *tb, uint8_t *block)
     return p - block;
 }
 
     return p - block;
 }
 

-/* The cpu state corresponding to 'searched_pc' is restored.  */
+/* The cpu state corresponding to 'searched_pc' is restored.
+ * Called with tb_lock held.
+ */

 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
                                      uintptr_t searched_pc)
 {
 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
                                      uintptr_t searched_pc)
 {


@@ -262,6 +323,8 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
     int64_t ti = profile_getclock();
 #endif
 
     int64_t ti = profile_getclock();
 #endif
 

+    searched_pc -= GETPC_ADJ;
+

     if (searched_pc < host_pc) {
         return -1;
     }
     if (searched_pc < host_pc) {
         return -1;
     }


@@ -300,19 +363,22 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
 {
     TranslationBlock *tb;
 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
 {
     TranslationBlock *tb;

+    bool r = false;

 
 

+    tb_lock();

     tb = tb_find_pc(retaddr);
     if (tb) {
         cpu_restore_state_from_tb(cpu, tb, retaddr);
         if (tb->cflags & CF_NOCACHE) {
             /* one-shot translation, invalidate it immediately */
     tb = tb_find_pc(retaddr);
     if (tb) {
         cpu_restore_state_from_tb(cpu, tb, retaddr);
         if (tb->cflags & CF_NOCACHE) {
             /* one-shot translation, invalidate it immediately */

-            cpu->current_tb = NULL;

             tb_phys_invalidate(tb, -1);
             tb_free(tb);
         }
             tb_phys_invalidate(tb, -1);
             tb_free(tb);
         }

-        return true;
+        r = true;

     }
     }

-    return false;
+    tb_unlock();
+
+    return r;

 }
 
 void page_size_init(void)
 }
 
 void page_size_init(void)


@@ -333,6 +399,8 @@ void page_size_init(void)
 static void page_init(void)
 {
     page_size_init();
 static void page_init(void)
 {
     page_size_init();

+    page_table_config_init();
+

 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
     {
 #ifdef HAVE_KINFO_GETVMMAP
 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
     {
 #ifdef HAVE_KINFO_GETVMMAP


@@ -400,6 +468,7 @@ static void page_init(void)
 }
 
 /* If alloc=1:
 }
 
 /* If alloc=1:

+ * Called with tb_lock held for system emulation.

  * Called with mmap_lock held for user-mode emulation.
  */
 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
  * Called with mmap_lock held for user-mode emulation.
  */
 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)


@@ -408,11 +477,15 @@ static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
     void **lp;
     int i;
 
     void **lp;
     int i;
 

+    if (alloc) {
+        assert_memory_lock();
+    }
+

     /* Level 1.  Always allocated.  */
     /* Level 1.  Always allocated.  */

-    lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
+    lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1));

 
     /* Level 2..N-1.  */
 
     /* Level 2..N-1.  */

-    for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
+    for (i = v_l2_levels; i > 0; i--) {

         void **p = atomic_rcu_read(lp);
 
         if (p == NULL) {
         void **p = atomic_rcu_read(lp);
 
         if (p == NULL) {


@@ -515,7 +588,7 @@ static inline size_t size_code_gen_buffer(size_t tb_size)
    that the buffer not cross a 256MB boundary.  */
 static inline bool cross_256mb(void *addr, size_t size)
 {
    that the buffer not cross a 256MB boundary.  */
 static inline bool cross_256mb(void *addr, size_t size)
 {

-    return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & 0xf0000000;
+    return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;

 }
 
 /* We weren't able to allocate a buffer without crossing that boundary,
 }
 
 /* We weren't able to allocate a buffer without crossing that boundary,


@@ -523,7 +596,7 @@ static inline bool cross_256mb(void *addr, size_t size)
    Returns the new base of the buffer, and adjusts code_gen_buffer_size.  */
 static inline void *split_cross_256mb(void *buf1, size_t size1)
 {
    Returns the new base of the buffer, and adjusts code_gen_buffer_size.  */
 static inline void *split_cross_256mb(void *buf1, size_t size1)
 {

-    void *buf2 = (void *)(((uintptr_t)buf1 + size1) & 0xf0000000);
+    void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);

     size_t size2 = buf1 + size1 - buf2;
 
     size1 = buf2 - buf1;
     size_t size2 = buf1 + size1 - buf2;
 
     size1 = buf2 - buf1;


@@ -680,15 +753,15 @@ static inline void *alloc_code_gen_buffer(void)
         size_t size2;
         void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
                           PROT_NONE, flags, -1, 0);
         size_t size2;
         void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
                           PROT_NONE, flags, -1, 0);

-        switch (buf2 != MAP_FAILED) {
+        switch ((int)(buf2 != MAP_FAILED)) {

         case 1:
             if (!cross_256mb(buf2, size)) {
                 /* Success!  Use the new buffer.  */
         case 1:
             if (!cross_256mb(buf2, size)) {
                 /* Success!  Use the new buffer.  */

-                munmap(buf, size);
+                munmap(buf, size + qemu_real_host_page_size);

                 break;
             }
             /* Failure.  Work with what we had.  */
                 break;
             }
             /* Failure.  Work with what we had.  */

-            munmap(buf2, size);
+            munmap(buf2, size + qemu_real_host_page_size);

             /* fallthru */
         default:
             /* Split the original buffer.  Free the smaller half.  */
             /* fallthru */
         default:
             /* Split the original buffer.  Free the smaller half.  */


@@ -736,6 +809,13 @@ static inline void code_gen_alloc(size_t tb_size)
     qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
 }
 
     qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
 }
 

+static void tb_htable_init(void)
+{
+    unsigned int mode = QHT_MODE_AUTO_RESIZE;
+
+    qht_init(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode);
+}
+

 /* Must be called before using the QEMU cpus. 'tb_size' is the size
    (in bytes) allocated to the translation buffer. Zero means default
    size. */
 /* Must be called before using the QEMU cpus. 'tb_size' is the size
    (in bytes) allocated to the translation buffer. Zero means default
    size. */


@@ -743,6 +823,7 @@ void tcg_exec_init(unsigned long tb_size)
 {
     cpu_gen_init();
     page_init();
 {
     cpu_gen_init();
     page_init();

+    tb_htable_init();

     code_gen_alloc(tb_size);
 #if defined(CONFIG_SOFTMMU)
     /* There's no guest base to take into account, so go ahead and
     code_gen_alloc(tb_size);
 #if defined(CONFIG_SOFTMMU)
     /* There's no guest base to take into account, so go ahead and


@@ -756,23 +837,33 @@ bool tcg_enabled(void)
     return tcg_ctx.code_gen_buffer != NULL;
 }
 
     return tcg_ctx.code_gen_buffer != NULL;
 }
 

-/* Allocate a new translation block. Flush the translation buffer if
-   too many translation blocks or too much generated code. */
+/*
+ * Allocate a new translation block. Flush the translation buffer if
+ * too many translation blocks or too much generated code.
+ *
+ * Called with tb_lock held.
+ */

 static TranslationBlock *tb_alloc(target_ulong pc)
 {
     TranslationBlock *tb;
 
 static TranslationBlock *tb_alloc(target_ulong pc)
 {
     TranslationBlock *tb;
 

+    assert_tb_lock();
+

     if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks) {
         return NULL;
     }
     tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
     tb->pc = pc;
     tb->cflags = 0;
     if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks) {
         return NULL;
     }
     tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
     tb->pc = pc;
     tb->cflags = 0;

+    tb->invalid = false;

     return tb;
 }
 
     return tb;
 }
 

+/* Called with tb_lock held.  */

 void tb_free(TranslationBlock *tb)
 {
 void tb_free(TranslationBlock *tb)
 {

+    assert_tb_lock();
+

     /* In practice this is mostly used for single use temporary TB
        Ignore the hard cases and just back up if this TB happens to
        be the last one generated.  */
     /* In practice this is mostly used for single use temporary TB
        Ignore the hard cases and just back up if this TB happens to
        be the last one generated.  */


@@ -785,9 +876,11 @@ void tb_free(TranslationBlock *tb)
 
 static inline void invalidate_page_bitmap(PageDesc *p)
 {
 
 static inline void invalidate_page_bitmap(PageDesc *p)
 {

+#ifdef CONFIG_SOFTMMU

     g_free(p->code_bitmap);
     p->code_bitmap = NULL;
     p->code_write_count = 0;
     g_free(p->code_bitmap);
     p->code_bitmap = NULL;
     p->code_write_count = 0;

+#endif

 }
 
 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
 }
 
 /* Set to NULL all the 'first_tb' fields in all PageDescs. */


@@ -816,18 +909,26 @@ static void page_flush_tb_1(int level, void **lp)
 
 static void page_flush_tb(void)
 {
 
 static void page_flush_tb(void)
 {

-    int i;
+    int i, l1_sz = v_l1_size;

 
 

-    for (i = 0; i < V_L1_SIZE; i++) {
-        page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
+    for (i = 0; i < l1_sz; i++) {
+        page_flush_tb_1(v_l2_levels, l1_map + i);

     }
 }
 
 /* flush all the translation blocks */
     }
 }
 
 /* flush all the translation blocks */

-/* XXX: tb_flush is currently not thread safe */
-void tb_flush(CPUState *cpu)
+static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)

 {
 {

-#if defined(DEBUG_FLUSH)
+    tb_lock();
+
+    /* If it is already been done on request of another CPU,
+     * just retry.
+     */
+    if (tcg_ctx.tb_ctx.tb_flush_count != tb_flush_count.host_int) {
+        goto done;
+    }
+
+#if defined(DEBUG_TB_FLUSH)

     printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
            (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
            tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
     printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
            (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
            tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?


@@ -838,77 +939,84 @@ void tb_flush(CPUState *cpu)
         > tcg_ctx.code_gen_buffer_size) {
         cpu_abort(cpu, "Internal error: code buffer overflow\n");
     }
         > tcg_ctx.code_gen_buffer_size) {
         cpu_abort(cpu, "Internal error: code buffer overflow\n");
     }

-    tcg_ctx.tb_ctx.nb_tbs = 0;

 
     CPU_FOREACH(cpu) {
 
     CPU_FOREACH(cpu) {

-        memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
+        int i;
+
+        for (i = 0; i < TB_JMP_CACHE_SIZE; ++i) {
+            atomic_set(&cpu->tb_jmp_cache[i], NULL);
+        }

     }
 
     }
 

-    memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
+    tcg_ctx.tb_ctx.nb_tbs = 0;
+    qht_reset_size(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE);

     page_flush_tb();
 
     tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
     /* XXX: flush processor icache at this point if cache flush is
        expensive */
     page_flush_tb();
 
     tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
     /* XXX: flush processor icache at this point if cache flush is
        expensive */

-    tcg_ctx.tb_ctx.tb_flush_count++;
+    atomic_mb_set(&tcg_ctx.tb_ctx.tb_flush_count,
+                  tcg_ctx.tb_ctx.tb_flush_count + 1);
+
+done:
+    tb_unlock();
+}
+
+void tb_flush(CPUState *cpu)
+{
+    if (tcg_enabled()) {
+        unsigned tb_flush_count = atomic_mb_read(&tcg_ctx.tb_ctx.tb_flush_count);
+        async_safe_run_on_cpu(cpu, do_tb_flush,
+                              RUN_ON_CPU_HOST_INT(tb_flush_count));
+    }

 }
 
 #ifdef DEBUG_TB_CHECK
 
 }
 
 #ifdef DEBUG_TB_CHECK
 

-static void tb_invalidate_check(target_ulong address)
+static void
+do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp)

 {
 {

-    TranslationBlock *tb;
-    int i;
+    TranslationBlock *tb = p;
+    target_ulong addr = *(target_ulong *)userp;

 
 

-    address &= TARGET_PAGE_MASK;
-    for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
-        for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
-             tb = tb->phys_hash_next) {
-            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
-                  address >= tb->pc + tb->size)) {
-                printf("ERROR invalidate: address=" TARGET_FMT_lx
-                       " PC=%08lx size=%04x\n",
-                       address, (long)tb->pc, tb->size);
-            }
-        }
+    if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) {
+        printf("ERROR invalidate: address=" TARGET_FMT_lx
+               " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size);

     }
 }
 
     }
 }
 

-/* verify that all the pages have correct rights for code */
-static void tb_page_check(void)
+/* verify that all the pages have correct rights for code
+ *
+ * Called with tb_lock held.
+ */
+static void tb_invalidate_check(target_ulong address)

 {
 {

-    TranslationBlock *tb;
-    int i, flags1, flags2;
-
-    for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
-        for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
-                tb = tb->phys_hash_next) {
-            flags1 = page_get_flags(tb->pc);
-            flags2 = page_get_flags(tb->pc + tb->size - 1);
-            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
-                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
-                       (long)tb->pc, tb->size, flags1, flags2);
-            }
-        }
-    }
+    address &= TARGET_PAGE_MASK;
+    qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_invalidate_check, &address);

 }
 
 }
 

-#endif
-
-static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
+static void
+do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp)

 {
 {

-    TranslationBlock *tb1;
+    TranslationBlock *tb = p;
+    int flags1, flags2;

 
 

-    for (;;) {
-        tb1 = *ptb;
-        if (tb1 == tb) {
-            *ptb = tb1->phys_hash_next;
-            break;
-        }
-        ptb = &tb1->phys_hash_next;
+    flags1 = page_get_flags(tb->pc);
+    flags2 = page_get_flags(tb->pc + tb->size - 1);
+    if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
+        printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
+               (long)tb->pc, tb->size, flags1, flags2);

     }
 }
 
     }
 }
 

+/* verify that all the pages have correct rights for code */
+static void tb_page_check(void)
+{
+    qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_page_check, NULL);
+}
+
+#endif
+

 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
 {
     TranslationBlock *tb1;
 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
 {
     TranslationBlock *tb1;


@@ -926,32 +1034,33 @@ static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
     }
 }
 
     }
 }
 

-static inline void tb_jmp_remove(TranslationBlock *tb, int n)
+/* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
+static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n)

 {
 {

-    TranslationBlock *tb1, **ptb;
+    TranslationBlock *tb1;
+    uintptr_t *ptb, ntb;

     unsigned int n1;
 
     unsigned int n1;
 

-    ptb = &tb->jmp_next[n];
-    tb1 = *ptb;
-    if (tb1) {
+    ptb = &tb->jmp_list_next[n];
+    if (*ptb) {

         /* find tb(n) in circular list */
         for (;;) {
         /* find tb(n) in circular list */
         for (;;) {

-            tb1 = *ptb;
-            n1 = (uintptr_t)tb1 & 3;
-            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+            ntb = *ptb;
+            n1 = ntb & 3;
+            tb1 = (TranslationBlock *)(ntb & ~3);

             if (n1 == n && tb1 == tb) {
                 break;
             }
             if (n1 == 2) {
             if (n1 == n && tb1 == tb) {
                 break;
             }
             if (n1 == 2) {

-                ptb = &tb1->jmp_first;
+                ptb = &tb1->jmp_list_first;

             } else {
             } else {

-                ptb = &tb1->jmp_next[n1];
+                ptb = &tb1->jmp_list_next[n1];

             }
         }
         /* now we can suppress tb(n) from the list */
             }
         }
         /* now we can suppress tb(n) from the list */

-        *ptb = tb->jmp_next[n];
+        *ptb = tb->jmp_list_next[n];

 
 

-        tb->jmp_next[n] = NULL;
+        tb->jmp_list_next[n] = (uintptr_t)NULL;

     }
 }
 
     }
 }
 


@@ -959,22 +1068,50 @@ static inline void tb_jmp_remove(TranslationBlock *tb, int n)
    another TB */
 static inline void tb_reset_jump(TranslationBlock *tb, int n)
 {
    another TB */
 static inline void tb_reset_jump(TranslationBlock *tb, int n)
 {

-    tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
+    uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]);
+    tb_set_jmp_target(tb, n, addr);
+}
+
+/* remove any jumps to the TB */
+static inline void tb_jmp_unlink(TranslationBlock *tb)
+{
+    TranslationBlock *tb1;
+    uintptr_t *ptb, ntb;
+    unsigned int n1;
+
+    ptb = &tb->jmp_list_first;
+    for (;;) {
+        ntb = *ptb;
+        n1 = ntb & 3;
+        tb1 = (TranslationBlock *)(ntb & ~3);
+        if (n1 == 2) {
+            break;
+        }
+        tb_reset_jump(tb1, n1);
+        *ptb = tb1->jmp_list_next[n1];
+        tb1->jmp_list_next[n1] = (uintptr_t)NULL;
+    }

 }
 
 }
 

-/* invalidate one TB */
+/* invalidate one TB
+ *
+ * Called with tb_lock held.
+ */

 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
 {
     CPUState *cpu;
     PageDesc *p;
 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
 {
     CPUState *cpu;
     PageDesc *p;

-    unsigned int h, n1;
+    uint32_t h;

     tb_page_addr_t phys_pc;
     tb_page_addr_t phys_pc;

-    TranslationBlock *tb1, *tb2;
+
+    assert_tb_lock();
+
+    atomic_set(&tb->invalid, true);

 
     /* remove the TB from the hash list */
     phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
 
     /* remove the TB from the hash list */
     phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);

-    h = tb_phys_hash_func(phys_pc);
-    tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
+    h = tb_hash_func(phys_pc, tb->pc, tb->flags);
+    qht_remove(&tcg_ctx.tb_ctx.htable, tb, h);

 
     /* remove the TB from the page list */
     if (tb->page_addr[0] != page_addr) {
 
     /* remove the TB from the page list */
     if (tb->page_addr[0] != page_addr) {


@@ -988,38 +1125,25 @@ void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
         invalidate_page_bitmap(p);
     }
 
         invalidate_page_bitmap(p);
     }
 

-    tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
-

     /* remove the TB from the hash list */
     h = tb_jmp_cache_hash_func(tb->pc);
     CPU_FOREACH(cpu) {
     /* remove the TB from the hash list */
     h = tb_jmp_cache_hash_func(tb->pc);
     CPU_FOREACH(cpu) {

-        if (cpu->tb_jmp_cache[h] == tb) {
-            cpu->tb_jmp_cache[h] = NULL;
+        if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) {
+            atomic_set(&cpu->tb_jmp_cache[h], NULL);

         }
     }
 
     /* suppress this TB from the two jump lists */
         }
     }
 
     /* suppress this TB from the two jump lists */

-    tb_jmp_remove(tb, 0);
-    tb_jmp_remove(tb, 1);
+    tb_remove_from_jmp_list(tb, 0);
+    tb_remove_from_jmp_list(tb, 1);

 
     /* suppress any remaining jumps to this TB */
 
     /* suppress any remaining jumps to this TB */

-    tb1 = tb->jmp_first;
-    for (;;) {
-        n1 = (uintptr_t)tb1 & 3;
-        if (n1 == 2) {
-            break;
-        }
-        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-        tb2 = tb1->jmp_next[n1];
-        tb_reset_jump(tb1, n1);
-        tb1->jmp_next[n1] = NULL;
-        tb1 = tb2;
-    }
-    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
+    tb_jmp_unlink(tb);

 
     tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
 }
 
 
     tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
 }
 

+#ifdef CONFIG_SOFTMMU

 static void build_page_bitmap(PageDesc *p)
 {
     int n, tb_start, tb_end;
 static void build_page_bitmap(PageDesc *p)
 {
     int n, tb_start, tb_end;


@@ -1039,7 +1163,7 @@ static void build_page_bitmap(PageDesc *p)
             tb_end = tb_start + tb->size;
             if (tb_end > TARGET_PAGE_SIZE) {
                 tb_end = TARGET_PAGE_SIZE;
             tb_end = tb_start + tb->size;
             if (tb_end > TARGET_PAGE_SIZE) {
                 tb_end = TARGET_PAGE_SIZE;

-            }
+             }

         } else {
             tb_start = 0;
             tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
         } else {
             tb_start = 0;
             tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);


@@ -1048,6 +1172,96 @@ static void build_page_bitmap(PageDesc *p)
         tb = tb->page_next[n];
     }
 }
         tb = tb->page_next[n];
     }
 }

+#endif
+
+/* add the tb in the target page and protect it if necessary
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
+static inline void tb_alloc_page(TranslationBlock *tb,
+                                 unsigned int n, tb_page_addr_t page_addr)
+{
+    PageDesc *p;
+#ifndef CONFIG_USER_ONLY
+    bool page_already_protected;
+#endif
+
+    assert_memory_lock();
+
+    tb->page_addr[n] = page_addr;
+    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
+    tb->page_next[n] = p->first_tb;
+#ifndef CONFIG_USER_ONLY
+    page_already_protected = p->first_tb != NULL;
+#endif
+    p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
+    invalidate_page_bitmap(p);
+
+#if defined(CONFIG_USER_ONLY)
+    if (p->flags & PAGE_WRITE) {
+        target_ulong addr;
+        PageDesc *p2;
+        int prot;
+
+        /* force the host page as non writable (writes will have a
+           page fault + mprotect overhead) */
+        page_addr &= qemu_host_page_mask;
+        prot = 0;
+        for (addr = page_addr; addr < page_addr + qemu_host_page_size;
+            addr += TARGET_PAGE_SIZE) {
+
+            p2 = page_find(addr >> TARGET_PAGE_BITS);
+            if (!p2) {
+                continue;
+            }
+            prot |= p2->flags;
+            p2->flags &= ~PAGE_WRITE;
+          }
+        mprotect(g2h(page_addr), qemu_host_page_size,
+                 (prot & PAGE_BITS) & ~PAGE_WRITE);
+#ifdef DEBUG_TB_INVALIDATE
+        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
+               page_addr);
+#endif
+    }
+#else
+    /* if some code is already present, then the pages are already
+       protected. So we handle the case where only the first TB is
+       allocated in a physical page */
+    if (!page_already_protected) {
+        tlb_protect_code(page_addr);
+    }
+#endif
+}
+
+/* add a new TB and link it to the physical page tables. phys_page2 is
+ * (-1) to indicate that only one page contains the TB.
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
+static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+                         tb_page_addr_t phys_page2)
+{
+    uint32_t h;
+
+    assert_memory_lock();
+
+    /* add in the page list */
+    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
+    if (phys_page2 != -1) {
+        tb_alloc_page(tb, 1, phys_page2);
+    } else {
+        tb->page_addr[1] = -1;
+    }
+
+    /* add in the hash table */
+    h = tb_hash_func(phys_pc, tb->pc, tb->flags);
+    qht_insert(&tcg_ctx.tb_ctx.htable, tb, h);
+
+#ifdef DEBUG_TB_CHECK
+    tb_page_check();
+#endif
+}

 
 /* Called with mmap_lock held for user mode emulation.  */
 TranslationBlock *tb_gen_code(CPUState *cpu,
 
 /* Called with mmap_lock held for user mode emulation.  */
 TranslationBlock *tb_gen_code(CPUState *cpu,


@@ -1063,6 +1277,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
 #ifdef CONFIG_PROFILER
     int64_t ti;
 #endif
 #ifdef CONFIG_PROFILER
     int64_t ti;
 #endif

+    assert_memory_lock();

 
     phys_pc = get_page_addr_code(env, pc);
     if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {
 
     phys_pc = get_page_addr_code(env, pc);
     if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {


@@ -1074,11 +1289,8 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
  buffer_overflow:
         /* flush must be done */
         tb_flush(cpu);
  buffer_overflow:
         /* flush must be done */
         tb_flush(cpu);

-        /* cannot fail at this point */
-        tb = tb_alloc(pc);
-        assert(tb != NULL);
-        /* Don't forget to invalidate previous TB info.  */
-        tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
+        mmap_unlock();
+        cpu_loop_exit(cpu);

     }
 
     gen_code_buf = tcg_ctx.code_gen_ptr;
     }
 
     gen_code_buf = tcg_ctx.code_gen_ptr;


@@ -1095,20 +1307,22 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
 
     tcg_func_start(&tcg_ctx);
 
 
     tcg_func_start(&tcg_ctx);
 

+    tcg_ctx.cpu = ENV_GET_CPU(env);

     gen_intermediate_code(env, tb);
     gen_intermediate_code(env, tb);

+    tcg_ctx.cpu = NULL;

 
     trace_translate_block(tb, tb->pc, tb->tc_ptr);
 
     /* generate machine code */
 
     trace_translate_block(tb, tb->pc, tb->tc_ptr);
 
     /* generate machine code */

-    tb->tb_next_offset[0] = 0xffff;
-    tb->tb_next_offset[1] = 0xffff;
-    tcg_ctx.tb_next_offset = tb->tb_next_offset;
+    tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
+    tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
+    tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset;

 #ifdef USE_DIRECT_JUMP
 #ifdef USE_DIRECT_JUMP

-    tcg_ctx.tb_jmp_offset = tb->tb_jmp_offset;
-    tcg_ctx.tb_next = NULL;
+    tcg_ctx.tb_jmp_insn_offset = tb->jmp_insn_offset;
+    tcg_ctx.tb_jmp_target_addr = NULL;

 #else
 #else

-    tcg_ctx.tb_jmp_offset = NULL;
-    tcg_ctx.tb_next = tb->tb_next;
+    tcg_ctx.tb_jmp_insn_offset = NULL;
+    tcg_ctx.tb_jmp_target_addr = tb->jmp_target_addr;

 #endif
 
 #ifdef CONFIG_PROFILER
 #endif
 
 #ifdef CONFIG_PROFILER


@@ -1141,10 +1355,12 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
         qemu_log_in_addr_range(tb->pc)) {
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
         qemu_log_in_addr_range(tb->pc)) {

+        qemu_log_lock();

         qemu_log("OUT: [size=%d]\n", gen_code_size);
         log_disas(tb->tc_ptr, gen_code_size);
         qemu_log("\n");
         qemu_log_flush();
         qemu_log("OUT: [size=%d]\n", gen_code_size);
         log_disas(tb->tc_ptr, gen_code_size);
         qemu_log("\n");
         qemu_log_flush();

+        qemu_log_unlock();

     }
 #endif
 
     }
 #endif
 


@@ -1152,12 +1368,31 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
         ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
                  CODE_GEN_ALIGN);
 
         ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
                  CODE_GEN_ALIGN);
 

+    /* init jump list */
+    assert(((uintptr_t)tb & 3) == 0);
+    tb->jmp_list_first = (uintptr_t)tb | 2;
+    tb->jmp_list_next[0] = (uintptr_t)NULL;
+    tb->jmp_list_next[1] = (uintptr_t)NULL;
+
+    /* init original jump addresses wich has been set during tcg_gen_code() */
+    if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
+        tb_reset_jump(tb, 0);
+    }
+    if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
+        tb_reset_jump(tb, 1);
+    }
+

     /* check next page if needed */
     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
     phys_page2 = -1;
     if ((pc & TARGET_PAGE_MASK) != virt_page2) {
         phys_page2 = get_page_addr_code(env, virt_page2);
     }
     /* check next page if needed */
     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
     phys_page2 = -1;
     if ((pc & TARGET_PAGE_MASK) != virt_page2) {
         phys_page2 = get_page_addr_code(env, virt_page2);
     }

+    /* As long as consistency of the TB stuff is provided by tb_lock in user
+     * mode and is implicit in single-threaded softmmu emulation, no explicit
+     * memory barrier is required before tb_link_page() makes the TB visible
+     * through the physical hash table and physical page list.
+     */

     tb_link_page(tb, phys_pc, phys_page2);
     return tb;
 }
     tb_link_page(tb, phys_pc, phys_page2);
     return tb;
 }


@@ -1169,9 +1404,10 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
  *
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
  *

- * Called with mmap_lock held for user-mode emulation
+ * Called with mmap_lock held for user-mode emulation, grabs tb_lock
+ * Called with tb_lock held for system-mode emulation

  */
  */

-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
+static void tb_invalidate_phys_range_1(tb_page_addr_t start, tb_page_addr_t end)

 {
     while (start < end) {
         tb_invalidate_phys_page_range(start, end, 0);
 {
     while (start < end) {
         tb_invalidate_phys_page_range(start, end, 0);


@@ -1180,6 +1416,21 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
     }
 }
 
     }
 }
 

+#ifdef CONFIG_SOFTMMU
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
+{
+    assert_tb_lock();
+    tb_invalidate_phys_range_1(start, end);
+}
+#else
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
+{
+    assert_memory_lock();
+    tb_lock();
+    tb_invalidate_phys_range_1(start, end);
+    tb_unlock();
+}
+#endif

 /*
  * Invalidate all TBs which intersect with the target physical address range
  * [start;end[. NOTE: start and end must refer to the *same* physical page.
 /*
  * Invalidate all TBs which intersect with the target physical address range
  * [start;end[. NOTE: start and end must refer to the *same* physical page.


@@ -1187,14 +1438,15 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
  *
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
  *

- * Called with mmap_lock held for user-mode emulation
+ * Called with tb_lock/mmap_lock held for user-mode emulation
+ * Called with tb_lock held for system-mode emulation

  */
 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
                                    int is_cpu_write_access)
 {
  */
 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
                                    int is_cpu_write_access)
 {

-    TranslationBlock *tb, *tb_next, *saved_tb;
-    CPUState *cpu = current_cpu;
+    TranslationBlock *tb, *tb_next;

 #if defined(TARGET_HAS_PRECISE_SMC)
 #if defined(TARGET_HAS_PRECISE_SMC)

+    CPUState *cpu = current_cpu;

     CPUArchState *env = NULL;
 #endif
     tb_page_addr_t tb_start, tb_end;
     CPUArchState *env = NULL;
 #endif
     tb_page_addr_t tb_start, tb_end;


@@ -1209,6 +1461,9 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
     uint32_t current_flags = 0;
 #endif /* TARGET_HAS_PRECISE_SMC */
 
     uint32_t current_flags = 0;
 #endif /* TARGET_HAS_PRECISE_SMC */
 

+    assert_memory_lock();
+    assert_tb_lock();
+

     p = page_find(start >> TARGET_PAGE_BITS);
     if (!p) {
         return;
     p = page_find(start >> TARGET_PAGE_BITS);
     if (!p) {
         return;


@@ -1261,20 +1516,7 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
                                      &current_flags);
             }
 #endif /* TARGET_HAS_PRECISE_SMC */
                                      &current_flags);
             }
 #endif /* TARGET_HAS_PRECISE_SMC */

-            /* we need to do that to handle the case where a signal
-               occurs while doing tb_phys_invalidate() */
-            saved_tb = NULL;
-            if (cpu != NULL) {
-                saved_tb = cpu->current_tb;
-                cpu->current_tb = NULL;
-            }

             tb_phys_invalidate(tb, -1);
             tb_phys_invalidate(tb, -1);

-            if (cpu != NULL) {
-                cpu->current_tb = saved_tb;
-                if (cpu->interrupt_request && cpu->current_tb) {
-                    cpu_interrupt(cpu, cpu->interrupt_request);
-                }
-            }

         }
         tb = tb_next;
     }
         }
         tb = tb_next;
     }


@@ -1290,14 +1532,17 @@ void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
         /* we generate a block containing just the instruction
            modifying the memory. It will ensure that it cannot modify
            itself */
         /* we generate a block containing just the instruction
            modifying the memory. It will ensure that it cannot modify
            itself */

-        cpu->current_tb = NULL;

         tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
         tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);

-        cpu_resume_from_signal(cpu, NULL);
+        cpu_loop_exit_noexc(cpu);

     }
 #endif
 }
 
     }
 #endif
 }
 

-/* len must be <= 8 and start must be a multiple of len */
+#ifdef CONFIG_SOFTMMU
+/* len must be <= 8 and start must be a multiple of len.
+ * Called via softmmu_template.h when code areas are written to with
+ * tb_lock held.
+ */

 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
 {
     PageDesc *p;
 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
 {
     PageDesc *p;


@@ -1311,13 +1556,17 @@ void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
                   (intptr_t)cpu_single_env->segs[R_CS].base);
     }
 #endif
                   (intptr_t)cpu_single_env->segs[R_CS].base);
     }
 #endif

+    assert_memory_lock();
+

     p = page_find(start >> TARGET_PAGE_BITS);
     if (!p) {
         return;
     }
     if (!p->code_bitmap &&
         ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
     p = page_find(start >> TARGET_PAGE_BITS);
     if (!p) {
         return;
     }
     if (!p->code_bitmap &&
         ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {

-        /* build code bitmap */
+        /* build code bitmap.  FIXME: writes should be protected by
+         * tb_lock, reads by tb_lock or RCU.
+         */

         build_page_bitmap(p);
     }
     if (p->code_bitmap) {
         build_page_bitmap(p);
     }
     if (p->code_bitmap) {


@@ -1334,12 +1583,14 @@ void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
         tb_invalidate_phys_page_range(start, start + len, 1);
     }
 }
         tb_invalidate_phys_page_range(start, start + len, 1);
     }
 }

-
-#if !defined(CONFIG_SOFTMMU)
-/* Called with mmap_lock held.  */
-static void tb_invalidate_phys_page(tb_page_addr_t addr,
-                                    uintptr_t pc, void *puc,
-                                    bool locked)
+#else
+/* Called with mmap_lock held. If pc is not 0 then it indicates the
+ * host PC of the faulting store instruction that caused this invalidate.
+ * Returns true if the caller needs to abort execution of the current
+ * TB (because it was modified by this store and the guest CPU has
+ * precise-SMC semantics).
+ */
+static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)

 {
     TranslationBlock *tb;
     PageDesc *p;
 {
     TranslationBlock *tb;
     PageDesc *p;


@@ -1354,11 +1605,15 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr,
     uint32_t current_flags = 0;
 #endif
 
     uint32_t current_flags = 0;
 #endif
 

+    assert_memory_lock();
+

     addr &= TARGET_PAGE_MASK;
     p = page_find(addr >> TARGET_PAGE_BITS);
     if (!p) {
     addr &= TARGET_PAGE_MASK;
     p = page_find(addr >> TARGET_PAGE_BITS);
     if (!p) {

-        return;
+        return false;

     }
     }

+
+    tb_lock();

     tb = p->first_tb;
 #ifdef TARGET_HAS_PRECISE_SMC
     if (tb && pc != 0) {
     tb = p->first_tb;
 #ifdef TARGET_HAS_PRECISE_SMC
     if (tb && pc != 0) {


@@ -1395,116 +1650,17 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr,
         /* we generate a block containing just the instruction
            modifying the memory. It will ensure that it cannot modify
            itself */
         /* we generate a block containing just the instruction
            modifying the memory. It will ensure that it cannot modify
            itself */

-        cpu->current_tb = NULL;

         tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
         tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);

-        if (locked) {
-            mmap_unlock();
-        }
-        cpu_resume_from_signal(cpu, puc);
+        /* tb_lock will be reset after cpu_loop_exit_noexc longjmps
+         * back into the cpu_exec loop. */
+        return true;

     }
 #endif
     }
 #endif

-}
-#endif
-
-/* add the tb in the target page and protect it if necessary
- *
- * Called with mmap_lock held for user-mode emulation.
- */
-static inline void tb_alloc_page(TranslationBlock *tb,
-                                 unsigned int n, tb_page_addr_t page_addr)
-{
-    PageDesc *p;
-#ifndef CONFIG_USER_ONLY
-    bool page_already_protected;
-#endif
-
-    tb->page_addr[n] = page_addr;
-    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
-    tb->page_next[n] = p->first_tb;
-#ifndef CONFIG_USER_ONLY
-    page_already_protected = p->first_tb != NULL;
-#endif
-    p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
-    invalidate_page_bitmap(p);
-
-#if defined(CONFIG_USER_ONLY)
-    if (p->flags & PAGE_WRITE) {
-        target_ulong addr;
-        PageDesc *p2;
-        int prot;
+    tb_unlock();

 
 

-        /* force the host page as non writable (writes will have a
-           page fault + mprotect overhead) */
-        page_addr &= qemu_host_page_mask;
-        prot = 0;
-        for (addr = page_addr; addr < page_addr + qemu_host_page_size;
-            addr += TARGET_PAGE_SIZE) {
-
-            p2 = page_find(addr >> TARGET_PAGE_BITS);
-            if (!p2) {
-                continue;
-            }
-            prot |= p2->flags;
-            p2->flags &= ~PAGE_WRITE;
-          }
-        mprotect(g2h(page_addr), qemu_host_page_size,
-                 (prot & PAGE_BITS) & ~PAGE_WRITE);
-#ifdef DEBUG_TB_INVALIDATE
-        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
-               page_addr);
-#endif
-    }
-#else
-    /* if some code is already present, then the pages are already
-       protected. So we handle the case where only the first TB is
-       allocated in a physical page */
-    if (!page_already_protected) {
-        tlb_protect_code(page_addr);
-    }
-#endif
+    return false;

 }
 }

-
-/* add a new TB and link it to the physical page tables. phys_page2 is
- * (-1) to indicate that only one page contains the TB.
- *
- * Called with mmap_lock held for user-mode emulation.
- */
-static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
-                         tb_page_addr_t phys_page2)
-{
-    unsigned int h;
-    TranslationBlock **ptb;
-
-    /* add in the physical hash table */
-    h = tb_phys_hash_func(phys_pc);
-    ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
-    tb->phys_hash_next = *ptb;
-    *ptb = tb;
-
-    /* add in the page list */
-    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
-    if (phys_page2 != -1) {
-        tb_alloc_page(tb, 1, phys_page2);
-    } else {
-        tb->page_addr[1] = -1;
-    }
-
-    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
-    tb->jmp_next[0] = NULL;
-    tb->jmp_next[1] = NULL;
-
-    /* init original jump addresses */
-    if (tb->tb_next_offset[0] != 0xffff) {
-        tb_reset_jump(tb, 0);
-    }
-    if (tb->tb_next_offset[1] != 0xffff) {
-        tb_reset_jump(tb, 1);
-    }
-
-#ifdef DEBUG_TB_CHECK
-    tb_page_check();

 #endif
 #endif

-}

 
 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
    tb[1].tc_ptr. Return NULL if not found */
 
 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
    tb[1].tc_ptr. Return NULL if not found */


@@ -1553,13 +1709,15 @@ void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
         rcu_read_unlock();
         return;
     }
         rcu_read_unlock();
         return;
     }

-    ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
-        + addr;
+    ram_addr = memory_region_get_ram_addr(mr) + addr;
+    tb_lock();

     tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
     tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);

+    tb_unlock();

     rcu_read_unlock();
 }
 #endif /* !defined(CONFIG_USER_ONLY) */
 
     rcu_read_unlock();
 }
 #endif /* !defined(CONFIG_USER_ONLY) */
 

+/* Called with tb_lock held.  */

 void tb_check_watchpoint(CPUState *cpu)
 {
     TranslationBlock *tb;
 void tb_check_watchpoint(CPUState *cpu)
 {
     TranslationBlock *tb;


@@ -1596,6 +1754,7 @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
     target_ulong pc, cs_base;
     uint32_t flags;
 
     target_ulong pc, cs_base;
     uint32_t flags;
 

+    tb_lock();

     tb = tb_find_pc(retaddr);
     if (!tb) {
         cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
     tb = tb_find_pc(retaddr);
     if (!tb) {
         cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",


@@ -1647,12 +1806,17 @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
     /* FIXME: In theory this could raise an exception.  In practice
        we have already translated the block once so it's probably ok.  */
     tb_gen_code(cpu, pc, cs_base, flags, cflags);
     /* FIXME: In theory this could raise an exception.  In practice
        we have already translated the block once so it's probably ok.  */
     tb_gen_code(cpu, pc, cs_base, flags, cflags);

+

     /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
     /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not

-       the first in the TB) then we end up generating a whole new TB and
-       repeating the fault, which is horribly inefficient.
-       Better would be to execute just this insn uncached, or generate a
-       second new TB.  */
-    cpu_resume_from_signal(cpu, NULL);
+     * the first in the TB) then we end up generating a whole new TB and
+     *  repeating the fault, which is horribly inefficient.
+     *  Better would be to execute just this insn uncached, or generate a
+     *  second new TB.
+     *
+     * cpu_loop_exit_noexc will longjmp back to cpu_exec where the
+     * tb_lock gets reset.
+     */
+    cpu_loop_exit_noexc(cpu);

 }
 
 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
 }
 
 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)


@@ -1670,11 +1834,52 @@ void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
 }
 
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
 }
 

+static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf,
+                                 struct qht_stats hst)
+{
+    uint32_t hgram_opts;
+    size_t hgram_bins;
+    char *hgram;
+
+    if (!hst.head_buckets) {
+        return;
+    }
+    cpu_fprintf(f, "TB hash buckets     %zu/%zu (%0.2f%% head buckets used)\n",
+                hst.used_head_buckets, hst.head_buckets,
+                (double)hst.used_head_buckets / hst.head_buckets * 100);
+
+    hgram_opts =  QDIST_PR_BORDER | QDIST_PR_LABELS;
+    hgram_opts |= QDIST_PR_100X   | QDIST_PR_PERCENT;
+    if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) {
+        hgram_opts |= QDIST_PR_NODECIMAL;
+    }
+    hgram = qdist_pr(&hst.occupancy, 10, hgram_opts);
+    cpu_fprintf(f, "TB hash occupancy   %0.2f%% avg chain occ. Histogram: %s\n",
+                qdist_avg(&hst.occupancy) * 100, hgram);
+    g_free(hgram);
+
+    hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
+    hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain);
+    if (hgram_bins > 10) {
+        hgram_bins = 10;
+    } else {
+        hgram_bins = 0;
+        hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE;
+    }
+    hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts);
+    cpu_fprintf(f, "TB hash avg chain   %0.3f buckets. Histogram: %s\n",
+                qdist_avg(&hst.chain), hgram);
+    g_free(hgram);
+}
+

 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
 {
     int i, target_code_size, max_target_code_size;
     int direct_jmp_count, direct_jmp2_count, cross_page;
     TranslationBlock *tb;
 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
 {
     int i, target_code_size, max_target_code_size;
     int direct_jmp_count, direct_jmp2_count, cross_page;
     TranslationBlock *tb;

+    struct qht_stats hst;
+
+    tb_lock();

 
     target_code_size = 0;
     max_target_code_size = 0;
 
     target_code_size = 0;
     max_target_code_size = 0;


@@ -1690,9 +1895,9 @@ void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
         if (tb->page_addr[1] != -1) {
             cross_page++;
         }
         if (tb->page_addr[1] != -1) {
             cross_page++;
         }

-        if (tb->tb_next_offset[0] != 0xffff) {
+        if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {

             direct_jmp_count++;
             direct_jmp_count++;

-            if (tb->tb_next_offset[1] != 0xffff) {
+            if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {

                 direct_jmp2_count++;
             }
         }
                 direct_jmp2_count++;
             }
         }


@@ -1725,12 +1930,20 @@ void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
                 direct_jmp2_count,
                 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
                         tcg_ctx.tb_ctx.nb_tbs : 0);
                 direct_jmp2_count,
                 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
                         tcg_ctx.tb_ctx.nb_tbs : 0);

+
+    qht_statistics_init(&tcg_ctx.tb_ctx.htable, &hst);
+    print_qht_statistics(f, cpu_fprintf, hst);
+    qht_statistics_destroy(&hst);
+

     cpu_fprintf(f, "\nStatistics:\n");
     cpu_fprintf(f, "\nStatistics:\n");

-    cpu_fprintf(f, "TB flush count      %d\n", tcg_ctx.tb_ctx.tb_flush_count);
+    cpu_fprintf(f, "TB flush count      %u\n",
+            atomic_read(&tcg_ctx.tb_ctx.tb_flush_count));

     cpu_fprintf(f, "TB invalidate count %d\n",
             tcg_ctx.tb_ctx.tb_phys_invalidate_count);
     cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
     tcg_dump_info(f, cpu_fprintf);
     cpu_fprintf(f, "TB invalidate count %d\n",
             tcg_ctx.tb_ctx.tb_phys_invalidate_count);
     cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
     tcg_dump_info(f, cpu_fprintf);

+
+    tb_unlock();

 }
 
 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
 }
 
 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)


@@ -1816,16 +2029,16 @@ static int walk_memory_regions_1(struct walk_memory_regions_data *data,
 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
 {
     struct walk_memory_regions_data data;
 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
 {
     struct walk_memory_regions_data data;

-    uintptr_t i;
+    uintptr_t i, l1_sz = v_l1_size;

 
     data.fn = fn;
     data.priv = priv;
     data.start = -1u;
     data.prot = 0;
 
 
     data.fn = fn;
     data.priv = priv;
     data.start = -1u;
     data.prot = 0;
 

-    for (i = 0; i < V_L1_SIZE; i++) {
-        int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
-                                       V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
+    for (i = 0; i < l1_sz; i++) {
+        target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS);
+        int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i);

         if (rc != 0) {
             return rc;
         }
         if (rc != 0) {
             return rc;
         }


@@ -1883,6 +2096,7 @@ void page_set_flags(target_ulong start, target_ulong end, int flags)
     assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
 #endif
     assert(start < end);
     assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
 #endif
     assert(start < end);

+    assert_memory_lock();

 
     start = start & TARGET_PAGE_MASK;
     end = TARGET_PAGE_ALIGN(end);
 
     start = start & TARGET_PAGE_MASK;
     end = TARGET_PAGE_ALIGN(end);


@@ -1901,7 +2115,7 @@ void page_set_flags(target_ulong start, target_ulong end, int flags)
         if (!(p->flags & PAGE_WRITE) &&
             (flags & PAGE_WRITE) &&
             p->first_tb) {
         if (!(p->flags & PAGE_WRITE) &&
             (flags & PAGE_WRITE) &&
             p->first_tb) {

-            tb_invalidate_phys_page(addr, 0, NULL, false);
+            tb_invalidate_phys_page(addr, 0);

         }
         p->flags = flags;
     }
         }
         p->flags = flags;
     }


@@ -1953,7 +2167,7 @@ int page_check_range(target_ulong start, target_ulong len, int flags)
             /* unprotect the page if it was put read-only because it
                contains translated code */
             if (!(p->flags & PAGE_WRITE)) {
             /* unprotect the page if it was put read-only because it
                contains translated code */
             if (!(p->flags & PAGE_WRITE)) {

-                if (!page_unprotect(addr, 0, NULL)) {
+                if (!page_unprotect(addr, 0)) {

                     return -1;
                 }
             }
                     return -1;
                 }
             }


@@ -1963,10 +2177,15 @@ int page_check_range(target_ulong start, target_ulong len, int flags)
 }
 
 /* called from signal handler: invalidate the code and unprotect the
 }
 
 /* called from signal handler: invalidate the code and unprotect the

-   page. Return TRUE if the fault was successfully handled. */
-int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
+ * page. Return 0 if the fault was not handled, 1 if it was handled,
+ * and 2 if it was handled but the caller must cause the TB to be
+ * immediately exited. (We can only return 2 if the 'pc' argument is
+ * non-zero.)
+ */
+int page_unprotect(target_ulong address, uintptr_t pc)

 {
     unsigned int prot;
 {
     unsigned int prot;

+    bool current_tb_invalidated;

     PageDesc *p;
     target_ulong host_start, host_end, addr;
 
     PageDesc *p;
     target_ulong host_start, host_end, addr;
 


@@ -1988,6 +2207,7 @@ int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
         host_end = host_start + qemu_host_page_size;
 
         prot = 0;
         host_end = host_start + qemu_host_page_size;
 
         prot = 0;

+        current_tb_invalidated = false;

         for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
             p = page_find(addr >> TARGET_PAGE_BITS);
             p->flags |= PAGE_WRITE;
         for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
             p = page_find(addr >> TARGET_PAGE_BITS);
             p->flags |= PAGE_WRITE;


@@ -1995,7 +2215,7 @@ int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
 
             /* and since the content will be modified, we must invalidate
                the corresponding translated code. */
 
             /* and since the content will be modified, we must invalidate
                the corresponding translated code. */

-            tb_invalidate_phys_page(addr, pc, puc, true);
+            current_tb_invalidated |= tb_invalidate_phys_page(addr, pc);

 #ifdef DEBUG_TB_CHECK
             tb_invalidate_check(addr);
 #endif
 #ifdef DEBUG_TB_CHECK
             tb_invalidate_check(addr);
 #endif


@@ -2004,7 +2224,8 @@ int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
                  prot & PAGE_BITS);
 
         mmap_unlock();
                  prot & PAGE_BITS);
 
         mmap_unlock();

-        return 1;
+        /* If current TB was invalidated return to main loop */
+        return current_tb_invalidated ? 2 : 1;

     }
     mmap_unlock();
     return 0;
     }
     mmap_unlock();
     return 0;