scsi: Clean up duplicated error in legacy if=scsi code

[qemu.git] / exec.c

diff --git a/exec.c b/exec.c
index c8494051a627e6abfc355429a0045c4aca9c93a7..a6cb4a241c83356e08cc75d19bf0234d3e7b72b1 100644 (file)
--- a/exec.c
+++ b/exec.c
@@ -44,7 +44,7 @@
 #include "trace.h"
 #endif
 #include "exec/cpu-all.h"
 #include "trace.h"
 #endif
 #include "exec/cpu-all.h"

-
+#include "qemu/rcu_queue.h"

 #include "exec/cputlb.h"
 #include "translate-all.h"
 
 #include "exec/cputlb.h"
 #include "translate-all.h"
 


@@ -58,7 +58,10 @@
 #if !defined(CONFIG_USER_ONLY)
 static bool in_migration;
 
 #if !defined(CONFIG_USER_ONLY)
 static bool in_migration;
 

-RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
+/* ram_list is read under rcu_read_lock()/rcu_read_unlock().  Writes
+ * are protected by the ramlist lock.
+ */
+RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };

 
 static MemoryRegion *system_memory;
 static MemoryRegion *system_io;
 
 static MemoryRegion *system_memory;
 static MemoryRegion *system_io;


@@ -75,6 +78,11 @@ static MemoryRegion io_mem_unassigned;
 /* RAM is mmap-ed with MAP_SHARED */
 #define RAM_SHARED     (1 << 1)
 
 /* RAM is mmap-ed with MAP_SHARED */
 #define RAM_SHARED     (1 << 1)
 

+/* Only a portion of RAM (used_length) is actually used, and migrated.
+ * This used_length size can change across reboots.
+ */
+#define RAM_RESIZEABLE (1 << 2)
+

 #endif
 
 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
 #endif
 
 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);


@@ -110,6 +118,8 @@ struct PhysPageEntry {
 typedef PhysPageEntry Node[P_L2_SIZE];
 
 typedef struct PhysPageMap {
 typedef PhysPageEntry Node[P_L2_SIZE];
 
 typedef struct PhysPageMap {

+    struct rcu_head rcu;
+

     unsigned sections_nb;
     unsigned sections_nb_alloc;
     unsigned nodes_nb;
     unsigned sections_nb;
     unsigned sections_nb_alloc;
     unsigned nodes_nb;


@@ -119,6 +129,8 @@ typedef struct PhysPageMap {
 } PhysPageMap;
 
 struct AddressSpaceDispatch {
 } PhysPageMap;
 
 struct AddressSpaceDispatch {

+    struct rcu_head rcu;
+

     /* This is a multi-level map on the physical address space.
      * The bottom level has pointers to MemoryRegionSections.
      */
     /* This is a multi-level map on the physical address space.
      * The bottom level has pointers to MemoryRegionSections.
      */


@@ -310,6 +322,7 @@ bool memory_region_is_unassigned(MemoryRegion *mr)
         && mr != &io_mem_watch;
 }
 
         && mr != &io_mem_watch;
 }
 

+/* Called from RCU critical section */

 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
                                                         hwaddr addr,
                                                         bool resolve_subpage)
 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
                                                         hwaddr addr,
                                                         bool resolve_subpage)


@@ -325,6 +338,7 @@ static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
     return section;
 }
 
     return section;
 }
 

+/* Called from RCU critical section */

 static MemoryRegionSection *
 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
                                  hwaddr *plen, bool resolve_subpage)
 static MemoryRegionSection *
 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
                                  hwaddr *plen, bool resolve_subpage)


@@ -365,15 +379,17 @@ MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
     MemoryRegion *mr;
     hwaddr len = *plen;
 
     MemoryRegion *mr;
     hwaddr len = *plen;
 

+    rcu_read_lock();

     for (;;) {
     for (;;) {

-        section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
+        AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
+        section = address_space_translate_internal(d, addr, &addr, plen, true);

         mr = section->mr;
 
         if (!mr->iommu_ops) {
             break;
         }
 
         mr = section->mr;
 
         if (!mr->iommu_ops) {
             break;
         }
 

-        iotlb = mr->iommu_ops->translate(mr, addr);
+        iotlb = mr->iommu_ops->translate(mr, addr, is_write);

         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
                 | (addr & iotlb.addr_mask));
         len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
                 | (addr & iotlb.addr_mask));
         len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);


@@ -392,15 +408,18 @@ MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
 
     *plen = len;
     *xlat = addr;
 
     *plen = len;
     *xlat = addr;

+    rcu_read_unlock();

     return mr;
 }
 
     return mr;
 }
 

+/* Called from RCU critical section */

 MemoryRegionSection *
 MemoryRegionSection *

-address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
-                                  hwaddr *plen)
+address_space_translate_for_iotlb(CPUState *cpu, hwaddr addr,
+                                  hwaddr *xlat, hwaddr *plen)

 {
     MemoryRegionSection *section;
 {
     MemoryRegionSection *section;

-    section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
+    section = address_space_translate_internal(cpu->memory_dispatch,
+                                               addr, xlat, plen, false);

 
     assert(!section->mr->iommu_ops);
     return section;
 
     assert(!section->mr->iommu_ops);
     return section;


@@ -430,15 +449,50 @@ static int cpu_common_post_load(void *opaque, int version_id)
     return 0;
 }
 
     return 0;
 }
 

+static int cpu_common_pre_load(void *opaque)
+{
+    CPUState *cpu = opaque;
+
+    cpu->exception_index = -1;
+
+    return 0;
+}
+
+static bool cpu_common_exception_index_needed(void *opaque)
+{
+    CPUState *cpu = opaque;
+
+    return tcg_enabled() && cpu->exception_index != -1;
+}
+
+static const VMStateDescription vmstate_cpu_common_exception_index = {
+    .name = "cpu_common/exception_index",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(exception_index, CPUState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+

 const VMStateDescription vmstate_cpu_common = {
     .name = "cpu_common",
     .version_id = 1,
     .minimum_version_id = 1,
 const VMStateDescription vmstate_cpu_common = {
     .name = "cpu_common",
     .version_id = 1,
     .minimum_version_id = 1,

+    .pre_load = cpu_common_pre_load,

     .post_load = cpu_common_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(halted, CPUState),
         VMSTATE_UINT32(interrupt_request, CPUState),
         VMSTATE_END_OF_LIST()
     .post_load = cpu_common_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(halted, CPUState),
         VMSTATE_UINT32(interrupt_request, CPUState),
         VMSTATE_END_OF_LIST()

+    },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &vmstate_cpu_common_exception_index,
+            .needed = cpu_common_exception_index_needed,
+        } , {
+            /* empty */
+        }

     }
 };
 
     }
 };
 


@@ -494,6 +548,7 @@ void cpu_exec_init(CPUArchState *env)
 #ifndef CONFIG_USER_ONLY
     cpu->as = &address_space_memory;
     cpu->thread_id = qemu_get_thread_id();
 #ifndef CONFIG_USER_ONLY
     cpu->as = &address_space_memory;
     cpu->thread_id = qemu_get_thread_id();

+    cpu_reload_memory_map(cpu);

 #endif
     QTAILQ_INSERT_TAIL(&cpus, cpu, node);
 #if defined(CONFIG_USER_ONLY)
 #endif
     QTAILQ_INSERT_TAIL(&cpus, cpu, node);
 #if defined(CONFIG_USER_ONLY)


@@ -513,7 +568,6 @@ void cpu_exec_init(CPUArchState *env)
     }
 }
 
     }
 }
 

-#if defined(TARGET_HAS_ICE)

 #if defined(CONFIG_USER_ONLY)
 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
 {
 #if defined(CONFIG_USER_ONLY)
 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
 {


@@ -529,7 +583,6 @@ static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
     }
 }
 #endif
     }
 }
 #endif

-#endif /* TARGET_HAS_ICE */

 
 #if defined(CONFIG_USER_ONLY)
 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
 
 #if defined(CONFIG_USER_ONLY)
 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)


@@ -537,6 +590,16 @@ void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
 {
 }
 
 {
 }
 

+int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
+                          int flags)
+{
+    return -ENOSYS;
+}
+
+void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint)
+{
+}
+

 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                           int flags, CPUWatchpoint **watchpoint)
 {
 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                           int flags, CPUWatchpoint **watchpoint)
 {


@@ -547,12 +610,10 @@ int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                           int flags, CPUWatchpoint **watchpoint)
 {
 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                           int flags, CPUWatchpoint **watchpoint)
 {

-    vaddr len_mask = ~(len - 1);

     CPUWatchpoint *wp;
 
     CPUWatchpoint *wp;
 

-    /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
-    if ((len & (len - 1)) || (addr & ~len_mask) ||
-            len == 0 || len > TARGET_PAGE_SIZE) {
+    /* forbid ranges which are empty or run off the end of the address space */
+    if (len == 0 || (addr + len - 1) < addr) {

         error_report("tried to set invalid watchpoint at %"
                      VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
         return -EINVAL;
         error_report("tried to set invalid watchpoint at %"
                      VADDR_PRIx ", len=%" VADDR_PRIu, addr, len);
         return -EINVAL;


@@ -560,7 +621,7 @@ int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
     wp = g_malloc(sizeof(*wp));
 
     wp->vaddr = addr;
     wp = g_malloc(sizeof(*wp));
 
     wp->vaddr = addr;

-    wp->len_mask = len_mask;
+    wp->len = len;

     wp->flags = flags;
 
     /* keep all GDB-injected watchpoints in front */
     wp->flags = flags;
 
     /* keep all GDB-injected watchpoints in front */


@@ -581,11 +642,10 @@ int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
                           int flags)
 {
 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr, vaddr len,
                           int flags)
 {

-    vaddr len_mask = ~(len - 1);

     CPUWatchpoint *wp;
 
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
     CPUWatchpoint *wp;
 
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {

-        if (addr == wp->vaddr && len_mask == wp->len_mask
+        if (addr == wp->vaddr && len == wp->len

                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
             cpu_watchpoint_remove_by_ref(cpu, wp);
             return 0;
                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
             cpu_watchpoint_remove_by_ref(cpu, wp);
             return 0;


@@ -615,13 +675,33 @@ void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
         }
     }
 }
         }
     }
 }

+
+/* Return true if this watchpoint address matches the specified
+ * access (ie the address range covered by the watchpoint overlaps
+ * partially or completely with the address range covered by the
+ * access).
+ */
+static inline bool cpu_watchpoint_address_matches(CPUWatchpoint *wp,
+                                                  vaddr addr,
+                                                  vaddr len)
+{
+    /* We know the lengths are non-zero, but a little caution is
+     * required to avoid errors in the case where the range ends
+     * exactly at the top of the address space and so addr + len
+     * wraps round to zero.
+     */
+    vaddr wpend = wp->vaddr + wp->len - 1;
+    vaddr addrend = addr + len - 1;
+
+    return !(addr > wpend || wp->vaddr > addrend);
+}
+

 #endif
 
 /* Add a breakpoint.  */
 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
                           CPUBreakpoint **breakpoint)
 {
 #endif
 
 /* Add a breakpoint.  */
 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
                           CPUBreakpoint **breakpoint)
 {

-#if defined(TARGET_HAS_ICE)

     CPUBreakpoint *bp;
 
     bp = g_malloc(sizeof(*bp));
     CPUBreakpoint *bp;
 
     bp = g_malloc(sizeof(*bp));


@@ -642,15 +722,11 @@ int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
         *breakpoint = bp;
     }
     return 0;
         *breakpoint = bp;
     }
     return 0;

-#else
-    return -ENOSYS;
-#endif

 }
 
 /* Remove a specific breakpoint.  */
 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
 {
 }
 
 /* Remove a specific breakpoint.  */
 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
 {

-#if defined(TARGET_HAS_ICE)

     CPUBreakpoint *bp;
 
     QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
     CPUBreakpoint *bp;
 
     QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {


@@ -660,27 +736,21 @@ int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
         }
     }
     return -ENOENT;
         }
     }
     return -ENOENT;

-#else
-    return -ENOSYS;
-#endif

 }
 
 /* Remove a specific breakpoint by reference.  */
 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
 {
 }
 
 /* Remove a specific breakpoint by reference.  */
 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
 {

-#if defined(TARGET_HAS_ICE)

     QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
 
     breakpoint_invalidate(cpu, breakpoint->pc);
 
     g_free(breakpoint);
     QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
 
     breakpoint_invalidate(cpu, breakpoint->pc);
 
     g_free(breakpoint);

-#endif

 }
 
 /* Remove all matching breakpoints. */
 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
 {
 }
 
 /* Remove all matching breakpoints. */
 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
 {

-#if defined(TARGET_HAS_ICE)

     CPUBreakpoint *bp, *next;
 
     QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
     CPUBreakpoint *bp, *next;
 
     QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {


@@ -688,14 +758,12 @@ void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
             cpu_breakpoint_remove_by_ref(cpu, bp);
         }
     }
             cpu_breakpoint_remove_by_ref(cpu, bp);
         }
     }

-#endif

 }
 
 /* enable or disable single step mode. EXCP_DEBUG is returned by the
    CPU loop after each instruction */
 void cpu_single_step(CPUState *cpu, int enabled)
 {
 }
 
 /* enable or disable single step mode. EXCP_DEBUG is returned by the
    CPU loop after each instruction */
 void cpu_single_step(CPUState *cpu, int enabled)
 {

-#if defined(TARGET_HAS_ICE)

     if (cpu->singlestep_enabled != enabled) {
         cpu->singlestep_enabled = enabled;
         if (kvm_enabled()) {
     if (cpu->singlestep_enabled != enabled) {
         cpu->singlestep_enabled = enabled;
         if (kvm_enabled()) {


@@ -707,7 +775,6 @@ void cpu_single_step(CPUState *cpu, int enabled)
             tb_flush(env);
         }
     }
             tb_flush(env);
         }
     }

-#endif

 }
 
 void cpu_abort(CPUState *cpu, const char *fmt, ...)
 }
 
 void cpu_abort(CPUState *cpu, const char *fmt, ...)


@@ -743,17 +810,17 @@ void cpu_abort(CPUState *cpu, const char *fmt, ...)
 }
 
 #if !defined(CONFIG_USER_ONLY)
 }
 
 #if !defined(CONFIG_USER_ONLY)

+/* Called from RCU critical section */

 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
 {
     RAMBlock *block;
 
 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
 {
     RAMBlock *block;
 

-    /* The list is protected by the iothread lock here.  */
-    block = ram_list.mru_block;
-    if (block && addr - block->offset < block->length) {
+    block = atomic_rcu_read(&ram_list.mru_block);
+    if (block && addr - block->offset < block->max_length) {

         goto found;
     }
         goto found;
     }

-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (addr - block->offset < block->length) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (addr - block->offset < block->max_length) {

             goto found;
         }
     }
             goto found;
         }
     }


@@ -762,6 +829,22 @@ static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
     abort();
 
 found:
     abort();
 
 found:

+    /* It is safe to write mru_block outside the iothread lock.  This
+     * is what happens:
+     *
+     *     mru_block = xxx
+     *     rcu_read_unlock()
+     *                                        xxx removed from list
+     *                  rcu_read_lock()
+     *                  read mru_block
+     *                                        mru_block = NULL;
+     *                                        call_rcu(reclaim_ramblock, xxx);
+     *                  rcu_read_unlock()
+     *
+     * atomic_rcu_set is not needed here.  The block was already published
+     * when it was placed into the list.  Here we're just making an extra
+     * copy of the pointer.
+     */

     ram_list.mru_block = block;
     return block;
 }
     ram_list.mru_block = block;
     return block;
 }


@@ -775,10 +858,12 @@ static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
     end = TARGET_PAGE_ALIGN(start + length);
     start &= TARGET_PAGE_MASK;
 
     end = TARGET_PAGE_ALIGN(start + length);
     start &= TARGET_PAGE_MASK;
 

+    rcu_read_lock();

     block = qemu_get_ram_block(start);
     assert(block == qemu_get_ram_block(end - 1));
     block = qemu_get_ram_block(start);
     assert(block == qemu_get_ram_block(end - 1));

-    start1 = (uintptr_t)block->host + (start - block->offset);
+    start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);

     cpu_tlb_reset_dirty_all(start1, length);
     cpu_tlb_reset_dirty_all(start1, length);

+    rcu_read_unlock();

 }
 
 /* Note: start and end must be within the same ram block.  */
 }
 
 /* Note: start and end must be within the same ram block.  */


@@ -787,7 +872,7 @@ void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
 {
     if (length == 0)
         return;
 {
     if (length == 0)
         return;

-    cpu_physical_memory_clear_dirty_range(start, length, client);
+    cpu_physical_memory_clear_dirty_range_type(start, length, client);

 
     if (tcg_enabled()) {
         tlb_reset_dirty_range_all(start, length);
 
     if (tcg_enabled()) {
         tlb_reset_dirty_range_all(start, length);


@@ -799,6 +884,7 @@ static void cpu_physical_memory_set_dirty_tracking(bool enable)
     in_migration = enable;
 }
 
     in_migration = enable;
 }
 

+/* Called from RCU critical section */

 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
                                        MemoryRegionSection *section,
                                        target_ulong vaddr,
 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
                                        MemoryRegionSection *section,
                                        target_ulong vaddr,


@@ -826,7 +912,7 @@ hwaddr memory_region_section_get_iotlb(CPUState *cpu,
     /* Make accesses to pages with watchpoints go via the
        watchpoint trap routines.  */
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
     /* Make accesses to pages with watchpoints go via the
        watchpoint trap routines.  */
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {

-        if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
+        if (cpu_watchpoint_address_matches(wp, vaddr, TARGET_PAGE_SIZE)) {

             /* Avoid trapping reads of pages with a write breakpoint. */
             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
                 iotlb = PHYS_SECTION_WATCH + paddr;
             /* Avoid trapping reads of pages with a write breakpoint. */
             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
                 iotlb = PHYS_SECTION_WATCH + paddr;


@@ -846,14 +932,15 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                              uint16_t section);
 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
 
                              uint16_t section);
 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
 

-static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
+static void *(*phys_mem_alloc)(size_t size, uint64_t *align) =
+                               qemu_anon_ram_alloc;

 
 /*
  * Set a custom physical guest memory alloator.
  * Accelerators with unusual needs may need this.  Hopefully, we can
  * get rid of it eventually.
  */
 
 /*
  * Set a custom physical guest memory alloator.
  * Accelerators with unusual needs may need this.  Hopefully, we can
  * get rid of it eventually.
  */

-void phys_mem_set_alloc(void *(*alloc)(size_t))
+void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align))

 {
     phys_mem_alloc = alloc;
 }
 {
     phys_mem_alloc = alloc;
 }


@@ -883,7 +970,7 @@ static void phys_section_destroy(MemoryRegion *mr)
 
     if (mr->subpage) {
         subpage_t *subpage = container_of(mr, subpage_t, iomem);
 
     if (mr->subpage) {
         subpage_t *subpage = container_of(mr, subpage_t, iomem);

-        memory_region_destroy(&subpage->iomem);
+        object_unref(OBJECT(&subpage->iomem));

         g_free(subpage);
     }
 }
         g_free(subpage);
     }
 }


@@ -996,7 +1083,7 @@ void qemu_mutex_unlock_ramlist(void)
 
 #define HUGETLBFS_MAGIC       0x958458f6
 
 
 #define HUGETLBFS_MAGIC       0x958458f6
 

-static long gethugepagesize(const char *path)
+static long gethugepagesize(const char *path, Error **errp)

 {
     struct statfs fs;
     int ret;
 {
     struct statfs fs;
     int ret;


@@ -1006,7 +1093,8 @@ static long gethugepagesize(const char *path)
     } while (ret != 0 && errno == EINTR);
 
     if (ret != 0) {
     } while (ret != 0 && errno == EINTR);
 
     if (ret != 0) {

-        perror(path);
+        error_setg_errno(errp, errno, "failed to get page size of file %s",
+                         path);

         return 0;
     }
 
         return 0;
     }
 


@@ -1024,17 +1112,23 @@ static void *file_ram_alloc(RAMBlock *block,
     char *filename;
     char *sanitized_name;
     char *c;
     char *filename;
     char *sanitized_name;
     char *c;

-    void *area;
+    void *area = NULL;

     int fd;
     int fd;

-    unsigned long hpagesize;
+    uint64_t hpagesize;
+    Error *local_err = NULL;

 
 

-    hpagesize = gethugepagesize(path);
-    if (!hpagesize) {
+    hpagesize = gethugepagesize(path, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);

         goto error;
     }
         goto error;
     }

+    block->mr->align = hpagesize;

 
     if (memory < hpagesize) {
 
     if (memory < hpagesize) {

-        return NULL;
+        error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
+                   "or larger than huge page size 0x%" PRIx64,
+                   memory, hpagesize);
+        goto error;

     }
 
     if (kvm_enabled() && !kvm_has_sync_mmu()) {
     }
 
     if (kvm_enabled() && !kvm_has_sync_mmu()) {


@@ -1044,7 +1138,7 @@ static void *file_ram_alloc(RAMBlock *block,
     }
 
     /* Make name safe to use with mkstemp by replacing '/' with '_'. */
     }
 
     /* Make name safe to use with mkstemp by replacing '/' with '_'. */

-    sanitized_name = g_strdup(block->mr->name);
+    sanitized_name = g_strdup(memory_region_name(block->mr));

     for (c = sanitized_name; *c != '\0'; c++) {
         if (*c == '/')
             *c = '_';
     for (c = sanitized_name; *c != '\0'; c++) {
         if (*c == '/')
             *c = '_';


@@ -1095,12 +1189,14 @@ static void *file_ram_alloc(RAMBlock *block,
 
 error:
     if (mem_prealloc) {
 
 error:
     if (mem_prealloc) {

+        error_report("%s\n", error_get_pretty(*errp));

         exit(1);
     }
     return NULL;
 }
 #endif
 
         exit(1);
     }
     return NULL;
 }
 #endif
 

+/* Called with the ramlist lock held.  */

 static ram_addr_t find_ram_offset(ram_addr_t size)
 {
     RAMBlock *block, *next_block;
 static ram_addr_t find_ram_offset(ram_addr_t size)
 {
     RAMBlock *block, *next_block;


@@ -1108,15 +1204,16 @@ static ram_addr_t find_ram_offset(ram_addr_t size)
 
     assert(size != 0); /* it would hand out same offset multiple times */
 
 
     assert(size != 0); /* it would hand out same offset multiple times */
 

-    if (QTAILQ_EMPTY(&ram_list.blocks))
+    if (QLIST_EMPTY_RCU(&ram_list.blocks)) {

         return 0;
         return 0;

+    }

 
 

-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {

         ram_addr_t end, next = RAM_ADDR_MAX;
 
         ram_addr_t end, next = RAM_ADDR_MAX;
 

-        end = block->offset + block->length;
+        end = block->offset + block->max_length;

 
 

-        QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
+        QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {

             if (next_block->offset >= end) {
                 next = MIN(next, next_block->offset);
             }
             if (next_block->offset >= end) {
                 next = MIN(next, next_block->offset);
             }


@@ -1141,9 +1238,11 @@ ram_addr_t last_ram_offset(void)
     RAMBlock *block;
     ram_addr_t last = 0;
 
     RAMBlock *block;
     ram_addr_t last = 0;
 

-    QTAILQ_FOREACH(block, &ram_list.blocks, next)
-        last = MAX(last, block->offset + block->length);
-
+    rcu_read_lock();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        last = MAX(last, block->offset + block->max_length);
+    }
+    rcu_read_unlock();

     return last;
 }
 
     return last;
 }
 


@@ -1163,11 +1262,14 @@ static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
     }
 }
 
     }
 }
 

+/* Called within an RCU critical section, or while the ramlist lock
+ * is held.
+ */

 static RAMBlock *find_ram_block(ram_addr_t addr)
 {
     RAMBlock *block;
 
 static RAMBlock *find_ram_block(ram_addr_t addr)
 {
     RAMBlock *block;
 

-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {

         if (block->offset == addr) {
             return block;
         }
         if (block->offset == addr) {
             return block;
         }


@@ -1176,11 +1278,13 @@ static RAMBlock *find_ram_block(ram_addr_t addr)
     return NULL;
 }
 
     return NULL;
 }
 

+/* Called with iothread lock held.  */

 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
 {
 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
 {

-    RAMBlock *new_block = find_ram_block(addr);
-    RAMBlock *block;
+    RAMBlock *new_block, *block;

 
 

+    rcu_read_lock();
+    new_block = find_ram_block(addr);

     assert(new_block);
     assert(!new_block->idstr[0]);
 
     assert(new_block);
     assert(!new_block->idstr[0]);
 


@@ -1193,25 +1297,32 @@ void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
     }
     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
 
     }
     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
 

-    /* This assumes the iothread lock is taken here too.  */
-    qemu_mutex_lock_ramlist();
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {

         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
                     new_block->idstr);
             abort();
         }
     }
         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
                     new_block->idstr);
             abort();
         }
     }

-    qemu_mutex_unlock_ramlist();
+    rcu_read_unlock();

 }
 
 }
 

+/* Called with iothread lock held.  */

 void qemu_ram_unset_idstr(ram_addr_t addr)
 {
 void qemu_ram_unset_idstr(ram_addr_t addr)
 {

-    RAMBlock *block = find_ram_block(addr);
+    RAMBlock *block;

 
 

+    /* FIXME: arch_init.c assumes that this is not called throughout
+     * migration.  Ignore the problem since hot-unplug during migration
+     * does not work anyway.
+     */
+
+    rcu_read_lock();
+    block = find_ram_block(addr);

     if (block) {
         memset(block->idstr, 0, sizeof(block->idstr));
     }
     if (block) {
         memset(block->idstr, 0, sizeof(block->idstr));
     }

+    rcu_read_unlock();

 }
 
 static int memory_try_enable_merging(void *addr, size_t len)
 }
 
 static int memory_try_enable_merging(void *addr, size_t len)


@@ -1224,44 +1335,101 @@ static int memory_try_enable_merging(void *addr, size_t len)
     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
 }
 
     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
 }
 

-static ram_addr_t ram_block_add(RAMBlock *new_block)
+/* Only legal before guest might have detected the memory size: e.g. on
+ * incoming migration, or right after reset.
+ *
+ * As memory core doesn't know how is memory accessed, it is up to
+ * resize callback to update device state and/or add assertions to detect
+ * misuse, if necessary.
+ */
+int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp)
+{
+    RAMBlock *block = find_ram_block(base);
+
+    assert(block);
+
+    newsize = TARGET_PAGE_ALIGN(newsize);
+
+    if (block->used_length == newsize) {
+        return 0;
+    }
+
+    if (!(block->flags & RAM_RESIZEABLE)) {
+        error_setg_errno(errp, EINVAL,
+                         "Length mismatch: %s: 0x" RAM_ADDR_FMT
+                         " in != 0x" RAM_ADDR_FMT, block->idstr,
+                         newsize, block->used_length);
+        return -EINVAL;
+    }
+
+    if (block->max_length < newsize) {
+        error_setg_errno(errp, EINVAL,
+                         "Length too large: %s: 0x" RAM_ADDR_FMT
+                         " > 0x" RAM_ADDR_FMT, block->idstr,
+                         newsize, block->max_length);
+        return -EINVAL;
+    }
+
+    cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
+    block->used_length = newsize;
+    cpu_physical_memory_set_dirty_range(block->offset, block->used_length);
+    memory_region_set_size(block->mr, newsize);
+    if (block->resized) {
+        block->resized(block->idstr, newsize, block->host);
+    }
+    return 0;
+}
+
+static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)

 {
     RAMBlock *block;
 {
     RAMBlock *block;

+    RAMBlock *last_block = NULL;

     ram_addr_t old_ram_size, new_ram_size;
 
     old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
 
     ram_addr_t old_ram_size, new_ram_size;
 
     old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
 

-    /* This assumes the iothread lock is taken here too.  */

     qemu_mutex_lock_ramlist();
     qemu_mutex_lock_ramlist();

-    new_block->offset = find_ram_offset(new_block->length);
+    new_block->offset = find_ram_offset(new_block->max_length);

 
     if (!new_block->host) {
         if (xen_enabled()) {
 
     if (!new_block->host) {
         if (xen_enabled()) {

-            xen_ram_alloc(new_block->offset, new_block->length, new_block->mr);
+            xen_ram_alloc(new_block->offset, new_block->max_length,
+                          new_block->mr);

         } else {
         } else {

-            new_block->host = phys_mem_alloc(new_block->length);
+            new_block->host = phys_mem_alloc(new_block->max_length,
+                                             &new_block->mr->align);

             if (!new_block->host) {
             if (!new_block->host) {

-                fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
-                        new_block->mr->name, strerror(errno));
-                exit(1);
+                error_setg_errno(errp, errno,
+                                 "cannot set up guest memory '%s'",
+                                 memory_region_name(new_block->mr));
+                qemu_mutex_unlock_ramlist();
+                return -1;

             }
             }

-            memory_try_enable_merging(new_block->host, new_block->length);
+            memory_try_enable_merging(new_block->host, new_block->max_length);

         }
     }
 
         }
     }
 

-    /* Keep the list sorted from biggest to smallest block.  */
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (block->length < new_block->length) {
+    /* Keep the list sorted from biggest to smallest block.  Unlike QTAILQ,
+     * QLIST (which has an RCU-friendly variant) does not have insertion at
+     * tail, so save the last element in last_block.
+     */
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        last_block = block;
+        if (block->max_length < new_block->max_length) {

             break;
         }
     }
     if (block) {
             break;
         }
     }
     if (block) {

-        QTAILQ_INSERT_BEFORE(block, new_block, next);
-    } else {
-        QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
+        QLIST_INSERT_BEFORE_RCU(block, new_block, next);
+    } else if (last_block) {
+        QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
+    } else { /* list is empty */
+        QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);

     }
     ram_list.mru_block = NULL;
 
     }
     ram_list.mru_block = NULL;
 

+    /* Write list before version */
+    smp_wmb();

     ram_list.version++;
     qemu_mutex_unlock_ramlist();
 
     ram_list.version++;
     qemu_mutex_unlock_ramlist();
 


@@ -1269,20 +1437,24 @@ static ram_addr_t ram_block_add(RAMBlock *new_block)
 
     if (new_ram_size > old_ram_size) {
         int i;
 
     if (new_ram_size > old_ram_size) {
         int i;

+
+        /* ram_list.dirty_memory[] is protected by the iothread lock.  */

         for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
             ram_list.dirty_memory[i] =
                 bitmap_zero_extend(ram_list.dirty_memory[i],
                                    old_ram_size, new_ram_size);
        }
     }
         for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
             ram_list.dirty_memory[i] =
                 bitmap_zero_extend(ram_list.dirty_memory[i],
                                    old_ram_size, new_ram_size);
        }
     }

-    cpu_physical_memory_set_dirty_range(new_block->offset, new_block->length);
-
-    qemu_ram_setup_dump(new_block->host, new_block->length);
-    qemu_madvise(new_block->host, new_block->length, QEMU_MADV_HUGEPAGE);
-    qemu_madvise(new_block->host, new_block->length, QEMU_MADV_DONTFORK);
+    cpu_physical_memory_set_dirty_range(new_block->offset,
+                                        new_block->used_length);

 
 

-    if (kvm_enabled()) {
-        kvm_setup_guest_memory(new_block->host, new_block->length);
+    if (new_block->host) {
+        qemu_ram_setup_dump(new_block->host, new_block->max_length);
+        qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
+        qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
+        if (kvm_enabled()) {
+            kvm_setup_guest_memory(new_block->host, new_block->max_length);
+        }

     }
 
     return new_block->offset;
     }
 
     return new_block->offset;


@@ -1294,6 +1466,8 @@ ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
                                     Error **errp)
 {
     RAMBlock *new_block;
                                     Error **errp)
 {
     RAMBlock *new_block;

+    ram_addr_t addr;
+    Error *local_err = NULL;

 
     if (xen_enabled()) {
         error_setg(errp, "-mem-path not supported with Xen");
 
     if (xen_enabled()) {
         error_setg(errp, "-mem-path not supported with Xen");


@@ -1314,7 +1488,8 @@ ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
     size = TARGET_PAGE_ALIGN(size);
     new_block = g_malloc0(sizeof(*new_block));
     new_block->mr = mr;
     size = TARGET_PAGE_ALIGN(size);
     new_block = g_malloc0(sizeof(*new_block));
     new_block->mr = mr;

-    new_block->length = size;
+    new_block->used_length = size;
+    new_block->max_length = size;

     new_block->flags = share ? RAM_SHARED : 0;
     new_block->host = file_ram_alloc(new_block, size,
                                      mem_path, errp);
     new_block->flags = share ? RAM_SHARED : 0;
     new_block->host = file_ram_alloc(new_block, size,
                                      mem_path, errp);


@@ -1323,79 +1498,126 @@ ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
         return -1;
     }
 
         return -1;
     }
 

-    return ram_block_add(new_block);
+    addr = ram_block_add(new_block, &local_err);
+    if (local_err) {
+        g_free(new_block);
+        error_propagate(errp, local_err);
+        return -1;
+    }
+    return addr;

 }
 #endif
 
 }
 #endif
 

-ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
-                                   MemoryRegion *mr)
+static
+ram_addr_t qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
+                                   void (*resized)(const char*,
+                                                   uint64_t length,
+                                                   void *host),
+                                   void *host, bool resizeable,
+                                   MemoryRegion *mr, Error **errp)

 {
     RAMBlock *new_block;
 {
     RAMBlock *new_block;

+    ram_addr_t addr;
+    Error *local_err = NULL;

 
     size = TARGET_PAGE_ALIGN(size);
 
     size = TARGET_PAGE_ALIGN(size);

+    max_size = TARGET_PAGE_ALIGN(max_size);

     new_block = g_malloc0(sizeof(*new_block));
     new_block->mr = mr;
     new_block = g_malloc0(sizeof(*new_block));
     new_block->mr = mr;

-    new_block->length = size;
+    new_block->resized = resized;
+    new_block->used_length = size;
+    new_block->max_length = max_size;
+    assert(max_size >= size);

     new_block->fd = -1;
     new_block->host = host;
     if (host) {
         new_block->flags |= RAM_PREALLOC;
     }
     new_block->fd = -1;
     new_block->host = host;
     if (host) {
         new_block->flags |= RAM_PREALLOC;
     }

-    return ram_block_add(new_block);
+    if (resizeable) {
+        new_block->flags |= RAM_RESIZEABLE;
+    }
+    addr = ram_block_add(new_block, &local_err);
+    if (local_err) {
+        g_free(new_block);
+        error_propagate(errp, local_err);
+        return -1;
+    }
+    return addr;

 }
 
 }
 

-ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
+ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
+                                   MemoryRegion *mr, Error **errp)

 {
 {

-    return qemu_ram_alloc_from_ptr(size, NULL, mr);
+    return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
+}
+
+ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
+{
+    return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
+}
+
+ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
+                                     void (*resized)(const char*,
+                                                     uint64_t length,
+                                                     void *host),
+                                     MemoryRegion *mr, Error **errp)
+{
+    return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);

 }
 
 void qemu_ram_free_from_ptr(ram_addr_t addr)
 {
     RAMBlock *block;
 
 }
 
 void qemu_ram_free_from_ptr(ram_addr_t addr)
 {
     RAMBlock *block;
 

-    /* This assumes the iothread lock is taken here too.  */

     qemu_mutex_lock_ramlist();
     qemu_mutex_lock_ramlist();

-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {

         if (addr == block->offset) {
         if (addr == block->offset) {

-            QTAILQ_REMOVE(&ram_list.blocks, block, next);
+            QLIST_REMOVE_RCU(block, next);

             ram_list.mru_block = NULL;
             ram_list.mru_block = NULL;

+            /* Write list before version */
+            smp_wmb();

             ram_list.version++;
             ram_list.version++;

-            g_free(block);
+            g_free_rcu(block, rcu);

             break;
         }
     }
     qemu_mutex_unlock_ramlist();
 }
 
             break;
         }
     }
     qemu_mutex_unlock_ramlist();
 }
 

+static void reclaim_ramblock(RAMBlock *block)
+{
+    if (block->flags & RAM_PREALLOC) {
+        ;
+    } else if (xen_enabled()) {
+        xen_invalidate_map_cache_entry(block->host);
+#ifndef _WIN32
+    } else if (block->fd >= 0) {
+        munmap(block->host, block->max_length);
+        close(block->fd);
+#endif
+    } else {
+        qemu_anon_ram_free(block->host, block->max_length);
+    }
+    g_free(block);
+}
+

 void qemu_ram_free(ram_addr_t addr)
 {
     RAMBlock *block;
 
 void qemu_ram_free(ram_addr_t addr)
 {
     RAMBlock *block;
 

-    /* This assumes the iothread lock is taken here too.  */

     qemu_mutex_lock_ramlist();
     qemu_mutex_lock_ramlist();

-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {

         if (addr == block->offset) {
         if (addr == block->offset) {

-            QTAILQ_REMOVE(&ram_list.blocks, block, next);
+            QLIST_REMOVE_RCU(block, next);

             ram_list.mru_block = NULL;
             ram_list.mru_block = NULL;

+            /* Write list before version */
+            smp_wmb();

             ram_list.version++;
             ram_list.version++;

-            if (block->flags & RAM_PREALLOC) {
-                ;
-            } else if (xen_enabled()) {
-                xen_invalidate_map_cache_entry(block->host);
-#ifndef _WIN32
-            } else if (block->fd >= 0) {
-                munmap(block->host, block->length);
-                close(block->fd);
-#endif
-            } else {
-                qemu_anon_ram_free(block->host, block->length);
-            }
-            g_free(block);
+            call_rcu(block, reclaim_ramblock, rcu);

             break;
         }
     }
     qemu_mutex_unlock_ramlist();
             break;
         }
     }
     qemu_mutex_unlock_ramlist();

-

 }
 
 #ifndef _WIN32
 }
 
 #ifndef _WIN32


@@ -1406,10 +1628,10 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
     int flags;
     void *area, *vaddr;
 
     int flags;
     void *area, *vaddr;
 

-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {

         offset = addr - block->offset;
         offset = addr - block->offset;

-        if (offset < block->length) {
-            vaddr = block->host + offset;
+        if (offset < block->max_length) {
+            vaddr = ramblock_ptr(block, offset);

             if (block->flags & RAM_PREALLOC) {
                 ;
             } else if (xen_enabled()) {
             if (block->flags & RAM_PREALLOC) {
                 ;
             } else if (xen_enabled()) {


@@ -1443,7 +1665,6 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
                 memory_try_enable_merging(vaddr, length);
                 qemu_ram_setup_dump(vaddr, length);
             }
                 memory_try_enable_merging(vaddr, length);
                 qemu_ram_setup_dump(vaddr, length);
             }

-            return;

         }
     }
 }
         }
     }
 }


@@ -1451,42 +1672,78 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
 
 int qemu_get_ram_fd(ram_addr_t addr)
 {
 
 int qemu_get_ram_fd(ram_addr_t addr)
 {

-    RAMBlock *block = qemu_get_ram_block(addr);
+    RAMBlock *block;
+    int fd;

 
 

-    return block->fd;
+    rcu_read_lock();
+    block = qemu_get_ram_block(addr);
+    fd = block->fd;
+    rcu_read_unlock();
+    return fd;

 }
 
 }
 

-/* Return a host pointer to ram allocated with qemu_ram_alloc.
-   With the exception of the softmmu code in this file, this should
-   only be used for local memory (e.g. video ram) that the device owns,
-   and knows it isn't going to access beyond the end of the block.
+void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
+{
+    RAMBlock *block;
+    void *ptr;
+
+    rcu_read_lock();
+    block = qemu_get_ram_block(addr);
+    ptr = ramblock_ptr(block, 0);
+    rcu_read_unlock();
+    return ptr;
+}

 
 

-   It should not be used for general purpose DMA.
-   Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
+/* Return a host pointer to ram allocated with qemu_ram_alloc.
+ * This should not be used for general purpose DMA.  Use address_space_map
+ * or address_space_rw instead. For local memory (e.g. video ram) that the
+ * device owns, use memory_region_get_ram_ptr.
+ *
+ * By the time this function returns, the returned pointer is not protected
+ * by RCU anymore.  If the caller is not within an RCU critical section and
+ * does not hold the iothread lock, it must have other means of protecting the
+ * pointer, such as a reference to the region that includes the incoming
+ * ram_addr_t.

  */
 void *qemu_get_ram_ptr(ram_addr_t addr)
 {
  */
 void *qemu_get_ram_ptr(ram_addr_t addr)
 {

-    RAMBlock *block = qemu_get_ram_block(addr);
+    RAMBlock *block;
+    void *ptr;

 
 

-    if (xen_enabled()) {
+    rcu_read_lock();
+    block = qemu_get_ram_block(addr);
+
+    if (xen_enabled() && block->host == NULL) {

         /* We need to check if the requested address is in the RAM
          * because we don't want to map the entire memory in QEMU.
          * In that case just map until the end of the page.
          */
         if (block->offset == 0) {
         /* We need to check if the requested address is in the RAM
          * because we don't want to map the entire memory in QEMU.
          * In that case just map until the end of the page.
          */
         if (block->offset == 0) {

-            return xen_map_cache(addr, 0, 0);
-        } else if (block->host == NULL) {
-            block->host =
-                xen_map_cache(block->offset, block->length, 1);
+            ptr = xen_map_cache(addr, 0, 0);
+            goto unlock;

         }
         }

+
+        block->host = xen_map_cache(block->offset, block->max_length, 1);

     }
     }

-    return block->host + (addr - block->offset);
+    ptr = ramblock_ptr(block, addr - block->offset);
+
+unlock:
+    rcu_read_unlock();
+    return ptr;

 }
 
 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
 }
 
 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr

- * but takes a size argument */
+ * but takes a size argument.
+ *
+ * By the time this function returns, the returned pointer is not protected
+ * by RCU anymore.  If the caller is not within an RCU critical section and
+ * does not hold the iothread lock, it must have other means of protecting the
+ * pointer, such as a reference to the region that includes the incoming
+ * ram_addr_t.
+ */

 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
 {
 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
 {

+    void *ptr;

     if (*size == 0) {
         return NULL;
     }
     if (*size == 0) {
         return NULL;
     }


@@ -1494,12 +1751,14 @@ static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
         return xen_map_cache(addr, *size, 1);
     } else {
         RAMBlock *block;
         return xen_map_cache(addr, *size, 1);
     } else {
         RAMBlock *block;

-
-        QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-            if (addr - block->offset < block->length) {
-                if (addr - block->offset + *size > block->length)
-                    *size = block->length - addr + block->offset;
-                return block->host + (addr - block->offset);
+        rcu_read_lock();
+        QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+            if (addr - block->offset < block->max_length) {
+                if (addr - block->offset + *size > block->max_length)
+                    *size = block->max_length - addr + block->offset;
+                ptr = ramblock_ptr(block, addr - block->offset);
+                rcu_read_unlock();
+                return ptr;

             }
         }
 
             }
         }
 


@@ -1509,37 +1768,52 @@ static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
 }
 
 /* Some of the softmmu routines need to translate from a host pointer
 }
 
 /* Some of the softmmu routines need to translate from a host pointer

-   (typically a TLB entry) back to a ram offset.  */
+ * (typically a TLB entry) back to a ram offset.
+ *
+ * By the time this function returns, the returned pointer is not protected
+ * by RCU anymore.  If the caller is not within an RCU critical section and
+ * does not hold the iothread lock, it must have other means of protecting the
+ * pointer, such as a reference to the region that includes the incoming
+ * ram_addr_t.
+ */

 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
 {
     RAMBlock *block;
     uint8_t *host = ptr;
 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
 {
     RAMBlock *block;
     uint8_t *host = ptr;

+    MemoryRegion *mr;

 
     if (xen_enabled()) {
 
     if (xen_enabled()) {

+        rcu_read_lock();

         *ram_addr = xen_ram_addr_from_mapcache(ptr);
         *ram_addr = xen_ram_addr_from_mapcache(ptr);

-        return qemu_get_ram_block(*ram_addr)->mr;
+        mr = qemu_get_ram_block(*ram_addr)->mr;
+        rcu_read_unlock();
+        return mr;

     }
 
     }
 

-    block = ram_list.mru_block;
-    if (block && block->host && host - block->host < block->length) {
+    rcu_read_lock();
+    block = atomic_rcu_read(&ram_list.mru_block);
+    if (block && block->host && host - block->host < block->max_length) {

         goto found;
     }
 
         goto found;
     }
 

-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {

         /* This case append when the block is not mapped. */
         if (block->host == NULL) {
             continue;
         }
         /* This case append when the block is not mapped. */
         if (block->host == NULL) {
             continue;
         }

-        if (host - block->host < block->length) {
+        if (host - block->host < block->max_length) {

             goto found;
         }
     }
 
             goto found;
         }
     }
 

+    rcu_read_unlock();

     return NULL;
 
 found:
     *ram_addr = block->offset + (host - block->host);
     return NULL;
 
 found:
     *ram_addr = block->offset + (host - block->host);

-    return block->mr;
+    mr = block->mr;
+    rcu_read_unlock();
+    return mr;

 }
 
 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
 }
 
 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,


@@ -1561,8 +1835,7 @@ static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
     default:
         abort();
     }
     default:
         abort();
     }

-    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
-    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
+    cpu_physical_memory_set_dirty_range_nocode(ram_addr, size);

     /* we remove the notdirty callback only if the code has been
        flushed */
     if (!cpu_physical_memory_is_clean(ram_addr)) {
     /* we remove the notdirty callback only if the code has been
        flushed */
     if (!cpu_physical_memory_is_clean(ram_addr)) {


@@ -1584,7 +1857,7 @@ static const MemoryRegionOps notdirty_mem_ops = {
 };
 
 /* Generate a debug exception if a watchpoint has been hit.  */
 };
 
 /* Generate a debug exception if a watchpoint has been hit.  */

-static void check_watchpoint(int offset, int len_mask, int flags)
+static void check_watchpoint(int offset, int len, int flags)

 {
     CPUState *cpu = current_cpu;
     CPUArchState *env = cpu->env_ptr;
 {
     CPUState *cpu = current_cpu;
     CPUArchState *env = cpu->env_ptr;


@@ -1602,9 +1875,14 @@ static void check_watchpoint(int offset, int len_mask, int flags)
     }
     vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
     }
     vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {

-        if ((vaddr == (wp->vaddr & len_mask) ||
-             (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
-            wp->flags |= BP_WATCHPOINT_HIT;
+        if (cpu_watchpoint_address_matches(wp, vaddr, len)
+            && (wp->flags & flags)) {
+            if (flags == BP_MEM_READ) {
+                wp->flags |= BP_WATCHPOINT_HIT_READ;
+            } else {
+                wp->flags |= BP_WATCHPOINT_HIT_WRITE;
+            }
+            wp->hitaddr = vaddr;

             if (!cpu->watchpoint_hit) {
                 cpu->watchpoint_hit = wp;
                 tb_check_watchpoint(cpu);
             if (!cpu->watchpoint_hit) {
                 cpu->watchpoint_hit = wp;
                 tb_check_watchpoint(cpu);


@@ -1629,7 +1907,7 @@ static void check_watchpoint(int offset, int len_mask, int flags)
 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {

-    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, BP_MEM_READ);

     switch (size) {
     case 1: return ldub_phys(&address_space_memory, addr);
     case 2: return lduw_phys(&address_space_memory, addr);
     switch (size) {
     case 1: return ldub_phys(&address_space_memory, addr);
     case 2: return lduw_phys(&address_space_memory, addr);


@@ -1641,7 +1919,7 @@ static uint64_t watch_mem_read(void *opaque, hwaddr addr,
 static void watch_mem_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
 static void watch_mem_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {

-    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, BP_MEM_WRITE);

     switch (size) {
     case 1:
         stb_phys(&address_space_memory, addr, val);
     switch (size) {
     case 1:
         stb_phys(&address_space_memory, addr, val);


@@ -1666,7 +1944,7 @@ static uint64_t subpage_read(void *opaque, hwaddr addr,
                              unsigned len)
 {
     subpage_t *subpage = opaque;
                              unsigned len)
 {
     subpage_t *subpage = opaque;

-    uint8_t buf[4];
+    uint8_t buf[8];

 
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
 
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,


@@ -1680,6 +1958,8 @@ static uint64_t subpage_read(void *opaque, hwaddr addr,
         return lduw_p(buf);
     case 4:
         return ldl_p(buf);
         return lduw_p(buf);
     case 4:
         return ldl_p(buf);

+    case 8:
+        return ldq_p(buf);

     default:
         abort();
     }
     default:
         abort();
     }


@@ -1689,7 +1969,7 @@ static void subpage_write(void *opaque, hwaddr addr,
                           uint64_t value, unsigned len)
 {
     subpage_t *subpage = opaque;
                           uint64_t value, unsigned len)
 {
     subpage_t *subpage = opaque;

-    uint8_t buf[4];
+    uint8_t buf[8];

 
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %u addr " TARGET_FMT_plx
 
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %u addr " TARGET_FMT_plx


@@ -1706,6 +1986,9 @@ static void subpage_write(void *opaque, hwaddr addr,
     case 4:
         stl_p(buf, value);
         break;
     case 4:
         stl_p(buf, value);
         break;

+    case 8:
+        stq_p(buf, value);
+        break;

     default:
         abort();
     }
     default:
         abort();
     }


@@ -1728,6 +2011,10 @@ static bool subpage_accepts(void *opaque, hwaddr addr,
 static const MemoryRegionOps subpage_ops = {
     .read = subpage_read,
     .write = subpage_write,
 static const MemoryRegionOps subpage_ops = {
     .read = subpage_read,
     .write = subpage_write,

+    .impl.min_access_size = 1,
+    .impl.max_access_size = 8,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 8,

     .valid.accepts = subpage_accepts,
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
     .valid.accepts = subpage_accepts,
     .endianness = DEVICE_NATIVE_ENDIAN,
 };


@@ -1761,7 +2048,7 @@ static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
     mmio->as = as;
     mmio->base = base;
     memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
     mmio->as = as;
     mmio->base = base;
     memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,

-                          "subpage", TARGET_PAGE_SIZE);
+                          NULL, TARGET_PAGE_SIZE);

     mmio->iomem.subpage = true;
 #if defined(DEBUG_SUBPAGE)
     printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
     mmio->iomem.subpage = true;
 #if defined(DEBUG_SUBPAGE)
     printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,


@@ -1787,20 +2074,23 @@ static uint16_t dummy_section(PhysPageMap *map, AddressSpace *as,
     return phys_section_add(map, &section);
 }
 
     return phys_section_add(map, &section);
 }
 

-MemoryRegion *iotlb_to_region(AddressSpace *as, hwaddr index)
+MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index)

 {
 {

-    return as->dispatch->map.sections[index & ~TARGET_PAGE_MASK].mr;
+    AddressSpaceDispatch *d = atomic_rcu_read(&cpu->memory_dispatch);
+    MemoryRegionSection *sections = d->map.sections;
+
+    return sections[index & ~TARGET_PAGE_MASK].mr;

 }
 
 static void io_mem_init(void)
 {
 }
 
 static void io_mem_init(void)
 {

-    memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
+    memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, NULL, UINT64_MAX);

     memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
     memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,

-                          "unassigned", UINT64_MAX);
+                          NULL, UINT64_MAX);

     memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
     memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,

-                          "notdirty", UINT64_MAX);
+                          NULL, UINT64_MAX);

     memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
     memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,

-                          "watch", UINT64_MAX);
+                          NULL, UINT64_MAX);

 }
 
 static void mem_begin(MemoryListener *listener)
 }
 
 static void mem_begin(MemoryListener *listener)


@@ -1823,6 +2113,12 @@ static void mem_begin(MemoryListener *listener)
     as->next_dispatch = d;
 }
 
     as->next_dispatch = d;
 }
 

+static void address_space_dispatch_free(AddressSpaceDispatch *d)
+{
+    phys_sections_free(&d->map);
+    g_free(d);
+}
+

 static void mem_commit(MemoryListener *listener)
 {
     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
 static void mem_commit(MemoryListener *listener)
 {
     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);


@@ -1831,11 +2127,9 @@ static void mem_commit(MemoryListener *listener)
 
     phys_page_compact_all(next, next->map.nodes_nb);
 
 
     phys_page_compact_all(next, next->map.nodes_nb);
 

-    as->dispatch = next;
-
+    atomic_rcu_set(&as->dispatch, next);

     if (cur) {
     if (cur) {

-        phys_sections_free(&cur->map);
-        g_free(cur);
+        call_rcu(cur, address_space_dispatch_free, rcu);

     }
 }
 
     }
 }
 


@@ -1852,7 +2146,7 @@ static void tcg_commit(MemoryListener *listener)
         if (cpu->tcg_as_listener != listener) {
             continue;
         }
         if (cpu->tcg_as_listener != listener) {
             continue;
         }

-        tlb_flush(cpu, 1);
+        cpu_reload_memory_map(cpu);

     }
 }
 
     }
 }
 


@@ -1885,13 +2179,19 @@ void address_space_init_dispatch(AddressSpace *as)
     memory_listener_register(&as->dispatch_listener, as);
 }
 
     memory_listener_register(&as->dispatch_listener, as);
 }
 

+void address_space_unregister(AddressSpace *as)
+{
+    memory_listener_unregister(&as->dispatch_listener);
+}
+

 void address_space_destroy_dispatch(AddressSpace *as)
 {
     AddressSpaceDispatch *d = as->dispatch;
 
 void address_space_destroy_dispatch(AddressSpace *as)
 {
     AddressSpaceDispatch *d = as->dispatch;
 

-    memory_listener_unregister(&as->dispatch_listener);
-    g_free(d);
-    as->dispatch = NULL;
+    atomic_rcu_set(&as->dispatch, NULL);
+    if (d) {
+        call_rcu(d, address_space_dispatch_free, rcu);
+    }

 }
 
 static void memory_map_init(void)
 }
 
 static void memory_map_init(void)


@@ -1967,12 +2267,9 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
 static void invalidate_and_set_dirty(hwaddr addr,
                                      hwaddr length)
 {
 static void invalidate_and_set_dirty(hwaddr addr,
                                      hwaddr length)
 {

-    if (cpu_physical_memory_is_clean(addr)) {
-        /* invalidate code */
-        tb_invalidate_phys_page_range(addr, addr + length, 0);
-        /* set dirty bit */
-        cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_VGA);
-        cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
+    if (cpu_physical_memory_range_includes_clean(addr, length)) {
+        tb_invalidate_phys_range(addr, addr + length, 0);
+        cpu_physical_memory_set_dirty_range_nocode(addr, length);

     }
     xen_modified_memory(addr, length);
 }
     }
     xen_modified_memory(addr, length);
 }


@@ -2328,15 +2625,7 @@ void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
         mr = qemu_ram_addr_from_host(buffer, &addr1);
         assert(mr != NULL);
         if (is_write) {
         mr = qemu_ram_addr_from_host(buffer, &addr1);
         assert(mr != NULL);
         if (is_write) {

-            while (access_len) {
-                unsigned l;
-                l = TARGET_PAGE_SIZE;
-                if (l > access_len)
-                    l = access_len;
-                invalidate_and_set_dirty(addr1, l);
-                addr1 += l;
-                access_len -= l;
-            }
+            invalidate_and_set_dirty(addr1, access_len);

         }
         if (xen_enabled()) {
             xen_invalidate_map_cache_entry(buffer);
         }
         if (xen_enabled()) {
             xen_invalidate_map_cache_entry(buffer);


@@ -2574,9 +2863,7 @@ void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
                 /* invalidate code */
                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                 /* set dirty bit */
                 /* invalidate code */
                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
                 /* set dirty bit */

-                cpu_physical_memory_set_dirty_flag(addr1,
-                                                   DIRTY_MEMORY_MIGRATION);
-                cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_VGA);
+                cpu_physical_memory_set_dirty_range_nocode(addr1, 4);

             }
         }
     }
             }
         }
     }


@@ -2752,14 +3039,12 @@ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
 }
 #endif
 
 }
 #endif
 

-#if !defined(CONFIG_USER_ONLY)
-

 /*
  * A helper function for the _utterly broken_ virtio device model to find out if
  * it's running on a big endian machine. Don't do this at home kids!
  */
 /*
  * A helper function for the _utterly broken_ virtio device model to find out if
  * it's running on a big endian machine. Don't do this at home kids!
  */

-bool virtio_is_big_endian(void);
-bool virtio_is_big_endian(void)
+bool target_words_bigendian(void);
+bool target_words_bigendian(void)

 {
 #if defined(TARGET_WORDS_BIGENDIAN)
     return true;
 {
 #if defined(TARGET_WORDS_BIGENDIAN)
     return true;


@@ -2768,8 +3053,6 @@ bool virtio_is_big_endian(void)
 #endif
 }
 
 #endif
 }
 

-#endif
-

 #ifndef CONFIG_USER_ONLY
 bool cpu_physical_memory_is_io(hwaddr phys_addr)
 {
 #ifndef CONFIG_USER_ONLY
 bool cpu_physical_memory_is_io(hwaddr phys_addr)
 {


@@ -2787,8 +3070,10 @@ void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
 {
     RAMBlock *block;
 
 {
     RAMBlock *block;
 

-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        func(block->host, block->offset, block->length, opaque);
+    rcu_read_lock();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        func(block->host, block->offset, block->used_length, opaque);

     }
     }

+    rcu_read_unlock();

 }
 #endif
 }
 #endif