target-arm: Add Hyp mode checks to bad_mode_switch()

[qemu.git] / memory.c

diff --git a/memory.c b/memory.c
index 03c536b857cbb7257e6adf15b4831f467649712e..0dd9695aec5a58cd9e6adf7ad4dc33017f8ca0bc 100644 (file)
--- a/memory.c
+++ b/memory.c
@@ -13,21 +13,25 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
  * GNU GPL, version 2 or (at your option) any later version.
  */
 

+#include "qemu/osdep.h"

 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 #include "exec/ioport.h"
 #include "qapi/visitor.h"
 #include "qemu/bitops.h"
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 #include "exec/ioport.h"
 #include "qapi/visitor.h"
 #include "qemu/bitops.h"

+#include "qemu/error-report.h"

 #include "qom/object.h"
 #include "trace.h"
 #include "qom/object.h"
 #include "trace.h"

-#include <assert.h>

 
 #include "exec/memory-internal.h"
 #include "exec/ram_addr.h"
 
 #include "exec/memory-internal.h"
 #include "exec/ram_addr.h"

+#include "sysemu/kvm.h"

 #include "sysemu/sysemu.h"
 
 //#define DEBUG_UNASSIGNED
 
 #include "sysemu/sysemu.h"
 
 //#define DEBUG_UNASSIGNED
 

+#define RAM_ADDR_INVALID (~(ram_addr_t)0)
+

 static unsigned memory_region_transaction_depth;
 static bool memory_region_update_pending;
 static bool ioeventfd_update_pending;
 static unsigned memory_region_transaction_depth;
 static bool memory_region_update_pending;
 static bool ioeventfd_update_pending;


@@ -152,7 +156,7 @@ static bool memory_listener_match(MemoryListener *listener,
     } while (0)
 
 /* No need to ref/unref .mr, the FlatRange keeps it alive.  */
     } while (0)
 
 /* No need to ref/unref .mr, the FlatRange keeps it alive.  */

-#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback)            \
+#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback, _args...)  \

     MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) {       \
         .mr = (fr)->mr,                                                 \
         .address_space = (as),                                          \
     MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) {       \
         .mr = (fr)->mr,                                                 \
         .address_space = (as),                                          \


@@ -160,7 +164,7 @@ static bool memory_listener_match(MemoryListener *listener,
         .size = (fr)->addr.size,                                        \
         .offset_within_address_space = int128_get64((fr)->addr.start),  \
         .readonly = (fr)->readonly,                                     \
         .size = (fr)->addr.size,                                        \
         .offset_within_address_space = int128_get64((fr)->addr.start),  \
         .readonly = (fr)->readonly,                                     \

-              }))
+              }), ##_args)

 
 struct CoalescedMemoryRange {
     AddrRange addr;
 
 struct CoalescedMemoryRange {
     AddrRange addr;


@@ -394,9 +398,6 @@ static MemTxResult  memory_region_read_accessor(MemoryRegion *mr,
 {
     uint64_t tmp;
 
 {
     uint64_t tmp;
 

-    if (mr->flush_coalesced_mmio) {
-        qemu_flush_coalesced_mmio_buffer();
-    }

     tmp = mr->ops->read(mr->opaque, addr, size);
     trace_memory_region_ops_read(mr, addr, tmp, size);
     *value |= (tmp & mask) << shift;
     tmp = mr->ops->read(mr->opaque, addr, size);
     trace_memory_region_ops_read(mr, addr, tmp, size);
     *value |= (tmp & mask) << shift;


@@ -414,9 +415,6 @@ static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,
     uint64_t tmp = 0;
     MemTxResult r;
 
     uint64_t tmp = 0;
     MemTxResult r;
 

-    if (mr->flush_coalesced_mmio) {
-        qemu_flush_coalesced_mmio_buffer();
-    }

     r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs);
     trace_memory_region_ops_read(mr, addr, tmp, size);
     *value |= (tmp & mask) << shift;
     r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs);
     trace_memory_region_ops_read(mr, addr, tmp, size);
     *value |= (tmp & mask) << shift;


@@ -449,9 +447,6 @@ static MemTxResult memory_region_write_accessor(MemoryRegion *mr,
 {
     uint64_t tmp;
 
 {
     uint64_t tmp;
 

-    if (mr->flush_coalesced_mmio) {
-        qemu_flush_coalesced_mmio_buffer();
-    }

     tmp = (*value >> shift) & mask;
     trace_memory_region_ops_write(mr, addr, tmp, size);
     mr->ops->write(mr->opaque, addr, tmp, size);
     tmp = (*value >> shift) & mask;
     trace_memory_region_ops_write(mr, addr, tmp, size);
     mr->ops->write(mr->opaque, addr, tmp, size);


@@ -468,9 +463,6 @@ static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,
 {
     uint64_t tmp;
 
 {
     uint64_t tmp;
 

-    if (mr->flush_coalesced_mmio) {
-        qemu_flush_coalesced_mmio_buffer();
-    }

     tmp = (*value >> shift) & mask;
     trace_memory_region_ops_write(mr, addr, tmp, size);
     return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs);
     tmp = (*value >> shift) & mask;
     trace_memory_region_ops_write(mr, addr, tmp, size);
     return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs);


@@ -588,7 +580,7 @@ static void render_memory_region(FlatView *view,
     remain = clip.size;
 
     fr.mr = mr;
     remain = clip.size;
 
     fr.mr = mr;

-    fr.dirty_log_mask = mr->dirty_log_mask;
+    fr.dirty_log_mask = memory_region_get_dirty_log_mask(mr);

     fr.romd_mode = mr->romd_mode;
     fr.readonly = readonly;
 
     fr.romd_mode = mr->romd_mode;
     fr.readonly = readonly;
 


@@ -774,10 +766,15 @@ static void address_space_update_topology_pass(AddressSpace *as,
 
             if (adding) {
                 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop);
 
             if (adding) {
                 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop);

-                if (frold->dirty_log_mask && !frnew->dirty_log_mask) {
-                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop);
-                } else if (frnew->dirty_log_mask && !frold->dirty_log_mask) {
-                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start);
+                if (frnew->dirty_log_mask & ~frold->dirty_log_mask) {
+                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start,
+                                                  frold->dirty_log_mask,
+                                                  frnew->dirty_log_mask);
+                }
+                if (frold->dirty_log_mask & ~frnew->dirty_log_mask) {
+                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop,
+                                                  frold->dirty_log_mask,
+                                                  frnew->dirty_log_mask);

                 }
             }
 
                 }
             }
 


@@ -864,16 +861,6 @@ static void memory_region_destructor_ram(MemoryRegion *mr)
     qemu_ram_free(mr->ram_addr);
 }
 
     qemu_ram_free(mr->ram_addr);
 }
 

-static void memory_region_destructor_alias(MemoryRegion *mr)
-{
-    memory_region_unref(mr->alias);
-}
-
-static void memory_region_destructor_ram_from_ptr(MemoryRegion *mr)
-{
-    qemu_ram_free_from_ptr(mr->ram_addr);
-}
-

 static void memory_region_destructor_rom_device(MemoryRegion *mr)
 {
     qemu_ram_free(mr->ram_addr & TARGET_PAGE_MASK);
 static void memory_region_destructor_rom_device(MemoryRegion *mr)
 {
     qemu_ram_free(mr->ram_addr & TARGET_PAGE_MASK);


@@ -918,20 +905,23 @@ void memory_region_init(MemoryRegion *mr,
                         const char *name,
                         uint64_t size)
 {
                         const char *name,
                         uint64_t size)
 {

-    if (!owner) {
-        owner = container_get(qdev_get_machine(), "/unattached");
-    }
-

     object_initialize(mr, sizeof(*mr), TYPE_MEMORY_REGION);
     mr->size = int128_make64(size);
     if (size == UINT64_MAX) {
         mr->size = int128_2_64();
     }
     mr->name = g_strdup(name);
     object_initialize(mr, sizeof(*mr), TYPE_MEMORY_REGION);
     mr->size = int128_make64(size);
     if (size == UINT64_MAX) {
         mr->size = int128_2_64();
     }
     mr->name = g_strdup(name);

+    mr->owner = owner;
+    mr->ram_block = NULL;

 
     if (name) {
         char *escaped_name = memory_region_escape_name(name);
         char *name_array = g_strdup_printf("%s[*]", escaped_name);
 
     if (name) {
         char *escaped_name = memory_region_escape_name(name);
         char *name_array = g_strdup_printf("%s[*]", escaped_name);

+
+        if (!owner) {
+            owner = container_get(qdev_get_machine(), "/unattached");
+        }
+

         object_property_add_child(owner, name_array, OBJECT(mr), &error_abort);
         object_unref(OBJECT(mr));
         g_free(name_array);
         object_property_add_child(owner, name_array, OBJECT(mr), &error_abort);
         object_unref(OBJECT(mr));
         g_free(name_array);


@@ -939,17 +929,18 @@ void memory_region_init(MemoryRegion *mr,
     }
 }
 
     }
 }
 

-static void memory_region_get_addr(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void memory_region_get_addr(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)

 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     uint64_t value = mr->addr;
 
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     uint64_t value = mr->addr;
 

-    visit_type_uint64(v, &value, name, errp);
+    visit_type_uint64(v, name, &value, errp);

 }
 
 }
 

-static void memory_region_get_container(Object *obj, Visitor *v, void *opaque,
-                                        const char *name, Error **errp)
+static void memory_region_get_container(Object *obj, Visitor *v,
+                                        const char *name, void *opaque,
+                                        Error **errp)

 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     gchar *path = (gchar *)"";
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     gchar *path = (gchar *)"";


@@ -957,7 +948,7 @@ static void memory_region_get_container(Object *obj, Visitor *v, void *opaque,
     if (mr->container) {
         path = object_get_canonical_path(OBJECT(mr->container));
     }
     if (mr->container) {
         path = object_get_canonical_path(OBJECT(mr->container));
     }

-    visit_type_str(v, &path, name, errp);
+    visit_type_str(v, name, &path, errp);

     if (mr->container) {
         g_free(path);
     }
     if (mr->container) {
         g_free(path);
     }


@@ -971,13 +962,14 @@ static Object *memory_region_resolve_container(Object *obj, void *opaque,
     return OBJECT(mr->container);
 }
 
     return OBJECT(mr->container);
 }
 

-static void memory_region_get_priority(Object *obj, Visitor *v, void *opaque,
-                                       const char *name, Error **errp)
+static void memory_region_get_priority(Object *obj, Visitor *v,
+                                       const char *name, void *opaque,
+                                       Error **errp)

 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     int32_t value = mr->priority;
 
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     int32_t value = mr->priority;
 

-    visit_type_int32(v, &value, name, errp);
+    visit_type_int32(v, name, &value, errp);

 }
 
 static bool memory_region_get_may_overlap(Object *obj, Error **errp)
 }
 
 static bool memory_region_get_may_overlap(Object *obj, Error **errp)


@@ -987,13 +979,13 @@ static bool memory_region_get_may_overlap(Object *obj, Error **errp)
     return mr->may_overlap;
 }
 
     return mr->may_overlap;
 }
 

-static void memory_region_get_size(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void memory_region_get_size(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)

 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     uint64_t value = memory_region_size(mr);
 
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     uint64_t value = memory_region_size(mr);
 

-    visit_type_uint64(v, &value, name, errp);
+    visit_type_uint64(v, name, &value, errp);

 }
 
 static void memory_region_initfn(Object *obj)
 }
 
 static void memory_region_initfn(Object *obj)


@@ -1002,8 +994,10 @@ static void memory_region_initfn(Object *obj)
     ObjectProperty *op;
 
     mr->ops = &unassigned_mem_ops;
     ObjectProperty *op;
 
     mr->ops = &unassigned_mem_ops;

+    mr->ram_addr = RAM_ADDR_INVALID;

     mr->enabled = true;
     mr->romd_mode = true;
     mr->enabled = true;
     mr->romd_mode = true;

+    mr->global_locking = true;

     mr->destructor = memory_region_destructor_none;
     QTAILQ_INIT(&mr->subregions);
     QTAILQ_INIT(&mr->coalesced);
     mr->destructor = memory_region_destructor_none;
     QTAILQ_INIT(&mr->subregions);
     QTAILQ_INIT(&mr->coalesced);


@@ -1149,6 +1143,32 @@ MemTxResult memory_region_dispatch_read(MemoryRegion *mr,
     return r;
 }
 
     return r;
 }
 

+/* Return true if an eventfd was signalled */
+static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr,
+                                                    hwaddr addr,
+                                                    uint64_t data,
+                                                    unsigned size,
+                                                    MemTxAttrs attrs)
+{
+    MemoryRegionIoeventfd ioeventfd = {
+        .addr = addrrange_make(int128_make64(addr), int128_make64(size)),
+        .data = data,
+    };
+    unsigned i;
+
+    for (i = 0; i < mr->ioeventfd_nb; i++) {
+        ioeventfd.match_data = mr->ioeventfds[i].match_data;
+        ioeventfd.e = mr->ioeventfds[i].e;
+
+        if (memory_region_ioeventfd_equal(ioeventfd, mr->ioeventfds[i])) {
+            event_notifier_set(ioeventfd.e);
+            return true;
+        }
+    }
+
+    return false;
+}
+

 MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
                                          hwaddr addr,
                                          uint64_t data,
 MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
                                          hwaddr addr,
                                          uint64_t data,


@@ -1162,6 +1182,11 @@ MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
 
     adjust_endianness(mr, &data, size);
 
 
     adjust_endianness(mr, &data, size);
 

+    if ((!kvm_eventfds_enabled()) &&
+        memory_region_dispatch_write_eventfds(mr, addr, data, size, attrs)) {
+        return MEMTX_OK;
+    }
+

     if (mr->ops->write) {
         return access_with_adjusted_size(addr, &data, size,
                                          mr->ops->impl.min_access_size,
     if (mr->ops->write) {
         return access_with_adjusted_size(addr, &data, size,
                                          mr->ops->impl.min_access_size,


@@ -1190,10 +1215,9 @@ void memory_region_init_io(MemoryRegion *mr,
                            uint64_t size)
 {
     memory_region_init(mr, owner, name, size);
                            uint64_t size)
 {
     memory_region_init(mr, owner, name, size);

-    mr->ops = ops;
+    mr->ops = ops ? ops : &unassigned_mem_ops;

     mr->opaque = opaque;
     mr->terminates = true;
     mr->opaque = opaque;
     mr->terminates = true;

-    mr->ram_addr = ~(ram_addr_t)0;

 }
 
 void memory_region_init_ram(MemoryRegion *mr,
 }
 
 void memory_region_init_ram(MemoryRegion *mr,


@@ -1207,6 +1231,7 @@ void memory_region_init_ram(MemoryRegion *mr,
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
     mr->ram_addr = qemu_ram_alloc(size, mr, errp);
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
     mr->ram_addr = qemu_ram_alloc(size, mr, errp);

+    mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;

 }
 
 void memory_region_init_resizeable_ram(MemoryRegion *mr,
 }
 
 void memory_region_init_resizeable_ram(MemoryRegion *mr,


@@ -1224,6 +1249,7 @@ void memory_region_init_resizeable_ram(MemoryRegion *mr,
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
     mr->ram_addr = qemu_ram_alloc_resizeable(size, max_size, resized, mr, errp);
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
     mr->ram_addr = qemu_ram_alloc_resizeable(size, max_size, resized, mr, errp);

+    mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;

 }
 
 #ifdef __linux__
 }
 
 #ifdef __linux__


@@ -1240,6 +1266,7 @@ void memory_region_init_ram_from_file(MemoryRegion *mr,
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
     mr->ram_addr = qemu_ram_alloc_from_file(size, mr, share, path, errp);
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
     mr->ram_addr = qemu_ram_alloc_from_file(size, mr, share, path, errp);

+    mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;

 }
 #endif
 
 }
 #endif
 


@@ -1252,11 +1279,12 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
     memory_region_init(mr, owner, name, size);
     mr->ram = true;
     mr->terminates = true;
     memory_region_init(mr, owner, name, size);
     mr->ram = true;
     mr->terminates = true;

-    mr->destructor = memory_region_destructor_ram_from_ptr;
+    mr->destructor = memory_region_destructor_ram;
+    mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;

 
     /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
     assert(ptr != NULL);
 
     /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
     assert(ptr != NULL);

-    mr->ram_addr = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort);
+    mr->ram_addr = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);

 }
 
 void memory_region_set_skip_dump(MemoryRegion *mr)
 }
 
 void memory_region_set_skip_dump(MemoryRegion *mr)


@@ -1272,8 +1300,6 @@ void memory_region_init_alias(MemoryRegion *mr,
                               uint64_t size)
 {
     memory_region_init(mr, owner, name, size);
                               uint64_t size)
 {
     memory_region_init(mr, owner, name, size);

-    memory_region_ref(orig);
-    mr->destructor = memory_region_destructor_alias;

     mr->alias = orig;
     mr->alias_offset = offset;
 }
     mr->alias = orig;
     mr->alias_offset = offset;
 }


@@ -1307,19 +1333,26 @@ void memory_region_init_iommu(MemoryRegion *mr,
     notifier_list_init(&mr->iommu_notify);
 }
 
     notifier_list_init(&mr->iommu_notify);
 }
 

-void memory_region_init_reservation(MemoryRegion *mr,
-                                    Object *owner,
-                                    const char *name,
-                                    uint64_t size)
-{
-    memory_region_init_io(mr, owner, &unassigned_mem_ops, mr, name, size);
-}
-

 static void memory_region_finalize(Object *obj)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
 
 static void memory_region_finalize(Object *obj)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
 

-    assert(QTAILQ_EMPTY(&mr->subregions));
+    assert(!mr->container);
+
+    /* We know the region is not visible in any address space (it
+     * does not have a container and cannot be a root either because
+     * it has no references, so we can blindly clear mr->enabled.
+     * memory_region_set_enabled instead could trigger a transaction
+     * and cause an infinite loop.
+     */
+    mr->enabled = false;
+    memory_region_transaction_begin();
+    while (!QTAILQ_EMPTY(&mr->subregions)) {
+        MemoryRegion *subregion = QTAILQ_FIRST(&mr->subregions);
+        memory_region_del_subregion(mr, subregion);
+    }
+    memory_region_transaction_commit();
+

     mr->destructor(mr);
     memory_region_clear_coalescing(mr);
     g_free((char *)mr->name);
     mr->destructor(mr);
     memory_region_clear_coalescing(mr);
     g_free((char *)mr->name);


@@ -1341,24 +1374,18 @@ void memory_region_ref(MemoryRegion *mr)
      * The memory region is a child of its owner.  As long as the
      * owner doesn't call unparent itself on the memory region,
      * ref-ing the owner will also keep the memory region alive.
      * The memory region is a child of its owner.  As long as the
      * owner doesn't call unparent itself on the memory region,
      * ref-ing the owner will also keep the memory region alive.

-     * Memory regions without an owner are supposed to never go away,
-     * but we still ref/unref them for debugging purposes.
+     * Memory regions without an owner are supposed to never go away;
+     * we do not ref/unref them because it slows down DMA sensibly.

      */
      */

-    Object *obj = OBJECT(mr);
-    if (obj && obj->parent) {
-        object_ref(obj->parent);
-    } else {
-        object_ref(obj);
+    if (mr && mr->owner) {
+        object_ref(mr->owner);

     }
 }
 
 void memory_region_unref(MemoryRegion *mr)
 {
     }
 }
 
 void memory_region_unref(MemoryRegion *mr)
 {

-    Object *obj = OBJECT(mr);
-    if (obj && obj->parent) {
-        object_unref(obj->parent);
-    } else {
-        object_unref(obj);
+    if (mr && mr->owner) {
+        object_unref(mr->owner);

     }
 }
 
     }
 }
 


@@ -1379,34 +1406,48 @@ const char *memory_region_name(const MemoryRegion *mr)
     return mr->name;
 }
 
     return mr->name;
 }
 

-bool memory_region_is_ram(MemoryRegion *mr)
-{
-    return mr->ram;
-}
-

 bool memory_region_is_skip_dump(MemoryRegion *mr)
 {
     return mr->skip_dump;
 }
 
 bool memory_region_is_skip_dump(MemoryRegion *mr)
 {
     return mr->skip_dump;
 }
 

-bool memory_region_is_logging(MemoryRegion *mr)
+uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr)

 {
 {

-    return mr->dirty_log_mask;
+    uint8_t mask = mr->dirty_log_mask;
+    if (global_dirty_log) {
+        mask |= (1 << DIRTY_MEMORY_MIGRATION);
+    }
+    return mask;

 }
 
 }
 

-bool memory_region_is_rom(MemoryRegion *mr)
+bool memory_region_is_logging(MemoryRegion *mr, uint8_t client)

 {
 {

-    return mr->ram && mr->readonly;
+    return memory_region_get_dirty_log_mask(mr) & (1 << client);

 }
 
 }
 

-bool memory_region_is_iommu(MemoryRegion *mr)
+void memory_region_register_iommu_notifier(MemoryRegion *mr, Notifier *n)

 {
 {

-    return mr->iommu_ops;
+    notifier_list_add(&mr->iommu_notify, n);

 }
 
 }
 

-void memory_region_register_iommu_notifier(MemoryRegion *mr, Notifier *n)
+void memory_region_iommu_replay(MemoryRegion *mr, Notifier *n,
+                                hwaddr granularity, bool is_write)

 {
 {

-    notifier_list_add(&mr->iommu_notify, n);
+    hwaddr addr;
+    IOMMUTLBEntry iotlb;
+
+    for (addr = 0; addr < memory_region_size(mr); addr += granularity) {
+        iotlb = mr->iommu_ops->translate(mr, addr, is_write);
+        if (iotlb.perm != IOMMU_NONE) {
+            n->notify(n, &iotlb);
+        }
+
+        /* if (2^64 - MR size) < granularity, it's possible to get an
+         * infinite loop here.  This should catch such a wraparound */
+        if ((addr + granularity) < addr) {
+            break;
+        }
+    }

 }
 
 void memory_region_unregister_iommu_notifier(Notifier *n)
 }
 
 void memory_region_unregister_iommu_notifier(Notifier *n)


@@ -1424,6 +1465,14 @@ void memory_region_notify_iommu(MemoryRegion *mr,
 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
 {
     uint8_t mask = 1 << client;
 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
 {
     uint8_t mask = 1 << client;

+    uint8_t old_logging;
+
+    assert(client == DIRTY_MEMORY_VGA);
+    old_logging = mr->vga_logging_count;
+    mr->vga_logging_count += log ? 1 : -1;
+    if (!!old_logging == !!mr->vga_logging_count) {
+        return;
+    }

 
     memory_region_transaction_begin();
     mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask);
 
     memory_region_transaction_begin();
     mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask);


@@ -1434,27 +1483,24 @@ void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
 bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr,
                              hwaddr size, unsigned client)
 {
 bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr,
                              hwaddr size, unsigned client)
 {

-    assert(mr->terminates);
+    assert(mr->ram_addr != RAM_ADDR_INVALID);

     return cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, client);
 }
 
 void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
                              hwaddr size)
 {
     return cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, client);
 }
 
 void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
                              hwaddr size)
 {

-    assert(mr->terminates);
-    cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size);
+    assert(mr->ram_addr != RAM_ADDR_INVALID);
+    cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size,
+                                        memory_region_get_dirty_log_mask(mr));

 }
 
 bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr,
                                         hwaddr size, unsigned client)
 {
 }
 
 bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr,
                                         hwaddr size, unsigned client)
 {

-    bool ret;
-    assert(mr->terminates);
-    ret = cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, client);
-    if (ret) {
-        cpu_physical_memory_reset_dirty(mr->ram_addr + addr, size, client);
-    }
-    return ret;
+    assert(mr->ram_addr != RAM_ADDR_INVALID);
+    return cpu_physical_memory_test_and_clear_dirty(mr->ram_addr + addr,
+                                                    size, client);

 }
 
 
 }
 
 


@@ -1497,8 +1543,9 @@ void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode)
 void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
                                hwaddr size, unsigned client)
 {
 void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
                                hwaddr size, unsigned client)
 {

-    assert(mr->terminates);
-    cpu_physical_memory_reset_dirty(mr->ram_addr + addr, size, client);
+    assert(mr->ram_addr != RAM_ADDR_INVALID);
+    cpu_physical_memory_test_and_clear_dirty(mr->ram_addr + addr, size,
+                                             client);

 }
 
 int memory_region_get_fd(MemoryRegion *mr)
 }
 
 int memory_region_get_fd(MemoryRegion *mr)


@@ -1507,25 +1554,31 @@ int memory_region_get_fd(MemoryRegion *mr)
         return memory_region_get_fd(mr->alias);
     }
 
         return memory_region_get_fd(mr->alias);
     }
 

-    assert(mr->terminates);
+    assert(mr->ram_addr != RAM_ADDR_INVALID);

 
     return qemu_get_ram_fd(mr->ram_addr & TARGET_PAGE_MASK);
 }
 
 void *memory_region_get_ram_ptr(MemoryRegion *mr)
 {
 
     return qemu_get_ram_fd(mr->ram_addr & TARGET_PAGE_MASK);
 }
 
 void *memory_region_get_ram_ptr(MemoryRegion *mr)
 {

-    if (mr->alias) {
-        return memory_region_get_ram_ptr(mr->alias) + mr->alias_offset;
-    }
+    void *ptr;
+    uint64_t offset = 0;

 
 

-    assert(mr->terminates);
+    rcu_read_lock();
+    while (mr->alias) {
+        offset += mr->alias_offset;
+        mr = mr->alias;
+    }
+    assert(mr->ram_addr != RAM_ADDR_INVALID);
+    ptr = qemu_get_ram_ptr(mr->ram_block, mr->ram_addr & TARGET_PAGE_MASK);
+    rcu_read_unlock();

 
 

-    return qemu_get_ram_ptr(mr->ram_addr & TARGET_PAGE_MASK);
+    return ptr + offset;

 }
 
 void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize, Error **errp)
 {
 }
 
 void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize, Error **errp)
 {

-    assert(mr->terminates);
+    assert(mr->ram_addr != RAM_ADDR_INVALID);

 
     qemu_ram_resize(mr->ram_addr, newsize, errp);
 }
 
     qemu_ram_resize(mr->ram_addr, newsize, errp);
 }


@@ -1627,6 +1680,18 @@ void memory_region_clear_flush_coalesced(MemoryRegion *mr)
     }
 }
 
     }
 }
 

+void memory_region_set_global_locking(MemoryRegion *mr)
+{
+    mr->global_locking = true;
+}
+
+void memory_region_clear_global_locking(MemoryRegion *mr)
+{
+    mr->global_locking = false;
+}
+
+static bool userspace_eventfd_warning;
+

 void memory_region_add_eventfd(MemoryRegion *mr,
                                hwaddr addr,
                                unsigned size,
 void memory_region_add_eventfd(MemoryRegion *mr,
                                hwaddr addr,
                                unsigned size,


@@ -1643,7 +1708,16 @@ void memory_region_add_eventfd(MemoryRegion *mr,
     };
     unsigned i;
 
     };
     unsigned i;
 

-    adjust_endianness(mr, &mrfd.data, size);
+    if (kvm_enabled() && (!(kvm_eventfds_enabled() ||
+                            userspace_eventfd_warning))) {
+        userspace_eventfd_warning = true;
+        error_report("Using eventfd without MMIO binding in KVM. "
+                     "Suboptimal performance expected");
+    }
+
+    if (size) {
+        adjust_endianness(mr, &mrfd.data, size);
+    }

     memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) {
     memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) {


@@ -1676,7 +1750,9 @@ void memory_region_del_eventfd(MemoryRegion *mr,
     };
     unsigned i;
 
     };
     unsigned i;
 

-    adjust_endianness(mr, &mrfd.data, size);
+    if (size) {
+        adjust_endianness(mr, &mrfd.data, size);
+    }

     memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) {
     memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) {


@@ -1840,11 +1916,6 @@ void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset)
     memory_region_transaction_commit();
 }
 
     memory_region_transaction_commit();
 }
 

-ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
-{
-    return mr->ram_addr;
-}
-

 uint64_t memory_region_get_alignment(const MemoryRegion *mr)
 {
     return mr->align;
 uint64_t memory_region_get_alignment(const MemoryRegion *mr)
 {
     return mr->align;


@@ -1869,23 +1940,16 @@ static FlatRange *flatview_lookup(FlatView *view, AddrRange addr)
                    sizeof(FlatRange), cmp_flatrange_addr);
 }
 
                    sizeof(FlatRange), cmp_flatrange_addr);
 }
 

-bool memory_region_present(MemoryRegion *container, hwaddr addr)
-{
-    MemoryRegion *mr = memory_region_find(container, addr, 1).mr;
-    if (!mr || (mr == container)) {
-        return false;
-    }
-    memory_region_unref(mr);
-    return true;
-}
-

 bool memory_region_is_mapped(MemoryRegion *mr)
 {
     return mr->container ? true : false;
 }
 
 bool memory_region_is_mapped(MemoryRegion *mr)
 {
     return mr->container ? true : false;
 }
 

-MemoryRegionSection memory_region_find(MemoryRegion *mr,
-                                       hwaddr addr, uint64_t size)
+/* Same as memory_region_find, but it does not add a reference to the
+ * returned region.  It must be called from an RCU critical section.
+ */
+static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr,
+                                                  hwaddr addr, uint64_t size)

 {
     MemoryRegionSection ret = { .mr = NULL };
     MemoryRegion *root;
 {
     MemoryRegionSection ret = { .mr = NULL };
     MemoryRegion *root;


@@ -1906,11 +1970,10 @@ MemoryRegionSection memory_region_find(MemoryRegion *mr,
     }
     range = addrrange_make(int128_make64(addr), int128_make64(size));
 
     }
     range = addrrange_make(int128_make64(addr), int128_make64(size));
 

-    rcu_read_lock();

     view = atomic_rcu_read(&as->current_map);
     fr = flatview_lookup(view, range);
     if (!fr) {
     view = atomic_rcu_read(&as->current_map);
     fr = flatview_lookup(view, range);
     if (!fr) {

-        goto out;
+        return ret;

     }
 
     while (fr > view->ranges && addrrange_intersects(fr[-1].addr, range)) {
     }
 
     while (fr > view->ranges && addrrange_intersects(fr[-1].addr, range)) {


@@ -1926,12 +1989,32 @@ MemoryRegionSection memory_region_find(MemoryRegion *mr,
     ret.size = range.size;
     ret.offset_within_address_space = int128_get64(range.start);
     ret.readonly = fr->readonly;
     ret.size = range.size;
     ret.offset_within_address_space = int128_get64(range.start);
     ret.readonly = fr->readonly;

-    memory_region_ref(ret.mr);
-out:
+    return ret;
+}
+
+MemoryRegionSection memory_region_find(MemoryRegion *mr,
+                                       hwaddr addr, uint64_t size)
+{
+    MemoryRegionSection ret;
+    rcu_read_lock();
+    ret = memory_region_find_rcu(mr, addr, size);
+    if (ret.mr) {
+        memory_region_ref(ret.mr);
+    }

     rcu_read_unlock();
     return ret;
 }
 
     rcu_read_unlock();
     return ret;
 }
 

+bool memory_region_present(MemoryRegion *container, hwaddr addr)
+{
+    MemoryRegion *mr;
+
+    rcu_read_lock();
+    mr = memory_region_find_rcu(container, addr, 1).mr;
+    rcu_read_unlock();
+    return mr && mr != container;
+}
+

 void address_space_sync_dirty_bitmap(AddressSpace *as)
 {
     FlatView *view;
 void address_space_sync_dirty_bitmap(AddressSpace *as)
 {
     FlatView *view;


@@ -1947,12 +2030,24 @@ void address_space_sync_dirty_bitmap(AddressSpace *as)
 void memory_global_dirty_log_start(void)
 {
     global_dirty_log = true;
 void memory_global_dirty_log_start(void)
 {
     global_dirty_log = true;

+

     MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);
     MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);

+
+    /* Refresh DIRTY_LOG_MIGRATION bit.  */
+    memory_region_transaction_begin();
+    memory_region_update_pending = true;
+    memory_region_transaction_commit();

 }
 
 void memory_global_dirty_log_stop(void)
 {
     global_dirty_log = false;
 }
 
 void memory_global_dirty_log_stop(void)
 {
     global_dirty_log = false;

+
+    /* Refresh DIRTY_LOG_MIGRATION bit.  */
+    memory_region_transaction_begin();
+    memory_region_update_pending = true;
+    memory_region_transaction_commit();
+

     MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
 }
 
     MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
 }
 


@@ -1967,6 +2062,9 @@ static void listener_add_address_space(MemoryListener *listener,
         return;
     }
 
         return;
     }
 

+    if (listener->begin) {
+        listener->begin(listener);
+    }

     if (global_dirty_log) {
         if (listener->log_global_start) {
             listener->log_global_start(listener);
     if (global_dirty_log) {
         if (listener->log_global_start) {
             listener->log_global_start(listener);


@@ -1983,10 +2081,16 @@ static void listener_add_address_space(MemoryListener *listener,
             .offset_within_address_space = int128_get64(fr->addr.start),
             .readonly = fr->readonly,
         };
             .offset_within_address_space = int128_get64(fr->addr.start),
             .readonly = fr->readonly,
         };

+        if (fr->dirty_log_mask && listener->log_start) {
+            listener->log_start(listener, &section, 0, fr->dirty_log_mask);
+        }

         if (listener->region_add) {
             listener->region_add(listener, &section);
         }
     }
         if (listener->region_add) {
             listener->region_add(listener, &section);
         }
     }

+    if (listener->commit) {
+        listener->commit(listener);
+    }

     flatview_unref(view);
 }
 
     flatview_unref(view);
 }
 


@@ -2023,7 +2127,9 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name)
 {
     memory_region_ref(root);
     memory_region_transaction_begin();
 {
     memory_region_ref(root);
     memory_region_transaction_begin();

+    as->ref_count = 1;

     as->root = root;
     as->root = root;

+    as->malloced = false;

     as->current_map = g_new(FlatView, 1);
     flatview_init(as->current_map);
     as->ioeventfd_nb = 0;
     as->current_map = g_new(FlatView, 1);
     flatview_init(as->current_map);
     as->ioeventfd_nb = 0;


@@ -2038,6 +2144,7 @@ void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name)
 static void do_address_space_destroy(AddressSpace *as)
 {
     MemoryListener *listener;
 static void do_address_space_destroy(AddressSpace *as)
 {
     MemoryListener *listener;

+    bool do_free = as->malloced;

 
     address_space_destroy_dispatch(as);
 
 
     address_space_destroy_dispatch(as);
 


@@ -2049,12 +2156,36 @@ static void do_address_space_destroy(AddressSpace *as)
     g_free(as->name);
     g_free(as->ioeventfds);
     memory_region_unref(as->root);
     g_free(as->name);
     g_free(as->ioeventfds);
     memory_region_unref(as->root);

+    if (do_free) {
+        g_free(as);
+    }
+}
+
+AddressSpace *address_space_init_shareable(MemoryRegion *root, const char *name)
+{
+    AddressSpace *as;
+
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        if (root == as->root && as->malloced) {
+            as->ref_count++;
+            return as;
+        }
+    }
+
+    as = g_malloc0(sizeof *as);
+    address_space_init(as, root, name);
+    as->malloced = true;
+    return as;

 }
 
 void address_space_destroy(AddressSpace *as)
 {
     MemoryRegion *root = as->root;
 
 }
 
 void address_space_destroy(AddressSpace *as)
 {
     MemoryRegion *root = as->root;
 

+    as->ref_count--;
+    if (as->ref_count) {
+        return;
+    }

     /* Flush out anything from MemoryListeners listening in on this */
     memory_region_transaction_begin();
     as->root = NULL;
     /* Flush out anything from MemoryListeners listening in on this */
     memory_region_transaction_begin();
     as->root = NULL;