X-Git-Url: https://repo.jachan.dev/qemu.git/blobdiff_plain/d634fc0499001c5ee49e1004576f9c92ea7c05ff..fa75ad1459f4f6abbeb6d375a812dfad61320f58:/memory.c diff --git a/memory.c b/memory.c index e26e5a3b1d..d852f1143d 100644 --- a/memory.c +++ b/memory.c @@ -19,7 +19,6 @@ #include "cpu.h" #include "exec/memory.h" #include "exec/address-spaces.h" -#include "exec/ioport.h" #include "qapi/visitor.h" #include "qemu/bitops.h" #include "qemu/error-report.h" @@ -30,7 +29,6 @@ #include "exec/ram_addr.h" #include "sysemu/kvm.h" #include "sysemu/sysemu.h" -#include "hw/misc/mmio_interface.h" #include "hw/qdev-properties.h" #include "migration/vmstate.h" @@ -173,45 +171,43 @@ struct MemoryRegionIoeventfd { EventNotifier *e; }; -static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd a, - MemoryRegionIoeventfd b) +static bool memory_region_ioeventfd_before(MemoryRegionIoeventfd *a, + MemoryRegionIoeventfd *b) { - if (int128_lt(a.addr.start, b.addr.start)) { + if (int128_lt(a->addr.start, b->addr.start)) { return true; - } else if (int128_gt(a.addr.start, b.addr.start)) { + } else if (int128_gt(a->addr.start, b->addr.start)) { return false; - } else if (int128_lt(a.addr.size, b.addr.size)) { + } else if (int128_lt(a->addr.size, b->addr.size)) { return true; - } else if (int128_gt(a.addr.size, b.addr.size)) { + } else if (int128_gt(a->addr.size, b->addr.size)) { return false; - } else if (a.match_data < b.match_data) { + } else if (a->match_data < b->match_data) { return true; - } else if (a.match_data > b.match_data) { + } else if (a->match_data > b->match_data) { return false; - } else if (a.match_data) { - if (a.data < b.data) { + } else if (a->match_data) { + if (a->data < b->data) { return true; - } else if (a.data > b.data) { + } else if (a->data > b->data) { return false; } } - if (a.e < b.e) { + if (a->e < b->e) { return true; - } else if (a.e > b.e) { + } else if (a->e > b->e) { return false; } return false; } -static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd a, - MemoryRegionIoeventfd b) +static bool memory_region_ioeventfd_equal(MemoryRegionIoeventfd *a, + MemoryRegionIoeventfd *b) { return !memory_region_ioeventfd_before(a, b) && !memory_region_ioeventfd_before(b, a); } -typedef struct FlatRange FlatRange; - /* Range of memory in the global map. Addresses are absolute. */ struct FlatRange { MemoryRegion *mr; @@ -222,21 +218,6 @@ struct FlatRange { bool readonly; }; -/* Flattened global view of current active memory hierarchy. Kept in sorted - * order. - */ -struct FlatView { - struct rcu_head rcu; - unsigned ref; - FlatRange *ranges; - unsigned nr; - unsigned nr_allocated; - struct AddressSpaceDispatch *dispatch; - MemoryRegion *root; -}; - -typedef struct AddressSpaceOps AddressSpaceOps; - #define FOR_EACH_FLAT_RANGE(var, view) \ for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) @@ -313,7 +294,7 @@ static bool flatview_ref(FlatView *view) return atomic_fetch_inc_nonzero(&view->ref) > 0; } -static void flatview_unref(FlatView *view) +void flatview_unref(FlatView *view) { if (atomic_fetch_dec(&view->ref) == 1) { trace_flatview_destroy_rcu(view, view->root); @@ -322,21 +303,6 @@ static void flatview_unref(FlatView *view) } } -FlatView *address_space_to_flatview(AddressSpace *as) -{ - return atomic_rcu_read(&as->current_map); -} - -AddressSpaceDispatch *flatview_to_dispatch(FlatView *fv) -{ - return fv->dispatch; -} - -AddressSpaceDispatch *address_space_to_dispatch(AddressSpace *as) -{ - return flatview_to_dispatch(address_space_to_flatview(as)); -} - static bool can_merge(FlatRange *r1, FlatRange *r2) { return int128_eq(addrrange_end(r1->addr), r2->addr.start) @@ -408,6 +374,33 @@ static void adjust_endianness(MemoryRegion *mr, uint64_t *data, unsigned size) } } +static inline void memory_region_shift_read_access(uint64_t *value, + signed shift, + uint64_t mask, + uint64_t tmp) +{ + if (shift >= 0) { + *value |= (tmp & mask) << shift; + } else { + *value |= (tmp & mask) >> -shift; + } +} + +static inline uint64_t memory_region_shift_write_access(uint64_t *value, + signed shift, + uint64_t mask) +{ + uint64_t tmp; + + if (shift >= 0) { + tmp = (*value >> shift) & mask; + } else { + tmp = (*value << -shift) & mask; + } + + return tmp; +} + static hwaddr memory_region_to_absolute_addr(MemoryRegion *mr, hwaddr offset) { MemoryRegion *root; @@ -430,37 +423,11 @@ static int get_cpu_index(void) return -1; } -static MemTxResult memory_region_oldmmio_read_accessor(MemoryRegion *mr, - hwaddr addr, - uint64_t *value, - unsigned size, - unsigned shift, - uint64_t mask, - MemTxAttrs attrs) -{ - uint64_t tmp; - - tmp = mr->ops->old_mmio.read[ctz32(size)](mr->opaque, addr); - if (mr->subpage) { - trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size); - } else if (mr == &io_mem_notdirty) { - /* Accesses to code which has previously been translated into a TB show - * up in the MMIO path, as accesses to the io_mem_notdirty - * MemoryRegion. */ - trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size); - } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) { - hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); - trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size); - } - *value |= (tmp & mask) << shift; - return MEMTX_OK; -} - static MemTxResult memory_region_read_accessor(MemoryRegion *mr, hwaddr addr, uint64_t *value, unsigned size, - unsigned shift, + signed shift, uint64_t mask, MemTxAttrs attrs) { @@ -478,7 +445,7 @@ static MemTxResult memory_region_read_accessor(MemoryRegion *mr, hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size); } - *value |= (tmp & mask) << shift; + memory_region_shift_read_access(value, shift, mask, tmp); return MEMTX_OK; } @@ -486,7 +453,7 @@ static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr, hwaddr addr, uint64_t *value, unsigned size, - unsigned shift, + signed shift, uint64_t mask, MemTxAttrs attrs) { @@ -505,47 +472,20 @@ static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr, hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size); } - *value |= (tmp & mask) << shift; + memory_region_shift_read_access(value, shift, mask, tmp); return r; } -static MemTxResult memory_region_oldmmio_write_accessor(MemoryRegion *mr, - hwaddr addr, - uint64_t *value, - unsigned size, - unsigned shift, - uint64_t mask, - MemTxAttrs attrs) -{ - uint64_t tmp; - - tmp = (*value >> shift) & mask; - if (mr->subpage) { - trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size); - } else if (mr == &io_mem_notdirty) { - /* Accesses to code which has previously been translated into a TB show - * up in the MMIO path, as accesses to the io_mem_notdirty - * MemoryRegion. */ - trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size); - } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) { - hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr); - trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size); - } - mr->ops->old_mmio.write[ctz32(size)](mr->opaque, addr, tmp); - return MEMTX_OK; -} - static MemTxResult memory_region_write_accessor(MemoryRegion *mr, hwaddr addr, uint64_t *value, unsigned size, - unsigned shift, + signed shift, uint64_t mask, MemTxAttrs attrs) { - uint64_t tmp; + uint64_t tmp = memory_region_shift_write_access(value, shift, mask); - tmp = (*value >> shift) & mask; if (mr->subpage) { trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size); } else if (mr == &io_mem_notdirty) { @@ -565,13 +505,12 @@ static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr, hwaddr addr, uint64_t *value, unsigned size, - unsigned shift, + signed shift, uint64_t mask, MemTxAttrs attrs) { - uint64_t tmp; + uint64_t tmp = memory_region_shift_write_access(value, shift, mask); - tmp = (*value >> shift) & mask; if (mr->subpage) { trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size); } else if (mr == &io_mem_notdirty) { @@ -596,7 +535,7 @@ static MemTxResult access_with_adjusted_size(hwaddr addr, hwaddr addr, uint64_t *value, unsigned size, - unsigned shift, + signed shift, uint64_t mask, MemTxAttrs attrs), MemoryRegion *mr, @@ -616,7 +555,7 @@ static MemTxResult access_with_adjusted_size(hwaddr addr, /* FIXME: support unaligned access? */ access_size = MAX(MIN(size, access_size_max), access_size_min); - access_mask = -1ULL >> (64 - access_size * 8); + access_mask = MAKE_64BIT_MASK(0, access_size * 8); if (memory_region_big_endian(mr)) { for (i = 0; i < size; i += access_size) { r |= access_fn(mr, addr + i, value, access_size, @@ -821,8 +760,8 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, while (iold < fds_old_nb || inew < fds_new_nb) { if (iold < fds_old_nb && (inew == fds_new_nb - || memory_region_ioeventfd_before(fds_old[iold], - fds_new[inew]))) { + || memory_region_ioeventfd_before(&fds_old[iold], + &fds_new[inew]))) { fd = &fds_old[iold]; section = (MemoryRegionSection) { .fv = address_space_to_flatview(as), @@ -834,8 +773,8 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, ++iold; } else if (inew < fds_new_nb && (iold == fds_old_nb - || memory_region_ioeventfd_before(fds_new[inew], - fds_old[iold]))) { + || memory_region_ioeventfd_before(&fds_new[inew], + &fds_old[iold]))) { fd = &fds_new[inew]; section = (MemoryRegionSection) { .fv = address_space_to_flatview(as), @@ -852,7 +791,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, } } -static FlatView *address_space_get_flatview(AddressSpace *as) +FlatView *address_space_get_flatview(AddressSpace *as) { FlatView *view; @@ -1091,6 +1030,7 @@ void memory_region_transaction_commit(void) address_space_update_ioeventfds(as); } memory_region_update_pending = false; + ioeventfd_update_pending = false; MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); } else if (ioeventfd_update_pending) { QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { @@ -1281,7 +1221,8 @@ static uint64_t unassigned_mem_read(void *opaque, hwaddr addr, printf("Unassigned mem read " TARGET_FMT_plx "\n", addr); #endif if (current_cpu != NULL) { - cpu_unassigned_access(current_cpu, addr, false, false, 0, size); + bool is_exec = current_cpu->mem_io_access_type == MMU_INST_FETCH; + cpu_unassigned_access(current_cpu, addr, false, is_exec, 0, size); } return 0; } @@ -1298,7 +1239,8 @@ static void unassigned_mem_write(void *opaque, hwaddr addr, } static bool unassigned_mem_accepts(void *opaque, hwaddr addr, - unsigned size, bool is_write) + unsigned size, bool is_write, + MemTxAttrs attrs) { return false; } @@ -1376,7 +1318,8 @@ static const MemoryRegionOps ram_device_mem_ops = { bool memory_region_access_valid(MemoryRegion *mr, hwaddr addr, unsigned size, - bool is_write) + bool is_write, + MemTxAttrs attrs) { int access_size_min, access_size_max; int access_size, i; @@ -1402,7 +1345,7 @@ bool memory_region_access_valid(MemoryRegion *mr, access_size = MAX(MIN(size, access_size_max), access_size_min); for (i = 0; i < size; i += access_size) { if (!mr->ops->valid.accepts(mr->opaque, addr + i, access_size, - is_write)) { + is_write, attrs)) { return false; } } @@ -1424,16 +1367,12 @@ static MemTxResult memory_region_dispatch_read1(MemoryRegion *mr, mr->ops->impl.max_access_size, memory_region_read_accessor, mr, attrs); - } else if (mr->ops->read_with_attrs) { + } else { return access_with_adjusted_size(addr, pval, size, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_read_with_attrs_accessor, mr, attrs); - } else { - return access_with_adjusted_size(addr, pval, size, 1, 4, - memory_region_oldmmio_read_accessor, - mr, attrs); } } @@ -1445,7 +1384,7 @@ MemTxResult memory_region_dispatch_read(MemoryRegion *mr, { MemTxResult r; - if (!memory_region_access_valid(mr, addr, size, false)) { + if (!memory_region_access_valid(mr, addr, size, false, attrs)) { *pval = unassigned_mem_read(mr, addr, size); return MEMTX_DECODE_ERROR; } @@ -1472,7 +1411,7 @@ static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr, ioeventfd.match_data = mr->ioeventfds[i].match_data; ioeventfd.e = mr->ioeventfds[i].e; - if (memory_region_ioeventfd_equal(ioeventfd, mr->ioeventfds[i])) { + if (memory_region_ioeventfd_equal(&ioeventfd, &mr->ioeventfds[i])) { event_notifier_set(ioeventfd.e); return true; } @@ -1487,7 +1426,7 @@ MemTxResult memory_region_dispatch_write(MemoryRegion *mr, unsigned size, MemTxAttrs attrs) { - if (!memory_region_access_valid(mr, addr, size, true)) { + if (!memory_region_access_valid(mr, addr, size, true, attrs)) { unassigned_mem_write(mr, addr, data, size); return MEMTX_DECODE_ERROR; } @@ -1505,17 +1444,13 @@ MemTxResult memory_region_dispatch_write(MemoryRegion *mr, mr->ops->impl.max_access_size, memory_region_write_accessor, mr, attrs); - } else if (mr->ops->write_with_attrs) { + } else { return access_with_adjusted_size(addr, &data, size, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_write_with_attrs_accessor, mr, attrs); - } else { - return access_with_adjusted_size(addr, &data, size, 1, 4, - memory_region_oldmmio_write_accessor, - mr, attrs); } } @@ -1538,12 +1473,28 @@ void memory_region_init_ram_nomigrate(MemoryRegion *mr, uint64_t size, Error **errp) { + memory_region_init_ram_shared_nomigrate(mr, owner, name, size, false, errp); +} + +void memory_region_init_ram_shared_nomigrate(MemoryRegion *mr, + Object *owner, + const char *name, + uint64_t size, + bool share, + Error **errp) +{ + Error *err = NULL; memory_region_init(mr, owner, name, size); mr->ram = true; mr->terminates = true; mr->destructor = memory_region_destructor_ram; - mr->ram_block = qemu_ram_alloc(size, mr, errp); + mr->ram_block = qemu_ram_alloc(size, share, mr, &err); mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0; + if (err) { + mr->size = int128_zero(); + object_unparent(OBJECT(mr)); + error_propagate(errp, err); + } } void memory_region_init_resizeable_ram(MemoryRegion *mr, @@ -1556,30 +1507,44 @@ void memory_region_init_resizeable_ram(MemoryRegion *mr, void *host), Error **errp) { + Error *err = NULL; memory_region_init(mr, owner, name, size); mr->ram = true; mr->terminates = true; mr->destructor = memory_region_destructor_ram; mr->ram_block = qemu_ram_alloc_resizeable(size, max_size, resized, - mr, errp); + mr, &err); mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0; + if (err) { + mr->size = int128_zero(); + object_unparent(OBJECT(mr)); + error_propagate(errp, err); + } } -#ifdef __linux__ +#ifdef CONFIG_POSIX void memory_region_init_ram_from_file(MemoryRegion *mr, struct Object *owner, const char *name, uint64_t size, - bool share, + uint64_t align, + uint32_t ram_flags, const char *path, Error **errp) { + Error *err = NULL; memory_region_init(mr, owner, name, size); mr->ram = true; mr->terminates = true; mr->destructor = memory_region_destructor_ram; - mr->ram_block = qemu_ram_alloc_from_file(size, mr, share, path, errp); + mr->align = align; + mr->ram_block = qemu_ram_alloc_from_file(size, mr, ram_flags, path, &err); mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0; + if (err) { + mr->size = int128_zero(); + object_unparent(OBJECT(mr)); + error_propagate(errp, err); + } } void memory_region_init_ram_from_fd(MemoryRegion *mr, @@ -1590,12 +1555,20 @@ void memory_region_init_ram_from_fd(MemoryRegion *mr, int fd, Error **errp) { + Error *err = NULL; memory_region_init(mr, owner, name, size); mr->ram = true; mr->terminates = true; mr->destructor = memory_region_destructor_ram; - mr->ram_block = qemu_ram_alloc_from_fd(size, mr, share, fd, errp); + mr->ram_block = qemu_ram_alloc_from_fd(size, mr, + share ? RAM_SHARED : 0, + fd, &err); mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0; + if (err) { + mr->size = int128_zero(); + object_unparent(OBJECT(mr)); + error_propagate(errp, err); + } } #endif @@ -1646,13 +1619,19 @@ void memory_region_init_rom_nomigrate(MemoryRegion *mr, uint64_t size, Error **errp) { + Error *err = NULL; memory_region_init(mr, owner, name, size); mr->ram = true; mr->readonly = true; mr->terminates = true; mr->destructor = memory_region_destructor_ram; - mr->ram_block = qemu_ram_alloc(size, mr, errp); + mr->ram_block = qemu_ram_alloc(size, false, mr, &err); mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0; + if (err) { + mr->size = int128_zero(); + object_unparent(OBJECT(mr)); + error_propagate(errp, err); + } } void memory_region_init_rom_device_nomigrate(MemoryRegion *mr, @@ -1663,6 +1642,7 @@ void memory_region_init_rom_device_nomigrate(MemoryRegion *mr, uint64_t size, Error **errp) { + Error *err = NULL; assert(ops); memory_region_init(mr, owner, name, size); mr->ops = ops; @@ -1670,7 +1650,12 @@ void memory_region_init_rom_device_nomigrate(MemoryRegion *mr, mr->terminates = true; mr->rom_device = true; mr->destructor = memory_region_destructor_ram; - mr->ram_block = qemu_ram_alloc(size, mr, errp); + mr->ram_block = qemu_ram_alloc(size, false, mr, &err); + if (err) { + mr->size = int128_zero(); + object_unparent(OBJECT(mr)); + error_propagate(errp, err); + } } void memory_region_init_iommu(void *_iommu_mr, @@ -1817,6 +1802,9 @@ void memory_region_register_iommu_notifier(MemoryRegion *mr, iommu_mr = IOMMU_MEMORY_REGION(mr); assert(n->notifier_flags != IOMMU_NOTIFIER_NONE); assert(n->start <= n->end); + assert(n->iommu_idx >= 0 && + n->iommu_idx < memory_region_iommu_num_indexes(iommu_mr)); + QLIST_INSERT_HEAD(&iommu_mr->iommu_notify, n, node); memory_region_update_iommu_notify_flags(iommu_mr); } @@ -1847,7 +1835,7 @@ void memory_region_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) granularity = memory_region_iommu_get_min_page_size(iommu_mr); for (addr = 0; addr < memory_region_size(mr); addr += granularity) { - iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE); + iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, n->iommu_idx); if (iotlb.perm != IOMMU_NONE) { n->notify(n, &iotlb); } @@ -1909,6 +1897,7 @@ void memory_region_notify_one(IOMMUNotifier *notifier, } void memory_region_notify_iommu(IOMMUMemoryRegion *iommu_mr, + int iommu_idx, IOMMUTLBEntry entry) { IOMMUNotifier *iommu_notifier; @@ -1916,8 +1905,46 @@ void memory_region_notify_iommu(IOMMUMemoryRegion *iommu_mr, assert(memory_region_is_iommu(MEMORY_REGION(iommu_mr))); IOMMU_NOTIFIER_FOREACH(iommu_notifier, iommu_mr) { - memory_region_notify_one(iommu_notifier, &entry); + if (iommu_notifier->iommu_idx == iommu_idx) { + memory_region_notify_one(iommu_notifier, &entry); + } + } +} + +int memory_region_iommu_get_attr(IOMMUMemoryRegion *iommu_mr, + enum IOMMUMemoryRegionAttr attr, + void *data) +{ + IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); + + if (!imrc->get_attr) { + return -EINVAL; + } + + return imrc->get_attr(iommu_mr, attr, data); +} + +int memory_region_iommu_attrs_to_index(IOMMUMemoryRegion *iommu_mr, + MemTxAttrs attrs) +{ + IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); + + if (!imrc->attrs_to_index) { + return 0; + } + + return imrc->attrs_to_index(iommu_mr, attrs); +} + +int memory_region_iommu_num_indexes(IOMMUMemoryRegion *iommu_mr) +{ + IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_GET_CLASS(iommu_mr); + + if (!imrc->num_indexes) { + return 1; } + + return imrc->num_indexes(iommu_mr); } void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) @@ -1955,33 +1982,7 @@ void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, memory_region_get_dirty_log_mask(mr)); } -bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr, - hwaddr size, unsigned client) -{ - assert(mr->ram_block); - return cpu_physical_memory_test_and_clear_dirty( - memory_region_get_ram_addr(mr) + addr, size, client); -} - -DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr, - hwaddr addr, - hwaddr size, - unsigned client) -{ - assert(mr->ram_block); - return cpu_physical_memory_snapshot_and_clear_dirty( - memory_region_get_ram_addr(mr) + addr, size, client); -} - -bool memory_region_snapshot_get_dirty(MemoryRegion *mr, DirtyBitmapSnapshot *snap, - hwaddr addr, hwaddr size) -{ - assert(mr->ram_block); - return cpu_physical_memory_snapshot_get_dirty(snap, - memory_region_get_ram_addr(mr) + addr, size); -} - -void memory_region_sync_dirty_bitmap(MemoryRegion *mr) +static void memory_region_sync_dirty_bitmap(MemoryRegion *mr) { MemoryListener *listener; AddressSpace *as; @@ -2000,7 +2001,7 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr) as = listener->address_space; view = address_space_get_flatview(as); FOR_EACH_FLAT_RANGE(fr, view) { - if (fr->mr == mr) { + if (fr->dirty_log_mask && (!mr || fr->mr == mr)) { MemoryRegionSection mrs = section_from_flat_range(fr, view); listener->log_sync(listener, &mrs); } @@ -2009,6 +2010,25 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr) } } +DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr, + hwaddr addr, + hwaddr size, + unsigned client) +{ + assert(mr->ram_block); + memory_region_sync_dirty_bitmap(mr); + return cpu_physical_memory_snapshot_and_clear_dirty( + memory_region_get_ram_addr(mr) + addr, size, client); +} + +bool memory_region_snapshot_get_dirty(MemoryRegion *mr, DirtyBitmapSnapshot *snap, + hwaddr addr, hwaddr size) +{ + assert(mr->ram_block); + return cpu_physical_memory_snapshot_get_dirty(snap, + memory_region_get_ram_addr(mr) + addr, size); +} + void memory_region_set_readonly(MemoryRegion *mr, bool readonly) { if (mr->readonly != readonly) { @@ -2189,11 +2209,6 @@ void memory_region_clear_flush_coalesced(MemoryRegion *mr) } } -void memory_region_set_global_locking(MemoryRegion *mr) -{ - mr->global_locking = true; -} - void memory_region_clear_global_locking(MemoryRegion *mr) { mr->global_locking = false; @@ -2229,7 +2244,7 @@ void memory_region_add_eventfd(MemoryRegion *mr, } memory_region_transaction_begin(); for (i = 0; i < mr->ioeventfd_nb; ++i) { - if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) { + if (memory_region_ioeventfd_before(&mrfd, &mr->ioeventfds[i])) { break; } } @@ -2264,7 +2279,7 @@ void memory_region_del_eventfd(MemoryRegion *mr, } memory_region_transaction_begin(); for (i = 0; i < mr->ioeventfd_nb; ++i) { - if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) { + if (memory_region_ioeventfd_equal(&mrfd, &mr->ioeventfds[i])) { break; } } @@ -2502,26 +2517,7 @@ bool memory_region_present(MemoryRegion *container, hwaddr addr) void memory_global_dirty_log_sync(void) { - MemoryListener *listener; - AddressSpace *as; - FlatView *view; - FlatRange *fr; - - QTAILQ_FOREACH(listener, &memory_listeners, link) { - if (!listener->log_sync) { - continue; - } - as = listener->address_space; - view = address_space_get_flatview(as); - FOR_EACH_FLAT_RANGE(fr, view) { - if (fr->dirty_log_mask) { - MemoryRegionSection mrs = section_from_flat_range(fr, view); - - listener->log_sync(listener, &mrs); - } - } - flatview_unref(view); - } + memory_region_sync_dirty_bitmap(NULL); } static VMChangeStateEntry *vmstate_change; @@ -2614,6 +2610,32 @@ static void listener_add_address_space(MemoryListener *listener, flatview_unref(view); } +static void listener_del_address_space(MemoryListener *listener, + AddressSpace *as) +{ + FlatView *view; + FlatRange *fr; + + if (listener->begin) { + listener->begin(listener); + } + view = address_space_get_flatview(as); + FOR_EACH_FLAT_RANGE(fr, view) { + MemoryRegionSection section = section_from_flat_range(fr, view); + + if (fr->dirty_log_mask && listener->log_stop) { + listener->log_stop(listener, §ion, fr->dirty_log_mask, 0); + } + if (listener->region_del) { + listener->region_del(listener, §ion); + } + } + if (listener->commit) { + listener->commit(listener); + } + flatview_unref(view); +} + void memory_listener_register(MemoryListener *listener, AddressSpace *as) { MemoryListener *other = NULL; @@ -2654,120 +2676,12 @@ void memory_listener_unregister(MemoryListener *listener) return; } + listener_del_address_space(listener, listener->address_space); QTAILQ_REMOVE(&memory_listeners, listener, link); QTAILQ_REMOVE(&listener->address_space->listeners, listener, link_as); listener->address_space = NULL; } -bool memory_region_request_mmio_ptr(MemoryRegion *mr, hwaddr addr) -{ - void *host; - unsigned size = 0; - unsigned offset = 0; - Object *new_interface; - - if (!mr || !mr->ops->request_ptr) { - return false; - } - - /* - * Avoid an update if the request_ptr call - * memory_region_invalidate_mmio_ptr which seems to be likely when we use - * a cache. - */ - memory_region_transaction_begin(); - - host = mr->ops->request_ptr(mr->opaque, addr - mr->addr, &size, &offset); - - if (!host || !size) { - memory_region_transaction_commit(); - return false; - } - - new_interface = object_new("mmio_interface"); - qdev_prop_set_uint64(DEVICE(new_interface), "start", offset); - qdev_prop_set_uint64(DEVICE(new_interface), "end", offset + size - 1); - qdev_prop_set_bit(DEVICE(new_interface), "ro", true); - qdev_prop_set_ptr(DEVICE(new_interface), "host_ptr", host); - qdev_prop_set_ptr(DEVICE(new_interface), "subregion", mr); - object_property_set_bool(OBJECT(new_interface), true, "realized", NULL); - - memory_region_transaction_commit(); - return true; -} - -typedef struct MMIOPtrInvalidate { - MemoryRegion *mr; - hwaddr offset; - unsigned size; - int busy; - int allocated; -} MMIOPtrInvalidate; - -#define MAX_MMIO_INVALIDATE 10 -static MMIOPtrInvalidate mmio_ptr_invalidate_list[MAX_MMIO_INVALIDATE]; - -static void memory_region_do_invalidate_mmio_ptr(CPUState *cpu, - run_on_cpu_data data) -{ - MMIOPtrInvalidate *invalidate_data = (MMIOPtrInvalidate *)data.host_ptr; - MemoryRegion *mr = invalidate_data->mr; - hwaddr offset = invalidate_data->offset; - unsigned size = invalidate_data->size; - MemoryRegionSection section = memory_region_find(mr, offset, size); - - qemu_mutex_lock_iothread(); - - /* Reset dirty so this doesn't happen later. */ - cpu_physical_memory_test_and_clear_dirty(offset, size, 1); - - if (section.mr != mr) { - /* memory_region_find add a ref on section.mr */ - memory_region_unref(section.mr); - if (MMIO_INTERFACE(section.mr->owner)) { - /* We found the interface just drop it. */ - object_property_set_bool(section.mr->owner, false, "realized", - NULL); - object_unref(section.mr->owner); - object_unparent(section.mr->owner); - } - } - - qemu_mutex_unlock_iothread(); - - if (invalidate_data->allocated) { - g_free(invalidate_data); - } else { - invalidate_data->busy = 0; - } -} - -void memory_region_invalidate_mmio_ptr(MemoryRegion *mr, hwaddr offset, - unsigned size) -{ - size_t i; - MMIOPtrInvalidate *invalidate_data = NULL; - - for (i = 0; i < MAX_MMIO_INVALIDATE; i++) { - if (atomic_cmpxchg(&(mmio_ptr_invalidate_list[i].busy), 0, 1) == 0) { - invalidate_data = &mmio_ptr_invalidate_list[i]; - break; - } - } - - if (!invalidate_data) { - invalidate_data = g_malloc0(sizeof(MMIOPtrInvalidate)); - invalidate_data->allocated = 1; - } - - invalidate_data->mr = mr; - invalidate_data->offset = offset; - invalidate_data->size = size; - - async_safe_run_on_cpu(first_cpu, memory_region_do_invalidate_mmio_ptr, - RUN_ON_CPU_HOST_PTR(invalidate_data)); -} - void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name) { memory_region_ref(root); @@ -2838,10 +2752,49 @@ typedef QTAILQ_HEAD(mrqueue, MemoryRegionList) MemoryRegionListHead; int128_sub((size), int128_one())) : 0) #define MTREE_INDENT " " +static void mtree_expand_owner(fprintf_function mon_printf, void *f, + const char *label, Object *obj) +{ + DeviceState *dev = (DeviceState *) object_dynamic_cast(obj, TYPE_DEVICE); + + mon_printf(f, " %s:{%s", label, dev ? "dev" : "obj"); + if (dev && dev->id) { + mon_printf(f, " id=%s", dev->id); + } else { + gchar *canonical_path = object_get_canonical_path(obj); + if (canonical_path) { + mon_printf(f, " path=%s", canonical_path); + g_free(canonical_path); + } else { + mon_printf(f, " type=%s", object_get_typename(obj)); + } + } + mon_printf(f, "}"); +} + +static void mtree_print_mr_owner(fprintf_function mon_printf, void *f, + const MemoryRegion *mr) +{ + Object *owner = mr->owner; + Object *parent = memory_region_owner((MemoryRegion *)mr); + + if (!owner && !parent) { + mon_printf(f, " orphan"); + return; + } + if (owner) { + mtree_expand_owner(mon_printf, f, "owner", owner); + } + if (parent && parent != owner) { + mtree_expand_owner(mon_printf, f, "parent", parent); + } +} + static void mtree_print_mr(fprintf_function mon_printf, void *f, const MemoryRegion *mr, unsigned int level, hwaddr base, - MemoryRegionListHead *alias_print_queue) + MemoryRegionListHead *alias_print_queue, + bool owner) { MemoryRegionList *new_ml, *ml, *next_ml; MemoryRegionListHead submr_print_queue; @@ -2887,7 +2840,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, } mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %s): alias %s @%s " TARGET_FMT_plx - "-" TARGET_FMT_plx "%s\n", + "-" TARGET_FMT_plx "%s", cur_start, cur_end, mr->priority, memory_region_type((MemoryRegion *)mr), @@ -2896,15 +2849,22 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, mr->alias_offset, mr->alias_offset + MR_SIZE(mr->size), mr->enabled ? "" : " [disabled]"); + if (owner) { + mtree_print_mr_owner(mon_printf, f, mr); + } } else { mon_printf(f, - TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %s): %s%s\n", + TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %s): %s%s", cur_start, cur_end, mr->priority, memory_region_type((MemoryRegion *)mr), memory_region_name(mr), mr->enabled ? "" : " [disabled]"); + if (owner) { + mtree_print_mr_owner(mon_printf, f, mr); + } } + mon_printf(f, "\n"); QTAILQ_INIT(&submr_print_queue); @@ -2927,7 +2887,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, QTAILQ_FOREACH(ml, &submr_print_queue, mrqueue) { mtree_print_mr(mon_printf, f, ml->mr, level + 1, cur_start, - alias_print_queue); + alias_print_queue, owner); } QTAILQ_FOREACH_SAFE(ml, &submr_print_queue, mrqueue, next_ml) { @@ -2940,6 +2900,7 @@ struct FlatViewInfo { void *f; int counter; bool dispatch_tree; + bool owner; }; static void mtree_print_flatview(gpointer key, gpointer value, @@ -2980,7 +2941,7 @@ static void mtree_print_flatview(gpointer key, gpointer value, mr = range->mr; if (range->offset_in_region) { p(f, MTREE_INDENT TARGET_FMT_plx "-" - TARGET_FMT_plx " (prio %d, %s): %s @" TARGET_FMT_plx "\n", + TARGET_FMT_plx " (prio %d, %s): %s @" TARGET_FMT_plx, int128_get64(range->addr.start), int128_get64(range->addr.start) + MR_SIZE(range->addr.size), mr->priority, @@ -2989,13 +2950,17 @@ static void mtree_print_flatview(gpointer key, gpointer value, range->offset_in_region); } else { p(f, MTREE_INDENT TARGET_FMT_plx "-" - TARGET_FMT_plx " (prio %d, %s): %s\n", + TARGET_FMT_plx " (prio %d, %s): %s", int128_get64(range->addr.start), int128_get64(range->addr.start) + MR_SIZE(range->addr.size), mr->priority, range->readonly ? "rom" : memory_region_type(mr), memory_region_name(mr)); } + if (fvi->owner) { + mtree_print_mr_owner(p, f, mr); + } + p(f, "\n"); range++; } @@ -3021,7 +2986,7 @@ static gboolean mtree_info_flatview_free(gpointer key, gpointer value, } void mtree_info(fprintf_function mon_printf, void *f, bool flatview, - bool dispatch_tree) + bool dispatch_tree, bool owner) { MemoryRegionListHead ml_head; MemoryRegionList *ml, *ml2; @@ -3033,7 +2998,8 @@ void mtree_info(fprintf_function mon_printf, void *f, bool flatview, .mon_printf = mon_printf, .f = f, .counter = 0, - .dispatch_tree = dispatch_tree + .dispatch_tree = dispatch_tree, + .owner = owner, }; GArray *fv_address_spaces; GHashTable *views = g_hash_table_new(g_direct_hash, g_direct_equal); @@ -3065,14 +3031,14 @@ void mtree_info(fprintf_function mon_printf, void *f, bool flatview, QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { mon_printf(f, "address-space: %s\n", as->name); - mtree_print_mr(mon_printf, f, as->root, 1, 0, &ml_head); + mtree_print_mr(mon_printf, f, as->root, 1, 0, &ml_head, owner); mon_printf(f, "\n"); } /* print aliased regions */ QTAILQ_FOREACH(ml, &ml_head, mrqueue) { mon_printf(f, "memory-region: %s\n", memory_region_name(ml->mr)); - mtree_print_mr(mon_printf, f, ml->mr, 1, 0, &ml_head); + mtree_print_mr(mon_printf, f, ml->mr, 1, 0, &ml_head, owner); mon_printf(f, "\n"); }