X-Git-Url: https://repo.jachan.dev/qemu.git/blobdiff_plain/09d0726c0351cbf71b96b41b9f39cc7857b45ccc..8312976e73fce9689ab831c1da565ec413680cff:/memory.c diff --git a/memory.c b/memory.c index 4f3ade06dd..5cb8f4a8b2 100644 --- a/memory.c +++ b/memory.c @@ -13,22 +13,27 @@ * GNU GPL, version 2 or (at your option) any later version. */ -#include "memory.h" -#include "exec-memory.h" -#include "ioport.h" -#include "bitops.h" -#include "kvm.h" +#include "exec/memory.h" +#include "exec/address-spaces.h" +#include "exec/ioport.h" +#include "qemu/bitops.h" +#include "sysemu/kvm.h" #include -#define WANT_EXEC_OBSOLETE -#include "exec-obsolete.h" +#include "exec/memory-internal.h" -unsigned memory_region_transaction_depth = 0; +//#define DEBUG_UNASSIGNED + +static unsigned memory_region_transaction_depth; +static bool memory_region_update_pending; static bool global_dirty_log = false; static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners = QTAILQ_HEAD_INITIALIZER(memory_listeners); +static QTAILQ_HEAD(, AddressSpace) address_spaces + = QTAILQ_HEAD_INITIALIZER(address_spaces); + typedef struct AddrRange AddrRange; /* @@ -97,13 +102,17 @@ static bool memory_listener_match(MemoryListener *listener, switch (_direction) { \ case Forward: \ QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ - _listener->_callback(_listener, ##_args); \ + if (_listener->_callback) { \ + _listener->_callback(_listener, ##_args); \ + } \ } \ break; \ case Reverse: \ QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ memory_listeners, link) { \ - _listener->_callback(_listener, ##_args); \ + if (_listener->_callback) { \ + _listener->_callback(_listener, ##_args); \ + } \ } \ break; \ default: \ @@ -118,7 +127,8 @@ static bool memory_listener_match(MemoryListener *listener, switch (_direction) { \ case Forward: \ QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ - if (memory_listener_match(_listener, _section)) { \ + if (_listener->_callback \ + && memory_listener_match(_listener, _section)) { \ _listener->_callback(_listener, _section, ##_args); \ } \ } \ @@ -126,7 +136,8 @@ static bool memory_listener_match(MemoryListener *listener, case Reverse: \ QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ memory_listeners, link) { \ - if (memory_listener_match(_listener, _section)) { \ + if (_listener->_callback \ + && memory_listener_match(_listener, _section)) { \ _listener->_callback(_listener, _section, ##_args); \ } \ } \ @@ -139,7 +150,7 @@ static bool memory_listener_match(MemoryListener *listener, #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback) \ MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) { \ .mr = (fr)->mr, \ - .address_space = (as)->root, \ + .address_space = (as), \ .offset_within_region = (fr)->offset_in_region, \ .size = int128_get64((fr)->addr.size), \ .offset_within_address_space = int128_get64((fr)->addr.start), \ @@ -201,10 +212,10 @@ typedef struct FlatView FlatView; /* Range of memory in the global map. Addresses are absolute. */ struct FlatRange { MemoryRegion *mr; - target_phys_addr_t offset_in_region; + hwaddr offset_in_region; AddrRange addr; uint8_t dirty_log_mask; - bool readable; + bool romd_mode; bool readonly; }; @@ -217,17 +228,8 @@ struct FlatView { unsigned nr_allocated; }; -typedef struct AddressSpace AddressSpace; typedef struct AddressSpaceOps AddressSpaceOps; -/* A system address space - I/O, memory, etc. */ -struct AddressSpace { - MemoryRegion *root; - FlatView current_map; - int ioeventfd_nb; - MemoryRegionIoeventfd *ioeventfds; -}; - #define FOR_EACH_FLAT_RANGE(var, view) \ for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) @@ -236,7 +238,7 @@ static bool flatrange_equal(FlatRange *a, FlatRange *b) return a->mr == b->mr && addrrange_equal(a->addr, b->addr) && a->offset_in_region == b->offset_in_region - && a->readable == b->readable + && a->romd_mode == b->romd_mode && a->readonly == b->readonly; } @@ -276,7 +278,7 @@ static bool can_merge(FlatRange *r1, FlatRange *r2) r1->addr.size), int128_make64(r2->offset_in_region)) && r1->dirty_log_mask == r2->dirty_log_mask - && r1->readable == r2->readable + && r1->romd_mode == r2->romd_mode && r1->readonly == r2->readonly; } @@ -300,8 +302,22 @@ static void flatview_simplify(FlatView *view) } } +static void memory_region_oldmmio_read_accessor(void *opaque, + hwaddr addr, + uint64_t *value, + unsigned size, + unsigned shift, + uint64_t mask) +{ + MemoryRegion *mr = opaque; + uint64_t tmp; + + tmp = mr->ops->old_mmio.read[ctz32(size)](mr->opaque, addr); + *value |= (tmp & mask) << shift; +} + static void memory_region_read_accessor(void *opaque, - target_phys_addr_t addr, + hwaddr addr, uint64_t *value, unsigned size, unsigned shift, @@ -317,8 +333,22 @@ static void memory_region_read_accessor(void *opaque, *value |= (tmp & mask) << shift; } +static void memory_region_oldmmio_write_accessor(void *opaque, + hwaddr addr, + uint64_t *value, + unsigned size, + unsigned shift, + uint64_t mask) +{ + MemoryRegion *mr = opaque; + uint64_t tmp; + + tmp = (*value >> shift) & mask; + mr->ops->old_mmio.write[ctz32(size)](mr->opaque, addr, tmp); +} + static void memory_region_write_accessor(void *opaque, - target_phys_addr_t addr, + hwaddr addr, uint64_t *value, unsigned size, unsigned shift, @@ -334,13 +364,13 @@ static void memory_region_write_accessor(void *opaque, mr->ops->write(mr->opaque, addr, tmp, size); } -static void access_with_adjusted_size(target_phys_addr_t addr, +static void access_with_adjusted_size(hwaddr addr, uint64_t *value, unsigned size, unsigned access_size_min, unsigned access_size_max, void (*access)(void *opaque, - target_phys_addr_t addr, + hwaddr addr, uint64_t *value, unsigned size, unsigned shift, @@ -357,16 +387,20 @@ static void access_with_adjusted_size(target_phys_addr_t addr, if (!access_size_max) { access_size_max = 4; } + + /* FIXME: support unaligned access? */ access_size = MAX(MIN(size, access_size_max), access_size_min); access_mask = -1ULL >> (64 - access_size * 8); for (i = 0; i < size; i += access_size) { - /* FIXME: big-endian support */ +#ifdef TARGET_WORDS_BIGENDIAN + access(opaque, addr + i, value, access_size, + (size - access_size - i) * 8, access_mask); +#else access(opaque, addr + i, value, access_size, i * 8, access_mask); +#endif } } -static AddressSpace address_space_memory; - static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset, unsigned width, bool write) { @@ -455,18 +489,17 @@ const IORangeOps memory_region_iorange_ops = { .destructor = memory_region_iorange_destructor, }; -static AddressSpace address_space_io; - static AddressSpace *memory_region_to_address_space(MemoryRegion *mr) { + AddressSpace *as; + while (mr->parent) { mr = mr->parent; } - if (mr == address_space_memory.root) { - return &address_space_memory; - } - if (mr == address_space_io.root) { - return &address_space_io; + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + if (mr == as->root) { + return as; + } } abort(); } @@ -482,7 +515,7 @@ static void render_memory_region(FlatView *view, { MemoryRegion *subregion; unsigned i; - target_phys_addr_t offset_in_region; + hwaddr offset_in_region; Int128 remain; Int128 now; FlatRange fr; @@ -535,7 +568,7 @@ static void render_memory_region(FlatView *view, fr.offset_in_region = offset_in_region; fr.addr = addrrange_make(base, now); fr.dirty_log_mask = mr->dirty_log_mask; - fr.readable = mr->readable; + fr.romd_mode = mr->romd_mode; fr.readonly = readonly; flatview_insert(view, i, &fr); ++i; @@ -543,19 +576,19 @@ static void render_memory_region(FlatView *view, offset_in_region += int128_get64(now); int128_subfrom(&remain, now); } - if (int128_eq(base, view->ranges[i].addr.start)) { - now = int128_min(remain, view->ranges[i].addr.size); - int128_addto(&base, now); - offset_in_region += int128_get64(now); - int128_subfrom(&remain, now); - } + now = int128_sub(int128_min(int128_add(base, remain), + addrrange_end(view->ranges[i].addr)), + base); + int128_addto(&base, now); + offset_in_region += int128_get64(now); + int128_subfrom(&remain, now); } if (int128_nz(remain)) { fr.mr = mr; fr.offset_in_region = offset_in_region; fr.addr = addrrange_make(base, remain); fr.dirty_log_mask = mr->dirty_log_mask; - fr.readable = mr->readable; + fr.romd_mode = mr->romd_mode; fr.readonly = readonly; flatview_insert(view, i, &fr); } @@ -568,8 +601,10 @@ static FlatView generate_memory_topology(MemoryRegion *mr) flatview_init(&view); - render_memory_region(&view, mr, int128_zero(), - addrrange_make(int128_zero(), int128_2_64()), false); + if (mr) { + render_memory_region(&view, mr, int128_zero(), + addrrange_make(int128_zero(), int128_2_64()), false); + } flatview_simplify(&view); return view; @@ -597,7 +632,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, fds_new[inew]))) { fd = &fds_old[iold]; section = (MemoryRegionSection) { - .address_space = as->root, + .address_space = as, .offset_within_address_space = int128_get64(fd->addr.start), .size = int128_get64(fd->addr.size), }; @@ -610,7 +645,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, fds_old[iold]))) { fd = &fds_new[inew]; section = (MemoryRegionSection) { - .address_space = as->root, + .address_space = as, .offset_within_address_space = int128_get64(fd->addr.start), .size = int128_get64(fd->addr.size), }; @@ -632,7 +667,7 @@ static void address_space_update_ioeventfds(AddressSpace *as) AddrRange tmp; unsigned i; - FOR_EACH_FLAT_RANGE(fr, &as->current_map) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { for (i = 0; i < fr->mr->ioeventfd_nb; ++i) { tmp = addrrange_shift(fr->mr->ioeventfds[i].addr, int128_sub(fr->addr.start, @@ -720,13 +755,13 @@ static void address_space_update_topology_pass(AddressSpace *as, static void address_space_update_topology(AddressSpace *as) { - FlatView old_view = as->current_map; + FlatView old_view = *as->current_map; FlatView new_view = generate_memory_topology(as->root); address_space_update_topology_pass(as, old_view, new_view, false); address_space_update_topology_pass(as, old_view, new_view, true); - as->current_map = new_view; + *as->current_map = new_view; flatview_destroy(&old_view); address_space_update_ioeventfds(as); } @@ -739,16 +774,16 @@ void memory_region_transaction_begin(void) void memory_region_transaction_commit(void) { + AddressSpace *as; + assert(memory_region_transaction_depth); --memory_region_transaction_depth; - if (!memory_region_transaction_depth) { + if (!memory_region_transaction_depth && memory_region_update_pending) { + memory_region_update_pending = false; MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); - if (address_space_memory.root) { - address_space_update_topology(&address_space_memory); - } - if (address_space_io.root) { - address_space_update_topology(&address_space_io); + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + address_space_update_topology(as); } MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); @@ -769,10 +804,6 @@ static void memory_region_destructor_ram_from_ptr(MemoryRegion *mr) qemu_ram_free_from_ptr(mr->ram_addr); } -static void memory_region_destructor_iomem(MemoryRegion *mr) -{ -} - static void memory_region_destructor_rom_device(MemoryRegion *mr) { qemu_ram_free(mr->ram_addr & TARGET_PAGE_MASK); @@ -791,7 +822,8 @@ void memory_region_init(MemoryRegion *mr, const char *name, uint64_t size) { - mr->ops = NULL; + mr->ops = &unassigned_mem_ops; + mr->opaque = NULL; mr->parent = NULL; mr->size = int128_make64(size); if (size == UINT64_MAX) { @@ -802,7 +834,7 @@ void memory_region_init(MemoryRegion *mr, mr->enabled = true; mr->terminates = false; mr->ram = false; - mr->readable = true; + mr->romd_mode = true; mr->readonly = false; mr->rom_device = false; mr->destructor = memory_region_destructor_none; @@ -819,52 +851,93 @@ void memory_region_init(MemoryRegion *mr, mr->flush_coalesced_mmio = false; } -static bool memory_region_access_valid(MemoryRegion *mr, - target_phys_addr_t addr, - unsigned size, - bool is_write) +static uint64_t unassigned_mem_read(void *opaque, hwaddr addr, + unsigned size) { - if (mr->ops->valid.accepts - && !mr->ops->valid.accepts(mr->opaque, addr, size, is_write)) { - return false; - } +#ifdef DEBUG_UNASSIGNED + printf("Unassigned mem read " TARGET_FMT_plx "\n", addr); +#endif +#if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE) + cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, size); +#endif + return 0; +} + +static void unassigned_mem_write(void *opaque, hwaddr addr, + uint64_t val, unsigned size) +{ +#ifdef DEBUG_UNASSIGNED + printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val); +#endif +#if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE) + cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, size); +#endif +} + +static bool unassigned_mem_accepts(void *opaque, hwaddr addr, + unsigned size, bool is_write) +{ + return false; +} + +const MemoryRegionOps unassigned_mem_ops = { + .valid.accepts = unassigned_mem_accepts, + .endianness = DEVICE_NATIVE_ENDIAN, +}; + +bool memory_region_access_valid(MemoryRegion *mr, + hwaddr addr, + unsigned size, + bool is_write) +{ + int access_size_min, access_size_max; + int access_size, i; if (!mr->ops->valid.unaligned && (addr & (size - 1))) { return false; } - /* Treat zero as compatibility all valid */ - if (!mr->ops->valid.max_access_size) { + if (!mr->ops->valid.accepts) { return true; } - if (size > mr->ops->valid.max_access_size - || size < mr->ops->valid.min_access_size) { - return false; + access_size_min = mr->ops->valid.min_access_size; + if (!mr->ops->valid.min_access_size) { + access_size_min = 1; + } + + access_size_max = mr->ops->valid.max_access_size; + if (!mr->ops->valid.max_access_size) { + access_size_max = 4; + } + + access_size = MAX(MIN(size, access_size_max), access_size_min); + for (i = 0; i < size; i += access_size) { + if (!mr->ops->valid.accepts(mr->opaque, addr + i, access_size, + is_write)) { + return false; + } } + return true; } static uint64_t memory_region_dispatch_read1(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size) { uint64_t data = 0; - if (!memory_region_access_valid(mr, addr, size, false)) { - return -1U; /* FIXME: better signalling */ - } - - if (!mr->ops->read) { - return mr->ops->old_mmio.read[bitops_ffsl(size)](mr->opaque, addr); + if (mr->ops->read) { + access_with_adjusted_size(addr, &data, size, + mr->ops->impl.min_access_size, + mr->ops->impl.max_access_size, + memory_region_read_accessor, mr); + } else { + access_with_adjusted_size(addr, &data, size, 1, 4, + memory_region_oldmmio_read_accessor, mr); } - /* FIXME: support unaligned access */ - access_with_adjusted_size(addr, &data, size, - mr->ops->impl.min_access_size, - mr->ops->impl.max_access_size, - memory_region_read_accessor, mr); - return data; } @@ -880,44 +953,52 @@ static void adjust_endianness(MemoryRegion *mr, uint64_t *data, unsigned size) case 4: *data = bswap32(*data); break; + case 8: + *data = bswap64(*data); + break; default: abort(); } } } -static uint64_t memory_region_dispatch_read(MemoryRegion *mr, - target_phys_addr_t addr, - unsigned size) +static bool memory_region_dispatch_read(MemoryRegion *mr, + hwaddr addr, + uint64_t *pval, + unsigned size) { - uint64_t ret; + if (!memory_region_access_valid(mr, addr, size, false)) { + *pval = unassigned_mem_read(mr, addr, size); + return true; + } - ret = memory_region_dispatch_read1(mr, addr, size); - adjust_endianness(mr, &ret, size); - return ret; + *pval = memory_region_dispatch_read1(mr, addr, size); + adjust_endianness(mr, pval, size); + return false; } -static void memory_region_dispatch_write(MemoryRegion *mr, - target_phys_addr_t addr, +static bool memory_region_dispatch_write(MemoryRegion *mr, + hwaddr addr, uint64_t data, unsigned size) { if (!memory_region_access_valid(mr, addr, size, true)) { - return; /* FIXME: better signalling */ + unassigned_mem_write(mr, addr, data, size); + return true; } adjust_endianness(mr, &data, size); - if (!mr->ops->write) { - mr->ops->old_mmio.write[bitops_ffsl(size)](mr->opaque, addr, data); - return; + if (mr->ops->write) { + access_with_adjusted_size(addr, &data, size, + mr->ops->impl.min_access_size, + mr->ops->impl.max_access_size, + memory_region_write_accessor, mr); + } else { + access_with_adjusted_size(addr, &data, size, 1, 4, + memory_region_oldmmio_write_accessor, mr); } - - /* FIXME: support unaligned access */ - access_with_adjusted_size(addr, &data, size, - mr->ops->impl.min_access_size, - mr->ops->impl.max_access_size, - memory_region_write_accessor, mr); + return false; } void memory_region_init_io(MemoryRegion *mr, @@ -930,7 +1011,6 @@ void memory_region_init_io(MemoryRegion *mr, mr->ops = ops; mr->opaque = opaque; mr->terminates = true; - mr->destructor = memory_region_destructor_iomem; mr->ram_addr = ~(ram_addr_t)0; } @@ -960,7 +1040,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr, void memory_region_init_alias(MemoryRegion *mr, const char *name, MemoryRegion *orig, - target_phys_addr_t offset, + hwaddr offset, uint64_t size) { memory_region_init(mr, name, size); @@ -983,45 +1063,17 @@ void memory_region_init_rom_device(MemoryRegion *mr, mr->ram_addr = qemu_ram_alloc(size, mr); } -static uint64_t invalid_read(void *opaque, target_phys_addr_t addr, - unsigned size) -{ - MemoryRegion *mr = opaque; - - if (!mr->warning_printed) { - fprintf(stderr, "Invalid read from memory region %s\n", mr->name); - mr->warning_printed = true; - } - return -1U; -} - -static void invalid_write(void *opaque, target_phys_addr_t addr, uint64_t data, - unsigned size) -{ - MemoryRegion *mr = opaque; - - if (!mr->warning_printed) { - fprintf(stderr, "Invalid write to memory region %s\n", mr->name); - mr->warning_printed = true; - } -} - -static const MemoryRegionOps reservation_ops = { - .read = invalid_read, - .write = invalid_write, - .endianness = DEVICE_NATIVE_ENDIAN, -}; - void memory_region_init_reservation(MemoryRegion *mr, const char *name, uint64_t size) { - memory_region_init_io(mr, &reservation_ops, mr, name, size); + memory_region_init_io(mr, &unassigned_mem_ops, mr, name, size); } void memory_region_destroy(MemoryRegion *mr) { assert(QTAILQ_EMPTY(&mr->subregions)); + assert(memory_region_transaction_depth == 0); mr->destructor(mr); memory_region_clear_coalescing(mr); g_free((char *)mr->name); @@ -1062,32 +1114,51 @@ void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) memory_region_transaction_begin(); mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask); + memory_region_update_pending |= mr->enabled; memory_region_transaction_commit(); } -bool memory_region_get_dirty(MemoryRegion *mr, target_phys_addr_t addr, - target_phys_addr_t size, unsigned client) +bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size, unsigned client) { assert(mr->terminates); return cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, 1 << client); } -void memory_region_set_dirty(MemoryRegion *mr, target_phys_addr_t addr, - target_phys_addr_t size) +void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size) { assert(mr->terminates); return cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size, -1); } +bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size, unsigned client) +{ + bool ret; + assert(mr->terminates); + ret = cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, + 1 << client); + if (ret) { + cpu_physical_memory_reset_dirty(mr->ram_addr + addr, + mr->ram_addr + addr + size, + 1 << client); + } + return ret; +} + + void memory_region_sync_dirty_bitmap(MemoryRegion *mr) { + AddressSpace *as; FlatRange *fr; - FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) { - if (fr->mr == mr) { - MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, - Forward, log_sync); + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { + if (fr->mr == mr) { + MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync); + } } } } @@ -1097,21 +1168,23 @@ void memory_region_set_readonly(MemoryRegion *mr, bool readonly) if (mr->readonly != readonly) { memory_region_transaction_begin(); mr->readonly = readonly; + memory_region_update_pending |= mr->enabled; memory_region_transaction_commit(); } } -void memory_region_rom_device_set_readable(MemoryRegion *mr, bool readable) +void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode) { - if (mr->readable != readable) { + if (mr->romd_mode != romd_mode) { memory_region_transaction_begin(); - mr->readable = readable; + mr->romd_mode = romd_mode; + memory_region_update_pending |= mr->enabled; memory_region_transaction_commit(); } } -void memory_region_reset_dirty(MemoryRegion *mr, target_phys_addr_t addr, - target_phys_addr_t size, unsigned client) +void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr, + hwaddr size, unsigned client) { assert(mr->terminates); cpu_physical_memory_reset_dirty(mr->ram_addr + addr, @@ -1130,16 +1203,24 @@ void *memory_region_get_ram_ptr(MemoryRegion *mr) return qemu_get_ram_ptr(mr->ram_addr & TARGET_PAGE_MASK); } -static void memory_region_update_coalesced_range(MemoryRegion *mr) +static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpace *as) { FlatRange *fr; CoalescedMemoryRange *cmr; AddrRange tmp; + MemoryRegionSection section; - FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { if (fr->mr == mr) { - qemu_unregister_coalesced_mmio(int128_get64(fr->addr.start), - int128_get64(fr->addr.size)); + section = (MemoryRegionSection) { + .address_space = as, + .offset_within_address_space = int128_get64(fr->addr.start), + .size = int128_get64(fr->addr.size), + }; + + MEMORY_LISTENER_CALL(coalesced_mmio_del, Reverse, §ion, + int128_get64(fr->addr.start), + int128_get64(fr->addr.size)); QTAILQ_FOREACH(cmr, &mr->coalesced, link) { tmp = addrrange_shift(cmr->addr, int128_sub(fr->addr.start, @@ -1148,13 +1229,23 @@ static void memory_region_update_coalesced_range(MemoryRegion *mr) continue; } tmp = addrrange_intersection(tmp, fr->addr); - qemu_register_coalesced_mmio(int128_get64(tmp.start), - int128_get64(tmp.size)); + MEMORY_LISTENER_CALL(coalesced_mmio_add, Forward, §ion, + int128_get64(tmp.start), + int128_get64(tmp.size)); } } } } +static void memory_region_update_coalesced_range(MemoryRegion *mr) +{ + AddressSpace *as; + + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + memory_region_update_coalesced_range_as(mr, as); + } +} + void memory_region_set_coalescing(MemoryRegion *mr) { memory_region_clear_coalescing(mr); @@ -1162,7 +1253,7 @@ void memory_region_set_coalescing(MemoryRegion *mr) } void memory_region_add_coalescing(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, uint64_t size) { CoalescedMemoryRange *cmr = g_malloc(sizeof(*cmr)); @@ -1202,7 +1293,7 @@ void memory_region_clear_flush_coalesced(MemoryRegion *mr) } void memory_region_add_eventfd(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size, bool match_data, uint64_t data, @@ -1217,6 +1308,7 @@ void memory_region_add_eventfd(MemoryRegion *mr, }; unsigned i; + adjust_endianness(mr, &mrfd.data, size); memory_region_transaction_begin(); for (i = 0; i < mr->ioeventfd_nb; ++i) { if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) { @@ -1229,11 +1321,12 @@ void memory_region_add_eventfd(MemoryRegion *mr, memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i], sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i)); mr->ioeventfds[i] = mrfd; + memory_region_update_pending |= mr->enabled; memory_region_transaction_commit(); } void memory_region_del_eventfd(MemoryRegion *mr, - target_phys_addr_t addr, + hwaddr addr, unsigned size, bool match_data, uint64_t data, @@ -1248,6 +1341,7 @@ void memory_region_del_eventfd(MemoryRegion *mr, }; unsigned i; + adjust_endianness(mr, &mrfd.data, size); memory_region_transaction_begin(); for (i = 0; i < mr->ioeventfd_nb; ++i) { if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) { @@ -1260,11 +1354,12 @@ void memory_region_del_eventfd(MemoryRegion *mr, --mr->ioeventfd_nb; mr->ioeventfds = g_realloc(mr->ioeventfds, sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1); + memory_region_update_pending |= mr->enabled; memory_region_transaction_commit(); } static void memory_region_add_subregion_common(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, MemoryRegion *subregion) { MemoryRegion *other; @@ -1278,7 +1373,7 @@ static void memory_region_add_subregion_common(MemoryRegion *mr, if (subregion->may_overlap || other->may_overlap) { continue; } - if (int128_gt(int128_make64(offset), + if (int128_ge(int128_make64(offset), int128_add(int128_make64(other->addr), other->size)) || int128_le(int128_add(int128_make64(offset), subregion->size), int128_make64(other->addr))) { @@ -1303,12 +1398,13 @@ static void memory_region_add_subregion_common(MemoryRegion *mr, } QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link); done: + memory_region_update_pending |= mr->enabled && subregion->enabled; memory_region_transaction_commit(); } void memory_region_add_subregion(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, MemoryRegion *subregion) { subregion->may_overlap = false; @@ -1317,7 +1413,7 @@ void memory_region_add_subregion(MemoryRegion *mr, } void memory_region_add_subregion_overlap(MemoryRegion *mr, - target_phys_addr_t offset, + hwaddr offset, MemoryRegion *subregion, unsigned priority) { @@ -1333,6 +1429,7 @@ void memory_region_del_subregion(MemoryRegion *mr, assert(subregion->parent == mr); subregion->parent = NULL; QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link); + memory_region_update_pending |= mr->enabled && subregion->enabled; memory_region_transaction_commit(); } @@ -1343,10 +1440,11 @@ void memory_region_set_enabled(MemoryRegion *mr, bool enabled) } memory_region_transaction_begin(); mr->enabled = enabled; + memory_region_update_pending = true; memory_region_transaction_commit(); } -void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr) +void memory_region_set_address(MemoryRegion *mr, hwaddr addr) { MemoryRegion *parent = mr->parent; unsigned priority = mr->priority; @@ -1367,7 +1465,7 @@ void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr) memory_region_transaction_commit(); } -void memory_region_set_alias_offset(MemoryRegion *mr, target_phys_addr_t offset) +void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset) { assert(mr->alias); @@ -1377,6 +1475,7 @@ void memory_region_set_alias_offset(MemoryRegion *mr, target_phys_addr_t offset) memory_region_transaction_begin(); mr->alias_offset = offset; + memory_region_update_pending |= mr->enabled; memory_region_transaction_commit(); } @@ -1400,29 +1499,39 @@ static int cmp_flatrange_addr(const void *addr_, const void *fr_) static FlatRange *address_space_lookup(AddressSpace *as, AddrRange addr) { - return bsearch(&addr, as->current_map.ranges, as->current_map.nr, + return bsearch(&addr, as->current_map->ranges, as->current_map->nr, sizeof(FlatRange), cmp_flatrange_addr); } -MemoryRegionSection memory_region_find(MemoryRegion *address_space, - target_phys_addr_t addr, uint64_t size) +MemoryRegionSection memory_region_find(MemoryRegion *mr, + hwaddr addr, uint64_t size) { - AddressSpace *as = memory_region_to_address_space(address_space); - AddrRange range = addrrange_make(int128_make64(addr), - int128_make64(size)); - FlatRange *fr = address_space_lookup(as, range); MemoryRegionSection ret = { .mr = NULL, .size = 0 }; + MemoryRegion *root; + AddressSpace *as; + AddrRange range; + FlatRange *fr; + + addr += mr->addr; + for (root = mr; root->parent; ) { + root = root->parent; + addr += root->addr; + } + as = memory_region_to_address_space(root); + range = addrrange_make(int128_make64(addr), int128_make64(size)); + fr = address_space_lookup(as, range); if (!fr) { return ret; } - while (fr > as->current_map.ranges + while (fr > as->current_map->ranges && addrrange_intersects(fr[-1].addr, range)) { --fr; } ret.mr = fr->mr; + ret.address_space = as; range = addrrange_intersection(range, fr->addr); ret.offset_within_region = fr->offset_in_region; ret.offset_within_region += int128_get64(int128_sub(range.start, @@ -1433,12 +1542,11 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space, return ret; } -void memory_global_sync_dirty_bitmap(MemoryRegion *address_space) +void address_space_sync_dirty_bitmap(AddressSpace *as) { - AddressSpace *as = memory_region_to_address_space(address_space); FlatRange *fr; - FOR_EACH_FLAT_RANGE(fr, &as->current_map) { + FOR_EACH_FLAT_RANGE(fr, as->current_map) { MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync); } } @@ -1461,29 +1569,35 @@ static void listener_add_address_space(MemoryListener *listener, FlatRange *fr; if (listener->address_space_filter - && listener->address_space_filter != as->root) { + && listener->address_space_filter != as) { return; } if (global_dirty_log) { - listener->log_global_start(listener); + if (listener->log_global_start) { + listener->log_global_start(listener); + } } - FOR_EACH_FLAT_RANGE(fr, &as->current_map) { + + FOR_EACH_FLAT_RANGE(fr, as->current_map) { MemoryRegionSection section = { .mr = fr->mr, - .address_space = as->root, + .address_space = as, .offset_within_region = fr->offset_in_region, .size = int128_get64(fr->addr.size), .offset_within_address_space = int128_get64(fr->addr.start), .readonly = fr->readonly, }; - listener->region_add(listener, §ion); + if (listener->region_add) { + listener->region_add(listener, §ion); + } } } -void memory_listener_register(MemoryListener *listener, MemoryRegion *filter) +void memory_listener_register(MemoryListener *listener, AddressSpace *filter) { MemoryListener *other = NULL; + AddressSpace *as; listener->address_space_filter = filter; if (QTAILQ_EMPTY(&memory_listeners) @@ -1498,8 +1612,10 @@ void memory_listener_register(MemoryListener *listener, MemoryRegion *filter) } QTAILQ_INSERT_BEFORE(other, listener, link); } - listener_add_address_space(listener, &address_space_memory); - listener_add_address_space(listener, &address_space_io); + + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + listener_add_address_space(listener, as); + } } void memory_listener_unregister(MemoryListener *listener) @@ -1507,29 +1623,43 @@ void memory_listener_unregister(MemoryListener *listener) QTAILQ_REMOVE(&memory_listeners, listener, link); } -void set_system_memory_map(MemoryRegion *mr) +void address_space_init(AddressSpace *as, MemoryRegion *root) { memory_region_transaction_begin(); - address_space_memory.root = mr; + as->root = root; + as->current_map = g_new(FlatView, 1); + flatview_init(as->current_map); + as->ioeventfd_nb = 0; + as->ioeventfds = NULL; + QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link); + as->name = NULL; + address_space_init_dispatch(as); + memory_region_update_pending |= root->enabled; memory_region_transaction_commit(); } -void set_system_io_map(MemoryRegion *mr) +void address_space_destroy(AddressSpace *as) { + /* Flush out anything from MemoryListeners listening in on this */ memory_region_transaction_begin(); - address_space_io.root = mr; + as->root = NULL; memory_region_transaction_commit(); + QTAILQ_REMOVE(&address_spaces, as, address_spaces_link); + address_space_destroy_dispatch(as); + flatview_destroy(as->current_map); + g_free(as->current_map); + g_free(as->ioeventfds); } -uint64_t io_mem_read(MemoryRegion *mr, target_phys_addr_t addr, unsigned size) +bool io_mem_read(MemoryRegion *mr, hwaddr addr, uint64_t *pval, unsigned size) { - return memory_region_dispatch_read(mr, addr, size); + return memory_region_dispatch_read(mr, addr, pval, size); } -void io_mem_write(MemoryRegion *mr, target_phys_addr_t addr, +bool io_mem_write(MemoryRegion *mr, hwaddr addr, uint64_t val, unsigned size) { - memory_region_dispatch_write(mr, addr, val, size); + return memory_region_dispatch_write(mr, addr, val, size); } typedef struct MemoryRegionList MemoryRegionList; @@ -1544,7 +1674,7 @@ typedef QTAILQ_HEAD(queue, MemoryRegionList) MemoryRegionListHead; static void mtree_print_mr(fprintf_function mon_printf, void *f, const MemoryRegion *mr, unsigned int level, - target_phys_addr_t base, + hwaddr base, MemoryRegionListHead *alias_print_queue) { MemoryRegionList *new_ml, *ml, *next_ml; @@ -1552,7 +1682,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, const MemoryRegion *submr; unsigned int i; - if (!mr) { + if (!mr || !mr->enabled) { return; } @@ -1582,26 +1712,26 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f, "-" TARGET_FMT_plx "\n", base + mr->addr, base + mr->addr - + (target_phys_addr_t)int128_get64(mr->size) - 1, + + (hwaddr)int128_get64(mr->size) - 1, mr->priority, - mr->readable ? 'R' : '-', - !mr->readonly && !(mr->rom_device && mr->readable) ? 'W' - : '-', + mr->romd_mode ? 'R' : '-', + !mr->readonly && !(mr->rom_device && mr->romd_mode) ? 'W' + : '-', mr->name, mr->alias->name, mr->alias_offset, mr->alias_offset - + (target_phys_addr_t)int128_get64(mr->size) - 1); + + (hwaddr)int128_get64(mr->size) - 1); } else { mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %c%c): %s\n", base + mr->addr, base + mr->addr - + (target_phys_addr_t)int128_get64(mr->size) - 1, + + (hwaddr)int128_get64(mr->size) - 1, mr->priority, - mr->readable ? 'R' : '-', - !mr->readonly && !(mr->rom_device && mr->readable) ? 'W' - : '-', + mr->romd_mode ? 'R' : '-', + !mr->readonly && !(mr->rom_device && mr->romd_mode) ? 'W' + : '-', mr->name); } @@ -1638,16 +1768,16 @@ void mtree_info(fprintf_function mon_printf, void *f) { MemoryRegionListHead ml_head; MemoryRegionList *ml, *ml2; + AddressSpace *as; QTAILQ_INIT(&ml_head); - mon_printf(f, "memory\n"); - mtree_print_mr(mon_printf, f, address_space_memory.root, 0, 0, &ml_head); - - if (address_space_io.root && - !QTAILQ_EMPTY(&address_space_io.root->subregions)) { - mon_printf(f, "I/O\n"); - mtree_print_mr(mon_printf, f, address_space_io.root, 0, 0, &ml_head); + QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) { + if (!as->name) { + continue; + } + mon_printf(f, "%s\n", as->name); + mtree_print_mr(mon_printf, f, as->root, 0, 0, &ml_head); } mon_printf(f, "aliases\n");