X-Git-Url: https://repo.jachan.dev/qemu.git/blobdiff_plain/21a21b853a1bb606358af61e738abfb9aecbd720..2212092e11db4c92416f989db9000737daaf9d37:/exec.c diff --git a/exec.c b/exec.c index 50e3ee237c..47835c1dc1 100644 --- a/exec.c +++ b/exec.c @@ -93,6 +93,11 @@ static MemoryRegion io_mem_unassigned; #endif +#ifdef TARGET_PAGE_BITS_VARY +int target_page_bits; +bool target_page_bits_decided; +#endif + struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus); /* current CPU in the current thread. It is only valid inside cpu_exec() */ @@ -102,8 +107,37 @@ __thread CPUState *current_cpu; 2 = Adaptive rate instruction counting. */ int use_icount; +bool set_preferred_target_page_bits(int bits) +{ + /* The target page size is the lowest common denominator for all + * the CPUs in the system, so we can only make it smaller, never + * larger. And we can't make it smaller once we've committed to + * a particular size. + */ +#ifdef TARGET_PAGE_BITS_VARY + assert(bits >= TARGET_PAGE_BITS_MIN); + if (target_page_bits == 0 || target_page_bits > bits) { + if (target_page_bits_decided) { + return false; + } + target_page_bits = bits; + } +#endif + return true; +} + #if !defined(CONFIG_USER_ONLY) +static void finalize_target_page_bits(void) +{ +#ifdef TARGET_PAGE_BITS_VARY + if (target_page_bits == 0) { + target_page_bits = TARGET_PAGE_BITS_MIN; + } + target_page_bits_decided = true; +#endif +} + typedef struct PhysPageEntry PhysPageEntry; struct PhysPageEntry { @@ -153,7 +187,7 @@ typedef struct subpage_t { MemoryRegion iomem; AddressSpace *as; hwaddr base; - uint16_t sub_section[TARGET_PAGE_SIZE]; + uint16_t sub_section[]; } subpage_t; #define PHYS_SECTION_UNASSIGNED 0 @@ -255,7 +289,7 @@ static void phys_page_set(AddressSpaceDispatch *d, /* Compact a non leaf page entry. Simply detect that the entry has a single child, * and update our entry so we can skip it and go directly to the destination. */ -static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted) +static void phys_page_compact(PhysPageEntry *lp, Node *nodes) { unsigned valid_ptr = P_L2_SIZE; int valid = 0; @@ -275,7 +309,7 @@ static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *com valid_ptr = i; valid++; if (p[i].skip) { - phys_page_compact(&p[i], nodes, compacted); + phys_page_compact(&p[i], nodes); } } @@ -307,10 +341,8 @@ static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *com static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb) { - DECLARE_BITMAP(compacted, nodes_nb); - if (d->phys_map.skip) { - phys_page_compact(&d->phys_map, d->map.nodes, compacted); + phys_page_compact(&d->phys_map, d->map.nodes); } } @@ -320,9 +352,9 @@ static inline bool section_covers_addr(const MemoryRegionSection *section, /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means * the section must cover the entire address space. */ - return section->size.hi || + return int128_gethi(section->size) || range_covers_byte(section->offset_within_address_space, - section->size.lo, addr); + int128_getlo(section->size), addr); } static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr, @@ -416,6 +448,39 @@ address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *x return section; } +/* Called from RCU critical section */ +IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr, + bool is_write) +{ + IOMMUTLBEntry iotlb = {0}; + MemoryRegionSection *section; + MemoryRegion *mr; + + for (;;) { + AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch); + section = address_space_lookup_region(d, addr, false); + addr = addr - section->offset_within_address_space + + section->offset_within_region; + mr = section->mr; + + if (!mr->iommu_ops) { + break; + } + + iotlb = mr->iommu_ops->translate(mr, addr, is_write); + if (!(iotlb.perm & (1 << is_write))) { + iotlb.target_as = NULL; + break; + } + + addr = ((iotlb.translated_addr & ~iotlb.addr_mask) + | (addr & iotlb.addr_mask)); + as = iotlb.target_as; + } + + return iotlb; +} + /* Called from RCU critical section */ MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr, hwaddr *xlat, hwaddr *plen, @@ -461,7 +526,7 @@ address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr, hwaddr *xlat, hwaddr *plen) { MemoryRegionSection *section; - AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch; + AddressSpaceDispatch *d = atomic_rcu_read(&cpu->cpu_ases[asidx].memory_dispatch); section = address_space_translate_internal(d, addr, xlat, plen, false); @@ -598,32 +663,11 @@ AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx) } #endif -static int cpu_get_free_index(void) -{ - CPUState *some_cpu; - int cpu_index = 0; - - CPU_FOREACH(some_cpu) { - cpu_index++; - } - return cpu_index; -} - -void cpu_exec_exit(CPUState *cpu) +void cpu_exec_unrealizefn(CPUState *cpu) { CPUClass *cc = CPU_GET_CLASS(cpu); - cpu_list_lock(); - if (cpu->node.tqe_prev == NULL) { - /* there is nothing to undo since cpu_exec_init() hasn't been called */ - cpu_list_unlock(); - return; - } - - QTAILQ_REMOVE(&cpus, cpu, node); - cpu->node.tqe_prev = NULL; - cpu->cpu_index = UNASSIGNED_CPU_INDEX; - cpu_list_unlock(); + cpu_list_remove(cpu); if (cc->vmsd != NULL) { vmstate_unregister(NULL, cc->vmsd, cpu); @@ -633,11 +677,8 @@ void cpu_exec_exit(CPUState *cpu) } } -void cpu_exec_init(CPUState *cpu, Error **errp) +void cpu_exec_initfn(CPUState *cpu) { - CPUClass *cc ATTRIBUTE_UNUSED = CPU_GET_CLASS(cpu); - Error *local_err ATTRIBUTE_UNUSED = NULL; - cpu->as = NULL; cpu->num_ases = 0; @@ -658,14 +699,13 @@ void cpu_exec_init(CPUState *cpu, Error **errp) cpu->memory = system_memory; object_ref(OBJECT(cpu->memory)); #endif +} - cpu_list_lock(); - if (cpu->cpu_index == UNASSIGNED_CPU_INDEX) { - cpu->cpu_index = cpu_get_free_index(); - assert(cpu->cpu_index != UNASSIGNED_CPU_INDEX); - } - QTAILQ_INSERT_TAIL(&cpus, cpu, node); - cpu_list_unlock(); +void cpu_exec_realizefn(CPUState *cpu, Error **errp) +{ + CPUClass *cc ATTRIBUTE_UNUSED = CPU_GET_CLASS(cpu); + + cpu_list_add(cpu); #ifndef CONFIG_USER_ONLY if (qdev_get_vmsd(DEVICE(cpu)) == NULL) { @@ -677,23 +717,15 @@ void cpu_exec_init(CPUState *cpu, Error **errp) #endif } -#if defined(CONFIG_USER_ONLY) -static void breakpoint_invalidate(CPUState *cpu, target_ulong pc) -{ - tb_invalidate_phys_page_range(pc, pc + 1, 0); -} -#else static void breakpoint_invalidate(CPUState *cpu, target_ulong pc) { - MemTxAttrs attrs; - hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs); - int asidx = cpu_asidx_from_attrs(cpu, attrs); - if (phys != -1) { - tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as, - phys | (pc & ~TARGET_PAGE_MASK)); - } + /* Flush the whole TB as this will not have race conditions + * even if we don't have proper locking yet. + * Ideally we would just invalidate the TBs for the + * specified PC. + */ + tb_flush(cpu); } -#endif #if defined(CONFIG_USER_ONLY) void cpu_watchpoint_remove_all(CPUState *cpu, int mask) @@ -899,11 +931,13 @@ void cpu_abort(CPUState *cpu, const char *fmt, ...) fprintf(stderr, "\n"); cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP); if (qemu_log_separate()) { + qemu_log_lock(); qemu_log("qemu: fatal: "); qemu_log_vprintf(fmt, ap2); qemu_log("\n"); log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP); qemu_log_flush(); + qemu_log_unlock(); qemu_log_close(); } va_end(ap2); @@ -1217,6 +1251,15 @@ void qemu_mutex_unlock_ramlist(void) } #ifdef __linux__ +static int64_t get_file_size(int fd) +{ + int64_t size = lseek(fd, 0, SEEK_END); + if (size < 0) { + return -errno; + } + return size; +} + static void *file_ram_alloc(RAMBlock *block, ram_addr_t memory, const char *path, @@ -1226,9 +1269,9 @@ static void *file_ram_alloc(RAMBlock *block, char *filename; char *sanitized_name; char *c; - void *area; + void *area = MAP_FAILED; int fd = -1; - int64_t page_size; + int64_t file_size; if (kvm_enabled() && !kvm_has_sync_mmu()) { error_setg(errp, @@ -1283,25 +1326,47 @@ static void *file_ram_alloc(RAMBlock *block, */ } - page_size = qemu_fd_getpagesize(fd); - block->mr->align = MAX(page_size, QEMU_VMALLOC_ALIGN); + block->page_size = qemu_fd_getpagesize(fd); + block->mr->align = block->page_size; +#if defined(__s390x__) + if (kvm_enabled()) { + block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN); + } +#endif + + file_size = get_file_size(fd); - if (memory < page_size) { + if (memory < block->page_size) { error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to " - "or larger than page size 0x%" PRIx64, - memory, page_size); + "or larger than page size 0x%zx", + memory, block->page_size); + goto error; + } + + if (file_size > 0 && file_size < memory) { + error_setg(errp, "backing store %s size 0x%" PRIx64 + " does not match 'size' option 0x" RAM_ADDR_FMT, + path, file_size, memory); goto error; } - memory = ROUND_UP(memory, page_size); + memory = ROUND_UP(memory, block->page_size); /* * ftruncate is not supported by hugetlbfs in older * hosts, so don't bother bailing out on errors. * If anything goes wrong with it under other filesystems, * mmap will fail. + * + * Do not truncate the non-empty backend file to avoid corrupting + * the existing data in the file. Disabling shrinking is not + * enough. For example, the current vNVDIMM implementation stores + * the guest NVDIMM labels at the end of the backend file. If the + * backend file is later extended, QEMU will not be able to find + * those labels. Therefore, extending the non-empty backend file + * is disabled as well. */ - if (ftruncate(fd, memory)) { + if (!file_size && ftruncate(fd, memory)) { perror("ftruncate"); } @@ -1314,13 +1379,19 @@ static void *file_ram_alloc(RAMBlock *block, } if (mem_prealloc) { - os_mem_prealloc(fd, area, memory); + os_mem_prealloc(fd, area, memory, errp); + if (errp && *errp) { + goto error; + } } block->fd = fd; return area; error: + if (area != MAP_FAILED) { + qemu_ram_munmap(area, memory); + } if (unlink_on_error) { unlink(path); } @@ -1442,6 +1513,11 @@ void qemu_ram_unset_idstr(RAMBlock *block) } } +size_t qemu_ram_pagesize(RAMBlock *rb) +{ + return rb->page_size; +} + static int memory_try_enable_merging(void *addr, size_t len) { if (!machine_mem_merge(current_machine)) { @@ -1609,10 +1685,8 @@ static void ram_block_add(RAMBlock *new_block, Error **errp) if (new_block->host) { qemu_ram_setup_dump(new_block->host, new_block->max_length); qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE); + /* MADV_DONTFORK is also needed by KVM in absence of synchronous MMU */ qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK); - if (kvm_enabled()) { - kvm_setup_guest_memory(new_block->host, new_block->max_length); - } } } @@ -1683,6 +1757,7 @@ RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size, new_block->max_length = max_size; assert(max_size >= size); new_block->fd = -1; + new_block->page_size = getpagesize(); new_block->host = host; if (host) { new_block->flags |= RAM_PREALLOC; @@ -1967,7 +2042,11 @@ ram_addr_t qemu_ram_addr_from_host(void *ptr) static void notdirty_mem_write(void *opaque, hwaddr ram_addr, uint64_t val, unsigned size) { + bool locked = false; + if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) { + locked = true; + tb_lock(); tb_invalidate_phys_page_fast(ram_addr, size); } switch (size) { @@ -1983,6 +2062,11 @@ static void notdirty_mem_write(void *opaque, hwaddr ram_addr, default: abort(); } + + if (locked) { + tb_unlock(); + } + /* Set both VGA and migration bits for simplicity and to remove * the notdirty callback faster. */ @@ -2043,6 +2127,12 @@ static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags) continue; } cpu->watchpoint_hit = wp; + + /* The tb_lock will be reset when cpu_loop_exit or + * cpu_loop_exit_noexc longjmp back into the cpu_exec + * main loop. + */ + tb_lock(); tb_check_watchpoint(cpu); if (wp->flags & BP_STOP_BEFORE_ACCESS) { cpu->exception_index = EXCP_DEBUG; @@ -2230,8 +2320,7 @@ static subpage_t *subpage_init(AddressSpace *as, hwaddr base) { subpage_t *mmio; - mmio = g_malloc0(sizeof(subpage_t)); - + mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t)); mmio->as = as; mmio->base = base; memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio, @@ -2336,7 +2425,7 @@ static void tcg_commit(MemoryListener *listener) * may have split the RCU critical section. */ d = atomic_rcu_read(&cpuas->as->dispatch); - cpuas->memory_dispatch = d; + atomic_rcu_set(&cpuas->memory_dispatch, d); tlb_flush(cpuas->cpu, 1); } @@ -2451,7 +2540,9 @@ static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask); } if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { + tb_lock(); tb_invalidate_phys_range(addr, addr + length); + tb_unlock(); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); } cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); @@ -2823,6 +2914,14 @@ void cpu_register_map_client(QEMUBH *bh) void cpu_exec_init_all(void) { qemu_mutex_init(&ram_list.mutex); + /* The data structures we set up here depend on knowing the page size, + * so no more changes can be made after this point. + * In an ideal world, nothing we did before we had finished the + * machine setup would care about the target page size, and we could + * do this much later, rather than requiring board models to state + * up front what their requirements are. + */ + finalize_target_page_bits(); io_mem_init(); memory_map_init(); qemu_mutex_init(&map_client_list_lock); @@ -2872,6 +2971,31 @@ bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_ return true; } +static hwaddr +address_space_extend_translation(AddressSpace *as, hwaddr addr, hwaddr target_len, + MemoryRegion *mr, hwaddr base, hwaddr len, + bool is_write) +{ + hwaddr done = 0; + hwaddr xlat; + MemoryRegion *this_mr; + + for (;;) { + target_len -= len; + addr += len; + done += len; + if (target_len == 0) { + return done; + } + + len = target_len; + this_mr = address_space_translate(as, addr, &xlat, &len, is_write); + if (this_mr != mr || xlat != base + done) { + return done; + } + } +} + /* Map a physical memory region into a host virtual address. * May map a subset of the requested range, given by and returned in *plen. * May return NULL if resources needed to perform the mapping are exhausted. @@ -2885,9 +3009,8 @@ void *address_space_map(AddressSpace *as, bool is_write) { hwaddr len = *plen; - hwaddr done = 0; - hwaddr l, xlat, base; - MemoryRegion *mr, *this_mr; + hwaddr l, xlat; + MemoryRegion *mr; void *ptr; if (len == 0) { @@ -2921,26 +3044,10 @@ void *address_space_map(AddressSpace *as, return bounce.buffer; } - base = xlat; - - for (;;) { - len -= l; - addr += l; - done += l; - if (len == 0) { - break; - } - - l = len; - this_mr = address_space_translate(as, addr, &xlat, &l, is_write); - if (this_mr != mr || xlat != base + done) { - break; - } - } memory_region_ref(mr); - *plen = done; - ptr = qemu_ram_ptr_length(mr->ram_block, base, plen); + *plen = address_space_extend_translation(as, addr, len, mr, xlat, l, is_write); + ptr = qemu_ram_ptr_length(mr->ram_block, xlat, plen); rcu_read_unlock(); return ptr; @@ -2992,597 +3099,92 @@ void cpu_physical_memory_unmap(void *buffer, hwaddr len, return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len); } -/* warning: addr must be aligned */ -static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, - MemTxResult *result, - enum device_endian endian) -{ - uint8_t *ptr; - uint64_t val; - MemoryRegion *mr; - hwaddr l = 4; - hwaddr addr1; - MemTxResult r; - bool release_lock = false; - - rcu_read_lock(); - mr = address_space_translate(as, addr, &addr1, &l, false); - if (l < 4 || !memory_access_is_direct(mr, false)) { - release_lock |= prepare_mmio_access(mr); - - /* I/O case */ - r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs); -#if defined(TARGET_WORDS_BIGENDIAN) - if (endian == DEVICE_LITTLE_ENDIAN) { - val = bswap32(val); - } -#else - if (endian == DEVICE_BIG_ENDIAN) { - val = bswap32(val); - } -#endif - } else { - /* RAM case */ - ptr = qemu_map_ram_ptr(mr->ram_block, addr1); - switch (endian) { - case DEVICE_LITTLE_ENDIAN: - val = ldl_le_p(ptr); - break; - case DEVICE_BIG_ENDIAN: - val = ldl_be_p(ptr); - break; - default: - val = ldl_p(ptr); - break; - } - r = MEMTX_OK; - } - if (result) { - *result = r; - } - if (release_lock) { - qemu_mutex_unlock_iothread(); - } - rcu_read_unlock(); - return val; -} - -uint32_t address_space_ldl(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, MemTxResult *result) -{ - return address_space_ldl_internal(as, addr, attrs, result, - DEVICE_NATIVE_ENDIAN); -} - -uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, MemTxResult *result) -{ - return address_space_ldl_internal(as, addr, attrs, result, - DEVICE_LITTLE_ENDIAN); -} - -uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, MemTxResult *result) -{ - return address_space_ldl_internal(as, addr, attrs, result, - DEVICE_BIG_ENDIAN); -} - -uint32_t ldl_phys(AddressSpace *as, hwaddr addr) -{ - return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL); -} - -uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr) -{ - return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL); -} - -uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr) -{ - return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL); -} - -/* warning: addr must be aligned */ -static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, - MemTxResult *result, - enum device_endian endian) -{ - uint8_t *ptr; - uint64_t val; - MemoryRegion *mr; - hwaddr l = 8; - hwaddr addr1; - MemTxResult r; - bool release_lock = false; - - rcu_read_lock(); - mr = address_space_translate(as, addr, &addr1, &l, - false); - if (l < 8 || !memory_access_is_direct(mr, false)) { - release_lock |= prepare_mmio_access(mr); - - /* I/O case */ - r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs); -#if defined(TARGET_WORDS_BIGENDIAN) - if (endian == DEVICE_LITTLE_ENDIAN) { - val = bswap64(val); - } -#else - if (endian == DEVICE_BIG_ENDIAN) { - val = bswap64(val); - } -#endif - } else { - /* RAM case */ - ptr = qemu_map_ram_ptr(mr->ram_block, addr1); - switch (endian) { - case DEVICE_LITTLE_ENDIAN: - val = ldq_le_p(ptr); - break; - case DEVICE_BIG_ENDIAN: - val = ldq_be_p(ptr); - break; - default: - val = ldq_p(ptr); - break; - } - r = MEMTX_OK; - } - if (result) { - *result = r; - } - if (release_lock) { - qemu_mutex_unlock_iothread(); - } - rcu_read_unlock(); - return val; -} - -uint64_t address_space_ldq(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, MemTxResult *result) -{ - return address_space_ldq_internal(as, addr, attrs, result, - DEVICE_NATIVE_ENDIAN); -} - -uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, MemTxResult *result) -{ - return address_space_ldq_internal(as, addr, attrs, result, - DEVICE_LITTLE_ENDIAN); -} - -uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, MemTxResult *result) -{ - return address_space_ldq_internal(as, addr, attrs, result, - DEVICE_BIG_ENDIAN); -} - -uint64_t ldq_phys(AddressSpace *as, hwaddr addr) -{ - return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL); -} - -uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr) -{ - return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL); -} - -uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr) -{ - return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL); -} - -/* XXX: optimize */ -uint32_t address_space_ldub(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, MemTxResult *result) -{ - uint8_t val; - MemTxResult r; - - r = address_space_rw(as, addr, attrs, &val, 1, 0); - if (result) { - *result = r; - } - return val; -} - -uint32_t ldub_phys(AddressSpace *as, hwaddr addr) -{ - return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL); -} - -/* warning: addr must be aligned */ -static inline uint32_t address_space_lduw_internal(AddressSpace *as, - hwaddr addr, - MemTxAttrs attrs, - MemTxResult *result, - enum device_endian endian) -{ - uint8_t *ptr; - uint64_t val; - MemoryRegion *mr; - hwaddr l = 2; - hwaddr addr1; - MemTxResult r; - bool release_lock = false; - - rcu_read_lock(); - mr = address_space_translate(as, addr, &addr1, &l, - false); - if (l < 2 || !memory_access_is_direct(mr, false)) { - release_lock |= prepare_mmio_access(mr); - - /* I/O case */ - r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs); -#if defined(TARGET_WORDS_BIGENDIAN) - if (endian == DEVICE_LITTLE_ENDIAN) { - val = bswap16(val); - } -#else - if (endian == DEVICE_BIG_ENDIAN) { - val = bswap16(val); - } -#endif - } else { - /* RAM case */ - ptr = qemu_map_ram_ptr(mr->ram_block, addr1); - switch (endian) { - case DEVICE_LITTLE_ENDIAN: - val = lduw_le_p(ptr); - break; - case DEVICE_BIG_ENDIAN: - val = lduw_be_p(ptr); - break; - default: - val = lduw_p(ptr); - break; - } - r = MEMTX_OK; - } - if (result) { - *result = r; - } - if (release_lock) { - qemu_mutex_unlock_iothread(); - } - rcu_read_unlock(); - return val; -} - -uint32_t address_space_lduw(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, MemTxResult *result) -{ - return address_space_lduw_internal(as, addr, attrs, result, - DEVICE_NATIVE_ENDIAN); -} - -uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, MemTxResult *result) -{ - return address_space_lduw_internal(as, addr, attrs, result, - DEVICE_LITTLE_ENDIAN); -} - -uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr, - MemTxAttrs attrs, MemTxResult *result) -{ - return address_space_lduw_internal(as, addr, attrs, result, - DEVICE_BIG_ENDIAN); -} - -uint32_t lduw_phys(AddressSpace *as, hwaddr addr) -{ - return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL); -} - -uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr) -{ - return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL); -} - -uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr) -{ - return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL); -} - -/* warning: addr must be aligned. The ram page is not masked as dirty - and the code inside is not invalidated. It is useful if the dirty - bits are used to track modified PTEs */ -void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val, - MemTxAttrs attrs, MemTxResult *result) -{ - uint8_t *ptr; - MemoryRegion *mr; - hwaddr l = 4; - hwaddr addr1; - MemTxResult r; - uint8_t dirty_log_mask; - bool release_lock = false; - - rcu_read_lock(); - mr = address_space_translate(as, addr, &addr1, &l, - true); - if (l < 4 || !memory_access_is_direct(mr, true)) { - release_lock |= prepare_mmio_access(mr); - - r = memory_region_dispatch_write(mr, addr1, val, 4, attrs); - } else { - ptr = qemu_map_ram_ptr(mr->ram_block, addr1); - stl_p(ptr, val); - - dirty_log_mask = memory_region_get_dirty_log_mask(mr); - dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); - cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, - 4, dirty_log_mask); - r = MEMTX_OK; - } - if (result) { - *result = r; - } - if (release_lock) { - qemu_mutex_unlock_iothread(); - } - rcu_read_unlock(); -} - -void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val) -{ - address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); -} - -/* warning: addr must be aligned */ -static inline void address_space_stl_internal(AddressSpace *as, - hwaddr addr, uint32_t val, - MemTxAttrs attrs, - MemTxResult *result, - enum device_endian endian) -{ - uint8_t *ptr; - MemoryRegion *mr; - hwaddr l = 4; - hwaddr addr1; - MemTxResult r; - bool release_lock = false; - - rcu_read_lock(); - mr = address_space_translate(as, addr, &addr1, &l, - true); - if (l < 4 || !memory_access_is_direct(mr, true)) { - release_lock |= prepare_mmio_access(mr); - -#if defined(TARGET_WORDS_BIGENDIAN) - if (endian == DEVICE_LITTLE_ENDIAN) { - val = bswap32(val); - } -#else - if (endian == DEVICE_BIG_ENDIAN) { - val = bswap32(val); - } -#endif - r = memory_region_dispatch_write(mr, addr1, val, 4, attrs); - } else { - /* RAM case */ - ptr = qemu_map_ram_ptr(mr->ram_block, addr1); - switch (endian) { - case DEVICE_LITTLE_ENDIAN: - stl_le_p(ptr, val); - break; - case DEVICE_BIG_ENDIAN: - stl_be_p(ptr, val); - break; - default: - stl_p(ptr, val); - break; - } - invalidate_and_set_dirty(mr, addr1, 4); - r = MEMTX_OK; - } - if (result) { - *result = r; - } - if (release_lock) { - qemu_mutex_unlock_iothread(); - } - rcu_read_unlock(); -} - -void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val, - MemTxAttrs attrs, MemTxResult *result) -{ - address_space_stl_internal(as, addr, val, attrs, result, - DEVICE_NATIVE_ENDIAN); -} - -void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val, - MemTxAttrs attrs, MemTxResult *result) -{ - address_space_stl_internal(as, addr, val, attrs, result, - DEVICE_LITTLE_ENDIAN); -} +#define ARG1_DECL AddressSpace *as +#define ARG1 as +#define SUFFIX +#define TRANSLATE(...) address_space_translate(as, __VA_ARGS__) +#define IS_DIRECT(mr, is_write) memory_access_is_direct(mr, is_write) +#define MAP_RAM(mr, ofs) qemu_map_ram_ptr((mr)->ram_block, ofs) +#define INVALIDATE(mr, ofs, len) invalidate_and_set_dirty(mr, ofs, len) +#define RCU_READ_LOCK(...) rcu_read_lock() +#define RCU_READ_UNLOCK(...) rcu_read_unlock() +#include "memory_ldst.inc.c" -void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val, - MemTxAttrs attrs, MemTxResult *result) +int64_t address_space_cache_init(MemoryRegionCache *cache, + AddressSpace *as, + hwaddr addr, + hwaddr len, + bool is_write) { - address_space_stl_internal(as, addr, val, attrs, result, - DEVICE_BIG_ENDIAN); -} - -void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val) -{ - address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); -} - -void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val) -{ - address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); -} - -void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val) -{ - address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); -} - -/* XXX: optimize */ -void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val, - MemTxAttrs attrs, MemTxResult *result) -{ - uint8_t v = val; - MemTxResult r; - - r = address_space_rw(as, addr, attrs, &v, 1, 1); - if (result) { - *result = r; - } -} - -void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val) -{ - address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); -} - -/* warning: addr must be aligned */ -static inline void address_space_stw_internal(AddressSpace *as, - hwaddr addr, uint32_t val, - MemTxAttrs attrs, - MemTxResult *result, - enum device_endian endian) -{ - uint8_t *ptr; + hwaddr l, xlat; MemoryRegion *mr; - hwaddr l = 2; - hwaddr addr1; - MemTxResult r; - bool release_lock = false; + void *ptr; - rcu_read_lock(); - mr = address_space_translate(as, addr, &addr1, &l, true); - if (l < 2 || !memory_access_is_direct(mr, true)) { - release_lock |= prepare_mmio_access(mr); + assert(len > 0); -#if defined(TARGET_WORDS_BIGENDIAN) - if (endian == DEVICE_LITTLE_ENDIAN) { - val = bswap16(val); - } -#else - if (endian == DEVICE_BIG_ENDIAN) { - val = bswap16(val); - } -#endif - r = memory_region_dispatch_write(mr, addr1, val, 2, attrs); - } else { - /* RAM case */ - ptr = qemu_map_ram_ptr(mr->ram_block, addr1); - switch (endian) { - case DEVICE_LITTLE_ENDIAN: - stw_le_p(ptr, val); - break; - case DEVICE_BIG_ENDIAN: - stw_be_p(ptr, val); - break; - default: - stw_p(ptr, val); - break; - } - invalidate_and_set_dirty(mr, addr1, 2); - r = MEMTX_OK; - } - if (result) { - *result = r; - } - if (release_lock) { - qemu_mutex_unlock_iothread(); + l = len; + mr = address_space_translate(as, addr, &xlat, &l, is_write); + if (!memory_access_is_direct(mr, is_write)) { + return -EINVAL; } - rcu_read_unlock(); -} -void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val, - MemTxAttrs attrs, MemTxResult *result) -{ - address_space_stw_internal(as, addr, val, attrs, result, - DEVICE_NATIVE_ENDIAN); -} + l = address_space_extend_translation(as, addr, len, mr, xlat, l, is_write); + ptr = qemu_ram_ptr_length(mr->ram_block, xlat, &l); -void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val, - MemTxAttrs attrs, MemTxResult *result) -{ - address_space_stw_internal(as, addr, val, attrs, result, - DEVICE_LITTLE_ENDIAN); -} + cache->xlat = xlat; + cache->is_write = is_write; + cache->mr = mr; + cache->ptr = ptr; + cache->len = l; + memory_region_ref(cache->mr); -void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val, - MemTxAttrs attrs, MemTxResult *result) -{ - address_space_stw_internal(as, addr, val, attrs, result, - DEVICE_BIG_ENDIAN); -} - -void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val) -{ - address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); -} - -void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val) -{ - address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); + return l; } -void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val) +void address_space_cache_invalidate(MemoryRegionCache *cache, + hwaddr addr, + hwaddr access_len) { - address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); + assert(cache->is_write); + invalidate_and_set_dirty(cache->mr, addr + cache->xlat, access_len); } -/* XXX: optimize */ -void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val, - MemTxAttrs attrs, MemTxResult *result) +void address_space_cache_destroy(MemoryRegionCache *cache) { - MemTxResult r; - val = tswap64(val); - r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1); - if (result) { - *result = r; + if (!cache->mr) { + return; } -} -void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val, - MemTxAttrs attrs, MemTxResult *result) -{ - MemTxResult r; - val = cpu_to_le64(val); - r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1); - if (result) { - *result = r; - } -} -void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val, - MemTxAttrs attrs, MemTxResult *result) -{ - MemTxResult r; - val = cpu_to_be64(val); - r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1); - if (result) { - *result = r; + if (xen_enabled()) { + xen_invalidate_map_cache_entry(cache->ptr); } + memory_region_unref(cache->mr); } -void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val) -{ - address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); -} - -void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val) -{ - address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); -} - -void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val) -{ - address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL); -} +/* Called from RCU critical section. This function has the same + * semantics as address_space_translate, but it only works on a + * predefined range of a MemoryRegion that was mapped with + * address_space_cache_init. + */ +static inline MemoryRegion *address_space_translate_cached( + MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat, + hwaddr *plen, bool is_write) +{ + assert(addr < cache->len && *plen <= cache->len - addr); + *xlat = addr + cache->xlat; + return cache->mr; +} + +#define ARG1_DECL MemoryRegionCache *cache +#define ARG1 cache +#define SUFFIX _cached +#define TRANSLATE(...) address_space_translate_cached(cache, __VA_ARGS__) +#define IS_DIRECT(mr, is_write) true +#define MAP_RAM(mr, ofs) (cache->ptr + (ofs - cache->xlat)) +#define INVALIDATE(mr, ofs, len) ((void)0) +#define RCU_READ_LOCK() ((void)0) +#define RCU_READ_UNLOCK() ((void)0) +#include "memory_ldst.inc.c" /* virtual memory access for debug (includes writing to ROM) */ int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,