exec.c

   1 /*
   2  *  Virtual page mapping
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "config.h"
  20 #ifdef _WIN32
  21 #include <windows.h>
  22 #else
  23 #include <sys/types.h>
  24 #include <sys/mman.h>
  25 #endif
  26
  27 #include "qemu-common.h"
  28 #include "cpu.h"
  29 #include "tcg.h"
  30 #include "hw/hw.h"
  31 #include "hw/qdev.h"
  32 #include "qemu/osdep.h"
  33 #include "sysemu/kvm.h"
  34 #include "hw/xen.h"
  35 #include "qemu/timer.h"
  36 #include "qemu/config-file.h"
  37 #include "exec/memory.h"
  38 #include "sysemu/dma.h"
  39 #include "exec/address-spaces.h"
  40 #if defined(CONFIG_USER_ONLY)
  41 #include <qemu.h>
  42 #else /* !CONFIG_USER_ONLY */
  43 #include "sysemu/xen-mapcache.h"
  44 #include "trace.h"
  45 #endif
  46 #include "exec/cpu-all.h"
  47
  48 #include "exec/cputlb.h"
  49 #include "translate-all.h"
  50
  51 #include "exec/memory-internal.h"
  52
  53 //#define DEBUG_UNASSIGNED
  54 //#define DEBUG_SUBPAGE
  55
  56 #if !defined(CONFIG_USER_ONLY)
  57 int phys_ram_fd;
  58 static int in_migration;
  59
  60 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
  61
  62 static MemoryRegion *system_memory;
  63 static MemoryRegion *system_io;
  64
  65 AddressSpace address_space_io;
  66 AddressSpace address_space_memory;
  67 DMAContext dma_context_memory;
  68
  69 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
  70 static MemoryRegion io_mem_subpage_ram;
  71
  72 #endif
  73
  74 CPUArchState *first_cpu;
  75 /* current CPU in the current thread. It is only valid inside
  76    cpu_exec() */
  77 DEFINE_TLS(CPUArchState *,cpu_single_env);
  78 /* 0 = Do not count executed instructions.
  79    1 = Precise instruction counting.
  80    2 = Adaptive rate instruction counting.  */
  81 int use_icount;
  82
  83 #if !defined(CONFIG_USER_ONLY)
  84
  85 static MemoryRegionSection *phys_sections;
  86 static unsigned phys_sections_nb, phys_sections_nb_alloc;
  87 static uint16_t phys_section_unassigned;
  88 static uint16_t phys_section_notdirty;
  89 static uint16_t phys_section_rom;
  90 static uint16_t phys_section_watch;
  91
  92 /* Simple allocator for PhysPageEntry nodes */
  93 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
  94 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
  95
  96 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
  97
  98 static void io_mem_init(void);
  99 static void memory_map_init(void);
 100 static void *qemu_safe_ram_ptr(ram_addr_t addr);
 101
 102 static MemoryRegion io_mem_watch;
 103 #endif
 104
 105 #if !defined(CONFIG_USER_ONLY)
 106
 107 static void phys_map_node_reserve(unsigned nodes)
 108 {
 109     if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
 110         typedef PhysPageEntry Node[L2_SIZE];
 111         phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
 112         phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
 113                                       phys_map_nodes_nb + nodes);
 114         phys_map_nodes = g_renew(Node, phys_map_nodes,
 115                                  phys_map_nodes_nb_alloc);
 116     }
 117 }
 118
 119 static uint16_t phys_map_node_alloc(void)
 120 {
 121     unsigned i;
 122     uint16_t ret;
 123
 124     ret = phys_map_nodes_nb++;
 125     assert(ret != PHYS_MAP_NODE_NIL);
 126     assert(ret != phys_map_nodes_nb_alloc);
 127     for (i = 0; i < L2_SIZE; ++i) {
 128         phys_map_nodes[ret][i].is_leaf = 0;
 129         phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
 130     }
 131     return ret;
 132 }
 133
 134 static void phys_map_nodes_reset(void)
 135 {
 136     phys_map_nodes_nb = 0;
 137 }
 138
 139
 140 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
 141                                 hwaddr *nb, uint16_t leaf,
 142                                 int level)
 143 {
 144     PhysPageEntry *p;
 145     int i;
 146     hwaddr step = (hwaddr)1 << (level * L2_BITS);
 147
 148     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
 149         lp->ptr = phys_map_node_alloc();
 150         p = phys_map_nodes[lp->ptr];
 151         if (level == 0) {
 152             for (i = 0; i < L2_SIZE; i++) {
 153                 p[i].is_leaf = 1;
 154                 p[i].ptr = phys_section_unassigned;
 155             }
 156         }
 157     } else {
 158         p = phys_map_nodes[lp->ptr];
 159     }
 160     lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
 161
 162     while (*nb && lp < &p[L2_SIZE]) {
 163         if ((*index & (step - 1)) == 0 && *nb >= step) {
 164             lp->is_leaf = true;
 165             lp->ptr = leaf;
 166             *index += step;
 167             *nb -= step;
 168         } else {
 169             phys_page_set_level(lp, index, nb, leaf, level - 1);
 170         }
 171         ++lp;
 172     }
 173 }
 174
 175 static void phys_page_set(AddressSpaceDispatch *d,
 176                           hwaddr index, hwaddr nb,
 177                           uint16_t leaf)
 178 {
 179     /* Wildly overreserve - it doesn't matter much. */
 180     phys_map_node_reserve(3 * P_L2_LEVELS);
 181
 182     phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 183 }
 184
 185 MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
 186 {
 187     PhysPageEntry lp = d->phys_map;
 188     PhysPageEntry *p;
 189     int i;
 190     uint16_t s_index = phys_section_unassigned;
 191
 192     for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
 193         if (lp.ptr == PHYS_MAP_NODE_NIL) {
 194             goto not_found;
 195         }
 196         p = phys_map_nodes[lp.ptr];
 197         lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
 198     }
 199
 200     s_index = lp.ptr;
 201 not_found:
 202     return &phys_sections[s_index];
 203 }
 204
 205 bool memory_region_is_unassigned(MemoryRegion *mr)
 206 {
 207     return mr != &io_mem_ram && mr != &io_mem_rom
 208         && mr != &io_mem_notdirty && !mr->rom_device
 209         && mr != &io_mem_watch;
 210 }
 211 #endif
 212
 213 void cpu_exec_init_all(void)
 214 {
 215 #if !defined(CONFIG_USER_ONLY)
 216     qemu_mutex_init(&ram_list.mutex);
 217     memory_map_init();
 218     io_mem_init();
 219 #endif
 220 }
 221
 222 #if !defined(CONFIG_USER_ONLY)
 223
 224 static int cpu_common_post_load(void *opaque, int version_id)
 225 {
 226     CPUState *cpu = opaque;
 227
 228     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
 229        version_id is increased. */
 230     cpu->interrupt_request &= ~0x01;
 231     tlb_flush(cpu->env_ptr, 1);
 232
 233     return 0;
 234 }
 235
 236 static const VMStateDescription vmstate_cpu_common = {
 237     .name = "cpu_common",
 238     .version_id = 1,
 239     .minimum_version_id = 1,
 240     .minimum_version_id_old = 1,
 241     .post_load = cpu_common_post_load,
 242     .fields      = (VMStateField []) {
 243         VMSTATE_UINT32(halted, CPUState),
 244         VMSTATE_UINT32(interrupt_request, CPUState),
 245         VMSTATE_END_OF_LIST()
 246     }
 247 };
 248 #else
 249 #define vmstate_cpu_common vmstate_dummy
 250 #endif
 251
 252 CPUState *qemu_get_cpu(int index)
 253 {
 254     CPUArchState *env = first_cpu;
 255     CPUState *cpu = NULL;
 256
 257     while (env) {
 258         cpu = ENV_GET_CPU(env);
 259         if (cpu->cpu_index == index) {
 260             break;
 261         }
 262         env = env->next_cpu;
 263     }
 264
 265     return env ? cpu : NULL;
 266 }
 267
 268 void cpu_exec_init(CPUArchState *env)
 269 {
 270     CPUState *cpu = ENV_GET_CPU(env);
 271     CPUClass *cc = CPU_GET_CLASS(cpu);
 272     CPUArchState **penv;
 273     int cpu_index;
 274
 275 #if defined(CONFIG_USER_ONLY)
 276     cpu_list_lock();
 277 #endif
 278     env->next_cpu = NULL;
 279     penv = &first_cpu;
 280     cpu_index = 0;
 281     while (*penv != NULL) {
 282         penv = &(*penv)->next_cpu;
 283         cpu_index++;
 284     }
 285     cpu->cpu_index = cpu_index;
 286     cpu->numa_node = 0;
 287     QTAILQ_INIT(&env->breakpoints);
 288     QTAILQ_INIT(&env->watchpoints);
 289 #ifndef CONFIG_USER_ONLY
 290     cpu->thread_id = qemu_get_thread_id();
 291 #endif
 292     *penv = env;
 293 #if defined(CONFIG_USER_ONLY)
 294     cpu_list_unlock();
 295 #endif
 296     vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
 297 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 298     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
 299                     cpu_save, cpu_load, env);
 300     assert(cc->vmsd == NULL);
 301 #endif
 302     if (cc->vmsd != NULL) {
 303         vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
 304     }
 305 }
 306
 307 #if defined(TARGET_HAS_ICE)
 308 #if defined(CONFIG_USER_ONLY)
 309 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 310 {
 311     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 312 }
 313 #else
 314 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 315 {
 316     tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
 317             (pc & ~TARGET_PAGE_MASK));
 318 }
 319 #endif
 320 #endif /* TARGET_HAS_ICE */
 321
 322 #if defined(CONFIG_USER_ONLY)
 323 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 324
 325 {
 326 }
 327
 328 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 329                           int flags, CPUWatchpoint **watchpoint)
 330 {
 331     return -ENOSYS;
 332 }
 333 #else
 334 /* Add a watchpoint.  */
 335 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 336                           int flags, CPUWatchpoint **watchpoint)
 337 {
 338     target_ulong len_mask = ~(len - 1);
 339     CPUWatchpoint *wp;
 340
 341     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
 342     if ((len & (len - 1)) || (addr & ~len_mask) ||
 343             len == 0 || len > TARGET_PAGE_SIZE) {
 344         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
 345                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
 346         return -EINVAL;
 347     }
 348     wp = g_malloc(sizeof(*wp));
 349
 350     wp->vaddr = addr;
 351     wp->len_mask = len_mask;
 352     wp->flags = flags;
 353
 354     /* keep all GDB-injected watchpoints in front */
 355     if (flags & BP_GDB)
 356         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
 357     else
 358         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
 359
 360     tlb_flush_page(env, addr);
 361
 362     if (watchpoint)
 363         *watchpoint = wp;
 364     return 0;
 365 }
 366
 367 /* Remove a specific watchpoint.  */
 368 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
 369                           int flags)
 370 {
 371     target_ulong len_mask = ~(len - 1);
 372     CPUWatchpoint *wp;
 373
 374     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 375         if (addr == wp->vaddr && len_mask == wp->len_mask
 376                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
 377             cpu_watchpoint_remove_by_ref(env, wp);
 378             return 0;
 379         }
 380     }
 381     return -ENOENT;
 382 }
 383
 384 /* Remove a specific watchpoint by reference.  */
 385 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
 386 {
 387     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
 388
 389     tlb_flush_page(env, watchpoint->vaddr);
 390
 391     g_free(watchpoint);
 392 }
 393
 394 /* Remove all matching watchpoints.  */
 395 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 396 {
 397     CPUWatchpoint *wp, *next;
 398
 399     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
 400         if (wp->flags & mask)
 401             cpu_watchpoint_remove_by_ref(env, wp);
 402     }
 403 }
 404 #endif
 405
 406 /* Add a breakpoint.  */
 407 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
 408                           CPUBreakpoint **breakpoint)
 409 {
 410 #if defined(TARGET_HAS_ICE)
 411     CPUBreakpoint *bp;
 412
 413     bp = g_malloc(sizeof(*bp));
 414
 415     bp->pc = pc;
 416     bp->flags = flags;
 417
 418     /* keep all GDB-injected breakpoints in front */
 419     if (flags & BP_GDB)
 420         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
 421     else
 422         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
 423
 424     breakpoint_invalidate(env, pc);
 425
 426     if (breakpoint)
 427         *breakpoint = bp;
 428     return 0;
 429 #else
 430     return -ENOSYS;
 431 #endif
 432 }
 433
 434 /* Remove a specific breakpoint.  */
 435 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
 436 {
 437 #if defined(TARGET_HAS_ICE)
 438     CPUBreakpoint *bp;
 439
 440     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 441         if (bp->pc == pc && bp->flags == flags) {
 442             cpu_breakpoint_remove_by_ref(env, bp);
 443             return 0;
 444         }
 445     }
 446     return -ENOENT;
 447 #else
 448     return -ENOSYS;
 449 #endif
 450 }
 451
 452 /* Remove a specific breakpoint by reference.  */
 453 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
 454 {
 455 #if defined(TARGET_HAS_ICE)
 456     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
 457
 458     breakpoint_invalidate(env, breakpoint->pc);
 459
 460     g_free(breakpoint);
 461 #endif
 462 }
 463
 464 /* Remove all matching breakpoints. */
 465 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
 466 {
 467 #if defined(TARGET_HAS_ICE)
 468     CPUBreakpoint *bp, *next;
 469
 470     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
 471         if (bp->flags & mask)
 472             cpu_breakpoint_remove_by_ref(env, bp);
 473     }
 474 #endif
 475 }
 476
 477 /* enable or disable single step mode. EXCP_DEBUG is returned by the
 478    CPU loop after each instruction */
 479 void cpu_single_step(CPUArchState *env, int enabled)
 480 {
 481 #if defined(TARGET_HAS_ICE)
 482     if (env->singlestep_enabled != enabled) {
 483         env->singlestep_enabled = enabled;
 484         if (kvm_enabled())
 485             kvm_update_guest_debug(env, 0);
 486         else {
 487             /* must flush all the translated code to avoid inconsistencies */
 488             /* XXX: only flush what is necessary */
 489             tb_flush(env);
 490         }
 491     }
 492 #endif
 493 }
 494
 495 void cpu_exit(CPUArchState *env)
 496 {
 497     CPUState *cpu = ENV_GET_CPU(env);
 498
 499     cpu->exit_request = 1;
 500     cpu->tcg_exit_req = 1;
 501 }
 502
 503 void cpu_abort(CPUArchState *env, const char *fmt, ...)
 504 {
 505     va_list ap;
 506     va_list ap2;
 507
 508     va_start(ap, fmt);
 509     va_copy(ap2, ap);
 510     fprintf(stderr, "qemu: fatal: ");
 511     vfprintf(stderr, fmt, ap);
 512     fprintf(stderr, "\n");
 513     cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 514     if (qemu_log_enabled()) {
 515         qemu_log("qemu: fatal: ");
 516         qemu_log_vprintf(fmt, ap2);
 517         qemu_log("\n");
 518         log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 519         qemu_log_flush();
 520         qemu_log_close();
 521     }
 522     va_end(ap2);
 523     va_end(ap);
 524 #if defined(CONFIG_USER_ONLY)
 525     {
 526         struct sigaction act;
 527         sigfillset(&act.sa_mask);
 528         act.sa_handler = SIG_DFL;
 529         sigaction(SIGABRT, &act, NULL);
 530     }
 531 #endif
 532     abort();
 533 }
 534
 535 CPUArchState *cpu_copy(CPUArchState *env)
 536 {
 537     CPUArchState *new_env = cpu_init(env->cpu_model_str);
 538     CPUArchState *next_cpu = new_env->next_cpu;
 539 #if defined(TARGET_HAS_ICE)
 540     CPUBreakpoint *bp;
 541     CPUWatchpoint *wp;
 542 #endif
 543
 544     memcpy(new_env, env, sizeof(CPUArchState));
 545
 546     /* Preserve chaining. */
 547     new_env->next_cpu = next_cpu;
 548
 549     /* Clone all break/watchpoints.
 550        Note: Once we support ptrace with hw-debug register access, make sure
 551        BP_CPU break/watchpoints are handled correctly on clone. */
 552     QTAILQ_INIT(&env->breakpoints);
 553     QTAILQ_INIT(&env->watchpoints);
 554 #if defined(TARGET_HAS_ICE)
 555     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 556         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
 557     }
 558     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 559         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
 560                               wp->flags, NULL);
 561     }
 562 #endif
 563
 564     return new_env;
 565 }
 566
 567 #if !defined(CONFIG_USER_ONLY)
 568 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
 569                                       uintptr_t length)
 570 {
 571     uintptr_t start1;
 572
 573     /* we modify the TLB cache so that the dirty bit will be set again
 574        when accessing the range */
 575     start1 = (uintptr_t)qemu_safe_ram_ptr(start);
 576     /* Check that we don't span multiple blocks - this breaks the
 577        address comparisons below.  */
 578     if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
 579             != (end - 1) - start) {
 580         abort();
 581     }
 582     cpu_tlb_reset_dirty_all(start1, length);
 583
 584 }
 585
 586 /* Note: start and end must be within the same ram block.  */
 587 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
 588                                      int dirty_flags)
 589 {
 590     uintptr_t length;
 591
 592     start &= TARGET_PAGE_MASK;
 593     end = TARGET_PAGE_ALIGN(end);
 594
 595     length = end - start;
 596     if (length == 0)
 597         return;
 598     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
 599
 600     if (tcg_enabled()) {
 601         tlb_reset_dirty_range_all(start, end, length);
 602     }
 603 }
 604
 605 static int cpu_physical_memory_set_dirty_tracking(int enable)
 606 {
 607     int ret = 0;
 608     in_migration = enable;
 609     return ret;
 610 }
 611
 612 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 613                                                    MemoryRegionSection *section,
 614                                                    target_ulong vaddr,
 615                                                    hwaddr paddr,
 616                                                    int prot,
 617                                                    target_ulong *address)
 618 {
 619     hwaddr iotlb;
 620     CPUWatchpoint *wp;
 621
 622     if (memory_region_is_ram(section->mr)) {
 623         /* Normal RAM.  */
 624         iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
 625             + memory_region_section_addr(section, paddr);
 626         if (!section->readonly) {
 627             iotlb |= phys_section_notdirty;
 628         } else {
 629             iotlb |= phys_section_rom;
 630         }
 631     } else {
 632         /* IO handlers are currently passed a physical address.
 633            It would be nice to pass an offset from the base address
 634            of that region.  This would avoid having to special case RAM,
 635            and avoid full address decoding in every device.
 636            We can't use the high bits of pd for this because
 637            IO_MEM_ROMD uses these as a ram address.  */
 638         iotlb = section - phys_sections;
 639         iotlb += memory_region_section_addr(section, paddr);
 640     }
 641
 642     /* Make accesses to pages with watchpoints go via the
 643        watchpoint trap routines.  */
 644     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 645         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
 646             /* Avoid trapping reads of pages with a write breakpoint. */
 647             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
 648                 iotlb = phys_section_watch + paddr;
 649                 *address |= TLB_MMIO;
 650                 break;
 651             }
 652         }
 653     }
 654
 655     return iotlb;
 656 }
 657 #endif /* defined(CONFIG_USER_ONLY) */
 658
 659 #if !defined(CONFIG_USER_ONLY)
 660
 661 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 662 typedef struct subpage_t {
 663     MemoryRegion iomem;
 664     hwaddr base;
 665     uint16_t sub_section[TARGET_PAGE_SIZE];
 666 } subpage_t;
 667
 668 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
 669                              uint16_t section);
 670 static subpage_t *subpage_init(hwaddr base);
 671 static void destroy_page_desc(uint16_t section_index)
 672 {
 673     MemoryRegionSection *section = &phys_sections[section_index];
 674     MemoryRegion *mr = section->mr;
 675
 676     if (mr->subpage) {
 677         subpage_t *subpage = container_of(mr, subpage_t, iomem);
 678         memory_region_destroy(&subpage->iomem);
 679         g_free(subpage);
 680     }
 681 }
 682
 683 static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
 684 {
 685     unsigned i;
 686     PhysPageEntry *p;
 687
 688     if (lp->ptr == PHYS_MAP_NODE_NIL) {
 689         return;
 690     }
 691
 692     p = phys_map_nodes[lp->ptr];
 693     for (i = 0; i < L2_SIZE; ++i) {
 694         if (!p[i].is_leaf) {
 695             destroy_l2_mapping(&p[i], level - 1);
 696         } else {
 697             destroy_page_desc(p[i].ptr);
 698         }
 699     }
 700     lp->is_leaf = 0;
 701     lp->ptr = PHYS_MAP_NODE_NIL;
 702 }
 703
 704 static void destroy_all_mappings(AddressSpaceDispatch *d)
 705 {
 706     destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
 707     phys_map_nodes_reset();
 708 }
 709
 710 static uint16_t phys_section_add(MemoryRegionSection *section)
 711 {
 712     if (phys_sections_nb == phys_sections_nb_alloc) {
 713         phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
 714         phys_sections = g_renew(MemoryRegionSection, phys_sections,
 715                                 phys_sections_nb_alloc);
 716     }
 717     phys_sections[phys_sections_nb] = *section;
 718     return phys_sections_nb++;
 719 }
 720
 721 static void phys_sections_clear(void)
 722 {
 723     phys_sections_nb = 0;
 724 }
 725
 726 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 727 {
 728     subpage_t *subpage;
 729     hwaddr base = section->offset_within_address_space
 730         & TARGET_PAGE_MASK;
 731     MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
 732     MemoryRegionSection subsection = {
 733         .offset_within_address_space = base,
 734         .size = TARGET_PAGE_SIZE,
 735     };
 736     hwaddr start, end;
 737
 738     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 739
 740     if (!(existing->mr->subpage)) {
 741         subpage = subpage_init(base);
 742         subsection.mr = &subpage->iomem;
 743         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
 744                       phys_section_add(&subsection));
 745     } else {
 746         subpage = container_of(existing->mr, subpage_t, iomem);
 747     }
 748     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
 749     end = start + section->size - 1;
 750     subpage_register(subpage, start, end, phys_section_add(section));
 751 }
 752
 753
 754 static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 755 {
 756     hwaddr start_addr = section->offset_within_address_space;
 757     ram_addr_t size = section->size;
 758     hwaddr addr;
 759     uint16_t section_index = phys_section_add(section);
 760
 761     assert(size);
 762
 763     addr = start_addr;
 764     phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
 765                   section_index);
 766 }
 767
 768 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 769 {
 770     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
 771     MemoryRegionSection now = *section, remain = *section;
 772
 773     if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
 774         || (now.size < TARGET_PAGE_SIZE)) {
 775         now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
 776                        - now.offset_within_address_space,
 777                        now.size);
 778         register_subpage(d, &now);
 779         remain.size -= now.size;
 780         remain.offset_within_address_space += now.size;
 781         remain.offset_within_region += now.size;
 782     }
 783     while (remain.size >= TARGET_PAGE_SIZE) {
 784         now = remain;
 785         if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
 786             now.size = TARGET_PAGE_SIZE;
 787             register_subpage(d, &now);
 788         } else {
 789             now.size &= TARGET_PAGE_MASK;
 790             register_multipage(d, &now);
 791         }
 792         remain.size -= now.size;
 793         remain.offset_within_address_space += now.size;
 794         remain.offset_within_region += now.size;
 795     }
 796     now = remain;
 797     if (now.size) {
 798         register_subpage(d, &now);
 799     }
 800 }
 801
 802 void qemu_flush_coalesced_mmio_buffer(void)
 803 {
 804     if (kvm_enabled())
 805         kvm_flush_coalesced_mmio_buffer();
 806 }
 807
 808 void qemu_mutex_lock_ramlist(void)
 809 {
 810     qemu_mutex_lock(&ram_list.mutex);
 811 }
 812
 813 void qemu_mutex_unlock_ramlist(void)
 814 {
 815     qemu_mutex_unlock(&ram_list.mutex);
 816 }
 817
 818 #if defined(__linux__) && !defined(TARGET_S390X)
 819
 820 #include <sys/vfs.h>
 821
 822 #define HUGETLBFS_MAGIC       0x958458f6
 823
 824 static long gethugepagesize(const char *path)
 825 {
 826     struct statfs fs;
 827     int ret;
 828
 829     do {
 830         ret = statfs(path, &fs);
 831     } while (ret != 0 && errno == EINTR);
 832
 833     if (ret != 0) {
 834         perror(path);
 835         return 0;
 836     }
 837
 838     if (fs.f_type != HUGETLBFS_MAGIC)
 839         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
 840
 841     return fs.f_bsize;
 842 }
 843
 844 static void *file_ram_alloc(RAMBlock *block,
 845                             ram_addr_t memory,
 846                             const char *path)
 847 {
 848     char *filename;
 849     char *sanitized_name;
 850     char *c;
 851     void *area;
 852     int fd;
 853 #ifdef MAP_POPULATE
 854     int flags;
 855 #endif
 856     unsigned long hpagesize;
 857
 858     hpagesize = gethugepagesize(path);
 859     if (!hpagesize) {
 860         return NULL;
 861     }
 862
 863     if (memory < hpagesize) {
 864         return NULL;
 865     }
 866
 867     if (kvm_enabled() && !kvm_has_sync_mmu()) {
 868         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
 869         return NULL;
 870     }
 871
 872     /* Make name safe to use with mkstemp by replacing '/' with '_'. */
 873     sanitized_name = g_strdup(block->mr->name);
 874     for (c = sanitized_name; *c != '\0'; c++) {
 875         if (*c == '/')
 876             *c = '_';
 877     }
 878
 879     filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
 880                                sanitized_name);
 881     g_free(sanitized_name);
 882
 883     fd = mkstemp(filename);
 884     if (fd < 0) {
 885         perror("unable to create backing store for hugepages");
 886         g_free(filename);
 887         return NULL;
 888     }
 889     unlink(filename);
 890     g_free(filename);
 891
 892     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 893
 894     /*
 895      * ftruncate is not supported by hugetlbfs in older
 896      * hosts, so don't bother bailing out on errors.
 897      * If anything goes wrong with it under other filesystems,
 898      * mmap will fail.
 899      */
 900     if (ftruncate(fd, memory))
 901         perror("ftruncate");
 902
 903 #ifdef MAP_POPULATE
 904     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
 905      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
 906      * to sidestep this quirk.
 907      */
 908     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
 909     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
 910 #else
 911     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 912 #endif
 913     if (area == MAP_FAILED) {
 914         perror("file_ram_alloc: can't mmap RAM pages");
 915         close(fd);
 916         return (NULL);
 917     }
 918     block->fd = fd;
 919     return area;
 920 }
 921 #endif
 922
 923 static ram_addr_t find_ram_offset(ram_addr_t size)
 924 {
 925     RAMBlock *block, *next_block;
 926     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
 927
 928     if (QTAILQ_EMPTY(&ram_list.blocks))
 929         return 0;
 930
 931     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
 932         ram_addr_t end, next = RAM_ADDR_MAX;
 933
 934         end = block->offset + block->length;
 935
 936         QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
 937             if (next_block->offset >= end) {
 938                 next = MIN(next, next_block->offset);
 939             }
 940         }
 941         if (next - end >= size && next - end < mingap) {
 942             offset = end;
 943             mingap = next - end;
 944         }
 945     }
 946
 947     if (offset == RAM_ADDR_MAX) {
 948         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
 949                 (uint64_t)size);
 950         abort();
 951     }
 952
 953     return offset;
 954 }
 955
 956 ram_addr_t last_ram_offset(void)
 957 {
 958     RAMBlock *block;
 959     ram_addr_t last = 0;
 960
 961     QTAILQ_FOREACH(block, &ram_list.blocks, next)
 962         last = MAX(last, block->offset + block->length);
 963
 964     return last;
 965 }
 966
 967 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
 968 {
 969     int ret;
 970     QemuOpts *machine_opts;
 971
 972     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
 973     machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
 974     if (machine_opts &&
 975         !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
 976         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
 977         if (ret) {
 978             perror("qemu_madvise");
 979             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
 980                             "but dump_guest_core=off specified\n");
 981         }
 982     }
 983 }
 984
 985 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
 986 {
 987     RAMBlock *new_block, *block;
 988
 989     new_block = NULL;
 990     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
 991         if (block->offset == addr) {
 992             new_block = block;
 993             break;
 994         }
 995     }
 996     assert(new_block);
 997     assert(!new_block->idstr[0]);
 998
 999     if (dev) {
1000         char *id = qdev_get_dev_path(dev);
1001         if (id) {
1002             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1003             g_free(id);
1004         }
1005     }
1006     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1007
1008     /* This assumes the iothread lock is taken here too.  */
1009     qemu_mutex_lock_ramlist();
1010     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1011         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1012             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1013                     new_block->idstr);
1014             abort();
1015         }
1016     }
1017     qemu_mutex_unlock_ramlist();
1018 }
1019
1020 static int memory_try_enable_merging(void *addr, size_t len)
1021 {
1022     QemuOpts *opts;
1023
1024     opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1025     if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1026         /* disabled by the user */
1027         return 0;
1028     }
1029
1030     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1031 }
1032
1033 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1034                                    MemoryRegion *mr)
1035 {
1036     RAMBlock *block, *new_block;
1037
1038     size = TARGET_PAGE_ALIGN(size);
1039     new_block = g_malloc0(sizeof(*new_block));
1040
1041     /* This assumes the iothread lock is taken here too.  */
1042     qemu_mutex_lock_ramlist();
1043     new_block->mr = mr;
1044     new_block->offset = find_ram_offset(size);
1045     if (host) {
1046         new_block->host = host;
1047         new_block->flags |= RAM_PREALLOC_MASK;
1048     } else {
1049         if (mem_path) {
1050 #if defined (__linux__) && !defined(TARGET_S390X)
1051             new_block->host = file_ram_alloc(new_block, size, mem_path);
1052             if (!new_block->host) {
1053                 new_block->host = qemu_vmalloc(size);
1054                 memory_try_enable_merging(new_block->host, size);
1055             }
1056 #else
1057             fprintf(stderr, "-mem-path option unsupported\n");
1058             exit(1);
1059 #endif
1060         } else {
1061             if (xen_enabled()) {
1062                 xen_ram_alloc(new_block->offset, size, mr);
1063             } else if (kvm_enabled()) {
1064                 /* some s390/kvm configurations have special constraints */
1065                 new_block->host = kvm_vmalloc(size);
1066             } else {
1067                 new_block->host = qemu_vmalloc(size);
1068             }
1069             memory_try_enable_merging(new_block->host, size);
1070         }
1071     }
1072     new_block->length = size;
1073
1074     /* Keep the list sorted from biggest to smallest block.  */
1075     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1076         if (block->length < new_block->length) {
1077             break;
1078         }
1079     }
1080     if (block) {
1081         QTAILQ_INSERT_BEFORE(block, new_block, next);
1082     } else {
1083         QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1084     }
1085     ram_list.mru_block = NULL;
1086
1087     ram_list.version++;
1088     qemu_mutex_unlock_ramlist();
1089
1090     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1091                                        last_ram_offset() >> TARGET_PAGE_BITS);
1092     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1093            0, size >> TARGET_PAGE_BITS);
1094     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1095
1096     qemu_ram_setup_dump(new_block->host, size);
1097     qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1098
1099     if (kvm_enabled())
1100         kvm_setup_guest_memory(new_block->host, size);
1101
1102     return new_block->offset;
1103 }
1104
1105 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1106 {
1107     return qemu_ram_alloc_from_ptr(size, NULL, mr);
1108 }
1109
1110 void qemu_ram_free_from_ptr(ram_addr_t addr)
1111 {
1112     RAMBlock *block;
1113
1114     /* This assumes the iothread lock is taken here too.  */
1115     qemu_mutex_lock_ramlist();
1116     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1117         if (addr == block->offset) {
1118             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1119             ram_list.mru_block = NULL;
1120             ram_list.version++;
1121             g_free(block);
1122             break;
1123         }
1124     }
1125     qemu_mutex_unlock_ramlist();
1126 }
1127
1128 void qemu_ram_free(ram_addr_t addr)
1129 {
1130     RAMBlock *block;
1131
1132     /* This assumes the iothread lock is taken here too.  */
1133     qemu_mutex_lock_ramlist();
1134     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1135         if (addr == block->offset) {
1136             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1137             ram_list.mru_block = NULL;
1138             ram_list.version++;
1139             if (block->flags & RAM_PREALLOC_MASK) {
1140                 ;
1141             } else if (mem_path) {
1142 #if defined (__linux__) && !defined(TARGET_S390X)
1143                 if (block->fd) {
1144                     munmap(block->host, block->length);
1145                     close(block->fd);
1146                 } else {
1147                     qemu_vfree(block->host);
1148                 }
1149 #else
1150                 abort();
1151 #endif
1152             } else {
1153 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1154                 munmap(block->host, block->length);
1155 #else
1156                 if (xen_enabled()) {
1157                     xen_invalidate_map_cache_entry(block->host);
1158                 } else {
1159                     qemu_vfree(block->host);
1160                 }
1161 #endif
1162             }
1163             g_free(block);
1164             break;
1165         }
1166     }
1167     qemu_mutex_unlock_ramlist();
1168
1169 }
1170
1171 #ifndef _WIN32
1172 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1173 {
1174     RAMBlock *block;
1175     ram_addr_t offset;
1176     int flags;
1177     void *area, *vaddr;
1178
1179     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1180         offset = addr - block->offset;
1181         if (offset < block->length) {
1182             vaddr = block->host + offset;
1183             if (block->flags & RAM_PREALLOC_MASK) {
1184                 ;
1185             } else {
1186                 flags = MAP_FIXED;
1187                 munmap(vaddr, length);
1188                 if (mem_path) {
1189 #if defined(__linux__) && !defined(TARGET_S390X)
1190                     if (block->fd) {
1191 #ifdef MAP_POPULATE
1192                         flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1193                             MAP_PRIVATE;
1194 #else
1195                         flags |= MAP_PRIVATE;
1196 #endif
1197                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1198                                     flags, block->fd, offset);
1199                     } else {
1200                         flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1201                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1202                                     flags, -1, 0);
1203                     }
1204 #else
1205                     abort();
1206 #endif
1207                 } else {
1208 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1209                     flags |= MAP_SHARED | MAP_ANONYMOUS;
1210                     area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1211                                 flags, -1, 0);
1212 #else
1213                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1214                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1215                                 flags, -1, 0);
1216 #endif
1217                 }
1218                 if (area != vaddr) {
1219                     fprintf(stderr, "Could not remap addr: "
1220                             RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1221                             length, addr);
1222                     exit(1);
1223                 }
1224                 memory_try_enable_merging(vaddr, length);
1225                 qemu_ram_setup_dump(vaddr, length);
1226             }
1227             return;
1228         }
1229     }
1230 }
1231 #endif /* !_WIN32 */
1232
1233 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1234    With the exception of the softmmu code in this file, this should
1235    only be used for local memory (e.g. video ram) that the device owns,
1236    and knows it isn't going to access beyond the end of the block.
1237
1238    It should not be used for general purpose DMA.
1239    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1240  */
1241 void *qemu_get_ram_ptr(ram_addr_t addr)
1242 {
1243     RAMBlock *block;
1244
1245     /* The list is protected by the iothread lock here.  */
1246     block = ram_list.mru_block;
1247     if (block && addr - block->offset < block->length) {
1248         goto found;
1249     }
1250     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1251         if (addr - block->offset < block->length) {
1252             goto found;
1253         }
1254     }
1255
1256     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1257     abort();
1258
1259 found:
1260     ram_list.mru_block = block;
1261     if (xen_enabled()) {
1262         /* We need to check if the requested address is in the RAM
1263          * because we don't want to map the entire memory in QEMU.
1264          * In that case just map until the end of the page.
1265          */
1266         if (block->offset == 0) {
1267             return xen_map_cache(addr, 0, 0);
1268         } else if (block->host == NULL) {
1269             block->host =
1270                 xen_map_cache(block->offset, block->length, 1);
1271         }
1272     }
1273     return block->host + (addr - block->offset);
1274 }
1275
1276 /* Return a host pointer to ram allocated with qemu_ram_alloc.  Same as
1277  * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1278  *
1279  * ??? Is this still necessary?
1280  */
1281 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1282 {
1283     RAMBlock *block;
1284
1285     /* The list is protected by the iothread lock here.  */
1286     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1287         if (addr - block->offset < block->length) {
1288             if (xen_enabled()) {
1289                 /* We need to check if the requested address is in the RAM
1290                  * because we don't want to map the entire memory in QEMU.
1291                  * In that case just map until the end of the page.
1292                  */
1293                 if (block->offset == 0) {
1294                     return xen_map_cache(addr, 0, 0);
1295                 } else if (block->host == NULL) {
1296                     block->host =
1297                         xen_map_cache(block->offset, block->length, 1);
1298                 }
1299             }
1300             return block->host + (addr - block->offset);
1301         }
1302     }
1303
1304     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1305     abort();
1306
1307     return NULL;
1308 }
1309
1310 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1311  * but takes a size argument */
1312 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1313 {
1314     if (*size == 0) {
1315         return NULL;
1316     }
1317     if (xen_enabled()) {
1318         return xen_map_cache(addr, *size, 1);
1319     } else {
1320         RAMBlock *block;
1321
1322         QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1323             if (addr - block->offset < block->length) {
1324                 if (addr - block->offset + *size > block->length)
1325                     *size = block->length - addr + block->offset;
1326                 return block->host + (addr - block->offset);
1327             }
1328         }
1329
1330         fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1331         abort();
1332     }
1333 }
1334
1335 void qemu_put_ram_ptr(void *addr)
1336 {
1337     trace_qemu_put_ram_ptr(addr);
1338 }
1339
1340 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1341 {
1342     RAMBlock *block;
1343     uint8_t *host = ptr;
1344
1345     if (xen_enabled()) {
1346         *ram_addr = xen_ram_addr_from_mapcache(ptr);
1347         return 0;
1348     }
1349
1350     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1351         /* This case append when the block is not mapped. */
1352         if (block->host == NULL) {
1353             continue;
1354         }
1355         if (host - block->host < block->length) {
1356             *ram_addr = block->offset + (host - block->host);
1357             return 0;
1358         }
1359     }
1360
1361     return -1;
1362 }
1363
1364 /* Some of the softmmu routines need to translate from a host pointer
1365    (typically a TLB entry) back to a ram offset.  */
1366 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1367 {
1368     ram_addr_t ram_addr;
1369
1370     if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1371         fprintf(stderr, "Bad ram pointer %p\n", ptr);
1372         abort();
1373     }
1374     return ram_addr;
1375 }
1376
1377 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1378                                     unsigned size)
1379 {
1380 #ifdef DEBUG_UNASSIGNED
1381     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1382 #endif
1383 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1384     cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, size);
1385 #endif
1386     return 0;
1387 }
1388
1389 static void unassigned_mem_write(void *opaque, hwaddr addr,
1390                                  uint64_t val, unsigned size)
1391 {
1392 #ifdef DEBUG_UNASSIGNED
1393     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1394 #endif
1395 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1396     cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, size);
1397 #endif
1398 }
1399
1400 static const MemoryRegionOps unassigned_mem_ops = {
1401     .read = unassigned_mem_read,
1402     .write = unassigned_mem_write,
1403     .endianness = DEVICE_NATIVE_ENDIAN,
1404 };
1405
1406 static uint64_t error_mem_read(void *opaque, hwaddr addr,
1407                                unsigned size)
1408 {
1409     abort();
1410 }
1411
1412 static void error_mem_write(void *opaque, hwaddr addr,
1413                             uint64_t value, unsigned size)
1414 {
1415     abort();
1416 }
1417
1418 static const MemoryRegionOps error_mem_ops = {
1419     .read = error_mem_read,
1420     .write = error_mem_write,
1421     .endianness = DEVICE_NATIVE_ENDIAN,
1422 };
1423
1424 static const MemoryRegionOps rom_mem_ops = {
1425     .read = error_mem_read,
1426     .write = unassigned_mem_write,
1427     .endianness = DEVICE_NATIVE_ENDIAN,
1428 };
1429
1430 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1431                                uint64_t val, unsigned size)
1432 {
1433     int dirty_flags;
1434     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1435     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1436 #if !defined(CONFIG_USER_ONLY)
1437         tb_invalidate_phys_page_fast(ram_addr, size);
1438         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1439 #endif
1440     }
1441     switch (size) {
1442     case 1:
1443         stb_p(qemu_get_ram_ptr(ram_addr), val);
1444         break;
1445     case 2:
1446         stw_p(qemu_get_ram_ptr(ram_addr), val);
1447         break;
1448     case 4:
1449         stl_p(qemu_get_ram_ptr(ram_addr), val);
1450         break;
1451     default:
1452         abort();
1453     }
1454     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1455     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1456     /* we remove the notdirty callback only if the code has been
1457        flushed */
1458     if (dirty_flags == 0xff)
1459         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1460 }
1461
1462 static const MemoryRegionOps notdirty_mem_ops = {
1463     .read = error_mem_read,
1464     .write = notdirty_mem_write,
1465     .endianness = DEVICE_NATIVE_ENDIAN,
1466 };
1467
1468 /* Generate a debug exception if a watchpoint has been hit.  */
1469 static void check_watchpoint(int offset, int len_mask, int flags)
1470 {
1471     CPUArchState *env = cpu_single_env;
1472     target_ulong pc, cs_base;
1473     target_ulong vaddr;
1474     CPUWatchpoint *wp;
1475     int cpu_flags;
1476
1477     if (env->watchpoint_hit) {
1478         /* We re-entered the check after replacing the TB. Now raise
1479          * the debug interrupt so that is will trigger after the
1480          * current instruction. */
1481         cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1482         return;
1483     }
1484     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1485     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1486         if ((vaddr == (wp->vaddr & len_mask) ||
1487              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1488             wp->flags |= BP_WATCHPOINT_HIT;
1489             if (!env->watchpoint_hit) {
1490                 env->watchpoint_hit = wp;
1491                 tb_check_watchpoint(env);
1492                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1493                     env->exception_index = EXCP_DEBUG;
1494                     cpu_loop_exit(env);
1495                 } else {
1496                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1497                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1498                     cpu_resume_from_signal(env, NULL);
1499                 }
1500             }
1501         } else {
1502             wp->flags &= ~BP_WATCHPOINT_HIT;
1503         }
1504     }
1505 }
1506
1507 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
1508    so these check for a hit then pass through to the normal out-of-line
1509    phys routines.  */
1510 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1511                                unsigned size)
1512 {
1513     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1514     switch (size) {
1515     case 1: return ldub_phys(addr);
1516     case 2: return lduw_phys(addr);
1517     case 4: return ldl_phys(addr);
1518     default: abort();
1519     }
1520 }
1521
1522 static void watch_mem_write(void *opaque, hwaddr addr,
1523                             uint64_t val, unsigned size)
1524 {
1525     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1526     switch (size) {
1527     case 1:
1528         stb_phys(addr, val);
1529         break;
1530     case 2:
1531         stw_phys(addr, val);
1532         break;
1533     case 4:
1534         stl_phys(addr, val);
1535         break;
1536     default: abort();
1537     }
1538 }
1539
1540 static const MemoryRegionOps watch_mem_ops = {
1541     .read = watch_mem_read,
1542     .write = watch_mem_write,
1543     .endianness = DEVICE_NATIVE_ENDIAN,
1544 };
1545
1546 static uint64_t subpage_read(void *opaque, hwaddr addr,
1547                              unsigned len)
1548 {
1549     subpage_t *mmio = opaque;
1550     unsigned int idx = SUBPAGE_IDX(addr);
1551     MemoryRegionSection *section;
1552 #if defined(DEBUG_SUBPAGE)
1553     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
1554            mmio, len, addr, idx);
1555 #endif
1556
1557     section = &phys_sections[mmio->sub_section[idx]];
1558     addr += mmio->base;
1559     addr -= section->offset_within_address_space;
1560     addr += section->offset_within_region;
1561     return io_mem_read(section->mr, addr, len);
1562 }
1563
1564 static void subpage_write(void *opaque, hwaddr addr,
1565                           uint64_t value, unsigned len)
1566 {
1567     subpage_t *mmio = opaque;
1568     unsigned int idx = SUBPAGE_IDX(addr);
1569     MemoryRegionSection *section;
1570 #if defined(DEBUG_SUBPAGE)
1571     printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1572            " idx %d value %"PRIx64"\n",
1573            __func__, mmio, len, addr, idx, value);
1574 #endif
1575
1576     section = &phys_sections[mmio->sub_section[idx]];
1577     addr += mmio->base;
1578     addr -= section->offset_within_address_space;
1579     addr += section->offset_within_region;
1580     io_mem_write(section->mr, addr, value, len);
1581 }
1582
1583 static const MemoryRegionOps subpage_ops = {
1584     .read = subpage_read,
1585     .write = subpage_write,
1586     .endianness = DEVICE_NATIVE_ENDIAN,
1587 };
1588
1589 static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
1590                                  unsigned size)
1591 {
1592     ram_addr_t raddr = addr;
1593     void *ptr = qemu_get_ram_ptr(raddr);
1594     switch (size) {
1595     case 1: return ldub_p(ptr);
1596     case 2: return lduw_p(ptr);
1597     case 4: return ldl_p(ptr);
1598     default: abort();
1599     }
1600 }
1601
1602 static void subpage_ram_write(void *opaque, hwaddr addr,
1603                               uint64_t value, unsigned size)
1604 {
1605     ram_addr_t raddr = addr;
1606     void *ptr = qemu_get_ram_ptr(raddr);
1607     switch (size) {
1608     case 1: return stb_p(ptr, value);
1609     case 2: return stw_p(ptr, value);
1610     case 4: return stl_p(ptr, value);
1611     default: abort();
1612     }
1613 }
1614
1615 static const MemoryRegionOps subpage_ram_ops = {
1616     .read = subpage_ram_read,
1617     .write = subpage_ram_write,
1618     .endianness = DEVICE_NATIVE_ENDIAN,
1619 };
1620
1621 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1622                              uint16_t section)
1623 {
1624     int idx, eidx;
1625
1626     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1627         return -1;
1628     idx = SUBPAGE_IDX(start);
1629     eidx = SUBPAGE_IDX(end);
1630 #if defined(DEBUG_SUBPAGE)
1631     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1632            mmio, start, end, idx, eidx, memory);
1633 #endif
1634     if (memory_region_is_ram(phys_sections[section].mr)) {
1635         MemoryRegionSection new_section = phys_sections[section];
1636         new_section.mr = &io_mem_subpage_ram;
1637         section = phys_section_add(&new_section);
1638     }
1639     for (; idx <= eidx; idx++) {
1640         mmio->sub_section[idx] = section;
1641     }
1642
1643     return 0;
1644 }
1645
1646 static subpage_t *subpage_init(hwaddr base)
1647 {
1648     subpage_t *mmio;
1649
1650     mmio = g_malloc0(sizeof(subpage_t));
1651
1652     mmio->base = base;
1653     memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1654                           "subpage", TARGET_PAGE_SIZE);
1655     mmio->iomem.subpage = true;
1656 #if defined(DEBUG_SUBPAGE)
1657     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1658            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1659 #endif
1660     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1661
1662     return mmio;
1663 }
1664
1665 static uint16_t dummy_section(MemoryRegion *mr)
1666 {
1667     MemoryRegionSection section = {
1668         .mr = mr,
1669         .offset_within_address_space = 0,
1670         .offset_within_region = 0,
1671         .size = UINT64_MAX,
1672     };
1673
1674     return phys_section_add(&section);
1675 }
1676
1677 MemoryRegion *iotlb_to_region(hwaddr index)
1678 {
1679     return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1680 }
1681
1682 static void io_mem_init(void)
1683 {
1684     memory_region_init_io(&io_mem_ram, &error_mem_ops, NULL, "ram", UINT64_MAX);
1685     memory_region_init_io(&io_mem_rom, &rom_mem_ops, NULL, "rom", UINT64_MAX);
1686     memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1687                           "unassigned", UINT64_MAX);
1688     memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1689                           "notdirty", UINT64_MAX);
1690     memory_region_init_io(&io_mem_subpage_ram, &subpage_ram_ops, NULL,
1691                           "subpage-ram", UINT64_MAX);
1692     memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1693                           "watch", UINT64_MAX);
1694 }
1695
1696 static void mem_begin(MemoryListener *listener)
1697 {
1698     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1699
1700     destroy_all_mappings(d);
1701     d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1702 }
1703
1704 static void core_begin(MemoryListener *listener)
1705 {
1706     phys_sections_clear();
1707     phys_section_unassigned = dummy_section(&io_mem_unassigned);
1708     phys_section_notdirty = dummy_section(&io_mem_notdirty);
1709     phys_section_rom = dummy_section(&io_mem_rom);
1710     phys_section_watch = dummy_section(&io_mem_watch);
1711 }
1712
1713 static void tcg_commit(MemoryListener *listener)
1714 {
1715     CPUArchState *env;
1716
1717     /* since each CPU stores ram addresses in its TLB cache, we must
1718        reset the modified entries */
1719     /* XXX: slow ! */
1720     for(env = first_cpu; env != NULL; env = env->next_cpu) {
1721         tlb_flush(env, 1);
1722     }
1723 }
1724
1725 static void core_log_global_start(MemoryListener *listener)
1726 {
1727     cpu_physical_memory_set_dirty_tracking(1);
1728 }
1729
1730 static void core_log_global_stop(MemoryListener *listener)
1731 {
1732     cpu_physical_memory_set_dirty_tracking(0);
1733 }
1734
1735 static void io_region_add(MemoryListener *listener,
1736                           MemoryRegionSection *section)
1737 {
1738     MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1739
1740     mrio->mr = section->mr;
1741     mrio->offset = section->offset_within_region;
1742     iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1743                  section->offset_within_address_space, section->size);
1744     ioport_register(&mrio->iorange);
1745 }
1746
1747 static void io_region_del(MemoryListener *listener,
1748                           MemoryRegionSection *section)
1749 {
1750     isa_unassign_ioport(section->offset_within_address_space, section->size);
1751 }
1752
1753 static MemoryListener core_memory_listener = {
1754     .begin = core_begin,
1755     .log_global_start = core_log_global_start,
1756     .log_global_stop = core_log_global_stop,
1757     .priority = 1,
1758 };
1759
1760 static MemoryListener io_memory_listener = {
1761     .region_add = io_region_add,
1762     .region_del = io_region_del,
1763     .priority = 0,
1764 };
1765
1766 static MemoryListener tcg_memory_listener = {
1767     .commit = tcg_commit,
1768 };
1769
1770 void address_space_init_dispatch(AddressSpace *as)
1771 {
1772     AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1773
1774     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1775     d->listener = (MemoryListener) {
1776         .begin = mem_begin,
1777         .region_add = mem_add,
1778         .region_nop = mem_add,
1779         .priority = 0,
1780     };
1781     as->dispatch = d;
1782     memory_listener_register(&d->listener, as);
1783 }
1784
1785 void address_space_destroy_dispatch(AddressSpace *as)
1786 {
1787     AddressSpaceDispatch *d = as->dispatch;
1788
1789     memory_listener_unregister(&d->listener);
1790     destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1791     g_free(d);
1792     as->dispatch = NULL;
1793 }
1794
1795 static void memory_map_init(void)
1796 {
1797     system_memory = g_malloc(sizeof(*system_memory));
1798     memory_region_init(system_memory, "system", INT64_MAX);
1799     address_space_init(&address_space_memory, system_memory);
1800     address_space_memory.name = "memory";
1801
1802     system_io = g_malloc(sizeof(*system_io));
1803     memory_region_init(system_io, "io", 65536);
1804     address_space_init(&address_space_io, system_io);
1805     address_space_io.name = "I/O";
1806
1807     memory_listener_register(&core_memory_listener, &address_space_memory);
1808     memory_listener_register(&io_memory_listener, &address_space_io);
1809     memory_listener_register(&tcg_memory_listener, &address_space_memory);
1810
1811     dma_context_init(&dma_context_memory, &address_space_memory,
1812                      NULL, NULL, NULL);
1813 }
1814
1815 MemoryRegion *get_system_memory(void)
1816 {
1817     return system_memory;
1818 }
1819
1820 MemoryRegion *get_system_io(void)
1821 {
1822     return system_io;
1823 }
1824
1825 #endif /* !defined(CONFIG_USER_ONLY) */
1826
1827 /* physical memory access (slow version, mainly for debug) */
1828 #if defined(CONFIG_USER_ONLY)
1829 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1830                         uint8_t *buf, int len, int is_write)
1831 {
1832     int l, flags;
1833     target_ulong page;
1834     void * p;
1835
1836     while (len > 0) {
1837         page = addr & TARGET_PAGE_MASK;
1838         l = (page + TARGET_PAGE_SIZE) - addr;
1839         if (l > len)
1840             l = len;
1841         flags = page_get_flags(page);
1842         if (!(flags & PAGE_VALID))
1843             return -1;
1844         if (is_write) {
1845             if (!(flags & PAGE_WRITE))
1846                 return -1;
1847             /* XXX: this code should not depend on lock_user */
1848             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1849                 return -1;
1850             memcpy(p, buf, l);
1851             unlock_user(p, addr, l);
1852         } else {
1853             if (!(flags & PAGE_READ))
1854                 return -1;
1855             /* XXX: this code should not depend on lock_user */
1856             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1857                 return -1;
1858             memcpy(buf, p, l);
1859             unlock_user(p, addr, 0);
1860         }
1861         len -= l;
1862         buf += l;
1863         addr += l;
1864     }
1865     return 0;
1866 }
1867
1868 #else
1869
1870 static void invalidate_and_set_dirty(hwaddr addr,
1871                                      hwaddr length)
1872 {
1873     if (!cpu_physical_memory_is_dirty(addr)) {
1874         /* invalidate code */
1875         tb_invalidate_phys_page_range(addr, addr + length, 0);
1876         /* set dirty bit */
1877         cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1878     }
1879     xen_modified_memory(addr, length);
1880 }
1881
1882 void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1883                       int len, bool is_write)
1884 {
1885     AddressSpaceDispatch *d = as->dispatch;
1886     int l;
1887     uint8_t *ptr;
1888     uint32_t val;
1889     hwaddr page;
1890     MemoryRegionSection *section;
1891
1892     while (len > 0) {
1893         page = addr & TARGET_PAGE_MASK;
1894         l = (page + TARGET_PAGE_SIZE) - addr;
1895         if (l > len)
1896             l = len;
1897         section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1898
1899         if (is_write) {
1900             if (!memory_region_is_ram(section->mr)) {
1901                 hwaddr addr1;
1902                 addr1 = memory_region_section_addr(section, addr);
1903                 /* XXX: could force cpu_single_env to NULL to avoid
1904                    potential bugs */
1905                 if (l >= 4 && ((addr1 & 3) == 0)) {
1906                     /* 32 bit write access */
1907                     val = ldl_p(buf);
1908                     io_mem_write(section->mr, addr1, val, 4);
1909                     l = 4;
1910                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1911                     /* 16 bit write access */
1912                     val = lduw_p(buf);
1913                     io_mem_write(section->mr, addr1, val, 2);
1914                     l = 2;
1915                 } else {
1916                     /* 8 bit write access */
1917                     val = ldub_p(buf);
1918                     io_mem_write(section->mr, addr1, val, 1);
1919                     l = 1;
1920                 }
1921             } else if (!section->readonly) {
1922                 ram_addr_t addr1;
1923                 addr1 = memory_region_get_ram_addr(section->mr)
1924                     + memory_region_section_addr(section, addr);
1925                 /* RAM case */
1926                 ptr = qemu_get_ram_ptr(addr1);
1927                 memcpy(ptr, buf, l);
1928                 invalidate_and_set_dirty(addr1, l);
1929                 qemu_put_ram_ptr(ptr);
1930             }
1931         } else {
1932             if (!(memory_region_is_ram(section->mr) ||
1933                   memory_region_is_romd(section->mr))) {
1934                 hwaddr addr1;
1935                 /* I/O case */
1936                 addr1 = memory_region_section_addr(section, addr);
1937                 if (l >= 4 && ((addr1 & 3) == 0)) {
1938                     /* 32 bit read access */
1939                     val = io_mem_read(section->mr, addr1, 4);
1940                     stl_p(buf, val);
1941                     l = 4;
1942                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1943                     /* 16 bit read access */
1944                     val = io_mem_read(section->mr, addr1, 2);
1945                     stw_p(buf, val);
1946                     l = 2;
1947                 } else {
1948                     /* 8 bit read access */
1949                     val = io_mem_read(section->mr, addr1, 1);
1950                     stb_p(buf, val);
1951                     l = 1;
1952                 }
1953             } else {
1954                 /* RAM case */
1955                 ptr = qemu_get_ram_ptr(section->mr->ram_addr
1956                                        + memory_region_section_addr(section,
1957                                                                     addr));
1958                 memcpy(buf, ptr, l);
1959                 qemu_put_ram_ptr(ptr);
1960             }
1961         }
1962         len -= l;
1963         buf += l;
1964         addr += l;
1965     }
1966 }
1967
1968 void address_space_write(AddressSpace *as, hwaddr addr,
1969                          const uint8_t *buf, int len)
1970 {
1971     address_space_rw(as, addr, (uint8_t *)buf, len, true);
1972 }
1973
1974 /**
1975  * address_space_read: read from an address space.
1976  *
1977  * @as: #AddressSpace to be accessed
1978  * @addr: address within that address space
1979  * @buf: buffer with the data transferred
1980  */
1981 void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1982 {
1983     address_space_rw(as, addr, buf, len, false);
1984 }
1985
1986
1987 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1988                             int len, int is_write)
1989 {
1990     return address_space_rw(&address_space_memory, addr, buf, len, is_write);
1991 }
1992
1993 /* used for ROM loading : can write in RAM and ROM */
1994 void cpu_physical_memory_write_rom(hwaddr addr,
1995                                    const uint8_t *buf, int len)
1996 {
1997     AddressSpaceDispatch *d = address_space_memory.dispatch;
1998     int l;
1999     uint8_t *ptr;
2000     hwaddr page;
2001     MemoryRegionSection *section;
2002
2003     while (len > 0) {
2004         page = addr & TARGET_PAGE_MASK;
2005         l = (page + TARGET_PAGE_SIZE) - addr;
2006         if (l > len)
2007             l = len;
2008         section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2009
2010         if (!(memory_region_is_ram(section->mr) ||
2011               memory_region_is_romd(section->mr))) {
2012             /* do nothing */
2013         } else {
2014             unsigned long addr1;
2015             addr1 = memory_region_get_ram_addr(section->mr)
2016                 + memory_region_section_addr(section, addr);
2017             /* ROM/RAM case */
2018             ptr = qemu_get_ram_ptr(addr1);
2019             memcpy(ptr, buf, l);
2020             invalidate_and_set_dirty(addr1, l);
2021             qemu_put_ram_ptr(ptr);
2022         }
2023         len -= l;
2024         buf += l;
2025         addr += l;
2026     }
2027 }
2028
2029 typedef struct {
2030     void *buffer;
2031     hwaddr addr;
2032     hwaddr len;
2033 } BounceBuffer;
2034
2035 static BounceBuffer bounce;
2036
2037 typedef struct MapClient {
2038     void *opaque;
2039     void (*callback)(void *opaque);
2040     QLIST_ENTRY(MapClient) link;
2041 } MapClient;
2042
2043 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2044     = QLIST_HEAD_INITIALIZER(map_client_list);
2045
2046 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2047 {
2048     MapClient *client = g_malloc(sizeof(*client));
2049
2050     client->opaque = opaque;
2051     client->callback = callback;
2052     QLIST_INSERT_HEAD(&map_client_list, client, link);
2053     return client;
2054 }
2055
2056 static void cpu_unregister_map_client(void *_client)
2057 {
2058     MapClient *client = (MapClient *)_client;
2059
2060     QLIST_REMOVE(client, link);
2061     g_free(client);
2062 }
2063
2064 static void cpu_notify_map_clients(void)
2065 {
2066     MapClient *client;
2067
2068     while (!QLIST_EMPTY(&map_client_list)) {
2069         client = QLIST_FIRST(&map_client_list);
2070         client->callback(client->opaque);
2071         cpu_unregister_map_client(client);
2072     }
2073 }
2074
2075 /* Map a physical memory region into a host virtual address.
2076  * May map a subset of the requested range, given by and returned in *plen.
2077  * May return NULL if resources needed to perform the mapping are exhausted.
2078  * Use only for reads OR writes - not for read-modify-write operations.
2079  * Use cpu_register_map_client() to know when retrying the map operation is
2080  * likely to succeed.
2081  */
2082 void *address_space_map(AddressSpace *as,
2083                         hwaddr addr,
2084                         hwaddr *plen,
2085                         bool is_write)
2086 {
2087     AddressSpaceDispatch *d = as->dispatch;
2088     hwaddr len = *plen;
2089     hwaddr todo = 0;
2090     int l;
2091     hwaddr page;
2092     MemoryRegionSection *section;
2093     ram_addr_t raddr = RAM_ADDR_MAX;
2094     ram_addr_t rlen;
2095     void *ret;
2096
2097     while (len > 0) {
2098         page = addr & TARGET_PAGE_MASK;
2099         l = (page + TARGET_PAGE_SIZE) - addr;
2100         if (l > len)
2101             l = len;
2102         section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2103
2104         if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
2105             if (todo || bounce.buffer) {
2106                 break;
2107             }
2108             bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2109             bounce.addr = addr;
2110             bounce.len = l;
2111             if (!is_write) {
2112                 address_space_read(as, addr, bounce.buffer, l);
2113             }
2114
2115             *plen = l;
2116             return bounce.buffer;
2117         }
2118         if (!todo) {
2119             raddr = memory_region_get_ram_addr(section->mr)
2120                 + memory_region_section_addr(section, addr);
2121         }
2122
2123         len -= l;
2124         addr += l;
2125         todo += l;
2126     }
2127     rlen = todo;
2128     ret = qemu_ram_ptr_length(raddr, &rlen);
2129     *plen = rlen;
2130     return ret;
2131 }
2132
2133 /* Unmaps a memory region previously mapped by address_space_map().
2134  * Will also mark the memory as dirty if is_write == 1.  access_len gives
2135  * the amount of memory that was actually read or written by the caller.
2136  */
2137 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2138                          int is_write, hwaddr access_len)
2139 {
2140     if (buffer != bounce.buffer) {
2141         if (is_write) {
2142             ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2143             while (access_len) {
2144                 unsigned l;
2145                 l = TARGET_PAGE_SIZE;
2146                 if (l > access_len)
2147                     l = access_len;
2148                 invalidate_and_set_dirty(addr1, l);
2149                 addr1 += l;
2150                 access_len -= l;
2151             }
2152         }
2153         if (xen_enabled()) {
2154             xen_invalidate_map_cache_entry(buffer);
2155         }
2156         return;
2157     }
2158     if (is_write) {
2159         address_space_write(as, bounce.addr, bounce.buffer, access_len);
2160     }
2161     qemu_vfree(bounce.buffer);
2162     bounce.buffer = NULL;
2163     cpu_notify_map_clients();
2164 }
2165
2166 void *cpu_physical_memory_map(hwaddr addr,
2167                               hwaddr *plen,
2168                               int is_write)
2169 {
2170     return address_space_map(&address_space_memory, addr, plen, is_write);
2171 }
2172
2173 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2174                                int is_write, hwaddr access_len)
2175 {
2176     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2177 }
2178
2179 /* warning: addr must be aligned */
2180 static inline uint32_t ldl_phys_internal(hwaddr addr,
2181                                          enum device_endian endian)
2182 {
2183     uint8_t *ptr;
2184     uint32_t val;
2185     MemoryRegionSection *section;
2186
2187     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2188
2189     if (!(memory_region_is_ram(section->mr) ||
2190           memory_region_is_romd(section->mr))) {
2191         /* I/O case */
2192         addr = memory_region_section_addr(section, addr);
2193         val = io_mem_read(section->mr, addr, 4);
2194 #if defined(TARGET_WORDS_BIGENDIAN)
2195         if (endian == DEVICE_LITTLE_ENDIAN) {
2196             val = bswap32(val);
2197         }
2198 #else
2199         if (endian == DEVICE_BIG_ENDIAN) {
2200             val = bswap32(val);
2201         }
2202 #endif
2203     } else {
2204         /* RAM case */
2205         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2206                                 & TARGET_PAGE_MASK)
2207                                + memory_region_section_addr(section, addr));
2208         switch (endian) {
2209         case DEVICE_LITTLE_ENDIAN:
2210             val = ldl_le_p(ptr);
2211             break;
2212         case DEVICE_BIG_ENDIAN:
2213             val = ldl_be_p(ptr);
2214             break;
2215         default:
2216             val = ldl_p(ptr);
2217             break;
2218         }
2219     }
2220     return val;
2221 }
2222
2223 uint32_t ldl_phys(hwaddr addr)
2224 {
2225     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2226 }
2227
2228 uint32_t ldl_le_phys(hwaddr addr)
2229 {
2230     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2231 }
2232
2233 uint32_t ldl_be_phys(hwaddr addr)
2234 {
2235     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2236 }
2237
2238 /* warning: addr must be aligned */
2239 static inline uint64_t ldq_phys_internal(hwaddr addr,
2240                                          enum device_endian endian)
2241 {
2242     uint8_t *ptr;
2243     uint64_t val;
2244     MemoryRegionSection *section;
2245
2246     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2247
2248     if (!(memory_region_is_ram(section->mr) ||
2249           memory_region_is_romd(section->mr))) {
2250         /* I/O case */
2251         addr = memory_region_section_addr(section, addr);
2252
2253         /* XXX This is broken when device endian != cpu endian.
2254                Fix and add "endian" variable check */
2255 #ifdef TARGET_WORDS_BIGENDIAN
2256         val = io_mem_read(section->mr, addr, 4) << 32;
2257         val |= io_mem_read(section->mr, addr + 4, 4);
2258 #else
2259         val = io_mem_read(section->mr, addr, 4);
2260         val |= io_mem_read(section->mr, addr + 4, 4) << 32;
2261 #endif
2262     } else {
2263         /* RAM case */
2264         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2265                                 & TARGET_PAGE_MASK)
2266                                + memory_region_section_addr(section, addr));
2267         switch (endian) {
2268         case DEVICE_LITTLE_ENDIAN:
2269             val = ldq_le_p(ptr);
2270             break;
2271         case DEVICE_BIG_ENDIAN:
2272             val = ldq_be_p(ptr);
2273             break;
2274         default:
2275             val = ldq_p(ptr);
2276             break;
2277         }
2278     }
2279     return val;
2280 }
2281
2282 uint64_t ldq_phys(hwaddr addr)
2283 {
2284     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2285 }
2286
2287 uint64_t ldq_le_phys(hwaddr addr)
2288 {
2289     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2290 }
2291
2292 uint64_t ldq_be_phys(hwaddr addr)
2293 {
2294     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2295 }
2296
2297 /* XXX: optimize */
2298 uint32_t ldub_phys(hwaddr addr)
2299 {
2300     uint8_t val;
2301     cpu_physical_memory_read(addr, &val, 1);
2302     return val;
2303 }
2304
2305 /* warning: addr must be aligned */
2306 static inline uint32_t lduw_phys_internal(hwaddr addr,
2307                                           enum device_endian endian)
2308 {
2309     uint8_t *ptr;
2310     uint64_t val;
2311     MemoryRegionSection *section;
2312
2313     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2314
2315     if (!(memory_region_is_ram(section->mr) ||
2316           memory_region_is_romd(section->mr))) {
2317         /* I/O case */
2318         addr = memory_region_section_addr(section, addr);
2319         val = io_mem_read(section->mr, addr, 2);
2320 #if defined(TARGET_WORDS_BIGENDIAN)
2321         if (endian == DEVICE_LITTLE_ENDIAN) {
2322             val = bswap16(val);
2323         }
2324 #else
2325         if (endian == DEVICE_BIG_ENDIAN) {
2326             val = bswap16(val);
2327         }
2328 #endif
2329     } else {
2330         /* RAM case */
2331         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2332                                 & TARGET_PAGE_MASK)
2333                                + memory_region_section_addr(section, addr));
2334         switch (endian) {
2335         case DEVICE_LITTLE_ENDIAN:
2336             val = lduw_le_p(ptr);
2337             break;
2338         case DEVICE_BIG_ENDIAN:
2339             val = lduw_be_p(ptr);
2340             break;
2341         default:
2342             val = lduw_p(ptr);
2343             break;
2344         }
2345     }
2346     return val;
2347 }
2348
2349 uint32_t lduw_phys(hwaddr addr)
2350 {
2351     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2352 }
2353
2354 uint32_t lduw_le_phys(hwaddr addr)
2355 {
2356     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2357 }
2358
2359 uint32_t lduw_be_phys(hwaddr addr)
2360 {
2361     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2362 }
2363
2364 /* warning: addr must be aligned. The ram page is not masked as dirty
2365    and the code inside is not invalidated. It is useful if the dirty
2366    bits are used to track modified PTEs */
2367 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2368 {
2369     uint8_t *ptr;
2370     MemoryRegionSection *section;
2371
2372     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2373
2374     if (!memory_region_is_ram(section->mr) || section->readonly) {
2375         addr = memory_region_section_addr(section, addr);
2376         if (memory_region_is_ram(section->mr)) {
2377             section = &phys_sections[phys_section_rom];
2378         }
2379         io_mem_write(section->mr, addr, val, 4);
2380     } else {
2381         unsigned long addr1 = (memory_region_get_ram_addr(section->mr)
2382                                & TARGET_PAGE_MASK)
2383             + memory_region_section_addr(section, addr);
2384         ptr = qemu_get_ram_ptr(addr1);
2385         stl_p(ptr, val);
2386
2387         if (unlikely(in_migration)) {
2388             if (!cpu_physical_memory_is_dirty(addr1)) {
2389                 /* invalidate code */
2390                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2391                 /* set dirty bit */
2392                 cpu_physical_memory_set_dirty_flags(
2393                     addr1, (0xff & ~CODE_DIRTY_FLAG));
2394             }
2395         }
2396     }
2397 }
2398
2399 void stq_phys_notdirty(hwaddr addr, uint64_t val)
2400 {
2401     uint8_t *ptr;
2402     MemoryRegionSection *section;
2403
2404     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2405
2406     if (!memory_region_is_ram(section->mr) || section->readonly) {
2407         addr = memory_region_section_addr(section, addr);
2408         if (memory_region_is_ram(section->mr)) {
2409             section = &phys_sections[phys_section_rom];
2410         }
2411 #ifdef TARGET_WORDS_BIGENDIAN
2412         io_mem_write(section->mr, addr, val >> 32, 4);
2413         io_mem_write(section->mr, addr + 4, (uint32_t)val, 4);
2414 #else
2415         io_mem_write(section->mr, addr, (uint32_t)val, 4);
2416         io_mem_write(section->mr, addr + 4, val >> 32, 4);
2417 #endif
2418     } else {
2419         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2420                                 & TARGET_PAGE_MASK)
2421                                + memory_region_section_addr(section, addr));
2422         stq_p(ptr, val);
2423     }
2424 }
2425
2426 /* warning: addr must be aligned */
2427 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2428                                      enum device_endian endian)
2429 {
2430     uint8_t *ptr;
2431     MemoryRegionSection *section;
2432
2433     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2434
2435     if (!memory_region_is_ram(section->mr) || section->readonly) {
2436         addr = memory_region_section_addr(section, addr);
2437         if (memory_region_is_ram(section->mr)) {
2438             section = &phys_sections[phys_section_rom];
2439         }
2440 #if defined(TARGET_WORDS_BIGENDIAN)
2441         if (endian == DEVICE_LITTLE_ENDIAN) {
2442             val = bswap32(val);
2443         }
2444 #else
2445         if (endian == DEVICE_BIG_ENDIAN) {
2446             val = bswap32(val);
2447         }
2448 #endif
2449         io_mem_write(section->mr, addr, val, 4);
2450     } else {
2451         unsigned long addr1;
2452         addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2453             + memory_region_section_addr(section, addr);
2454         /* RAM case */
2455         ptr = qemu_get_ram_ptr(addr1);
2456         switch (endian) {
2457         case DEVICE_LITTLE_ENDIAN:
2458             stl_le_p(ptr, val);
2459             break;
2460         case DEVICE_BIG_ENDIAN:
2461             stl_be_p(ptr, val);
2462             break;
2463         default:
2464             stl_p(ptr, val);
2465             break;
2466         }
2467         invalidate_and_set_dirty(addr1, 4);
2468     }
2469 }
2470
2471 void stl_phys(hwaddr addr, uint32_t val)
2472 {
2473     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2474 }
2475
2476 void stl_le_phys(hwaddr addr, uint32_t val)
2477 {
2478     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2479 }
2480
2481 void stl_be_phys(hwaddr addr, uint32_t val)
2482 {
2483     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2484 }
2485
2486 /* XXX: optimize */
2487 void stb_phys(hwaddr addr, uint32_t val)
2488 {
2489     uint8_t v = val;
2490     cpu_physical_memory_write(addr, &v, 1);
2491 }
2492
2493 /* warning: addr must be aligned */
2494 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2495                                      enum device_endian endian)
2496 {
2497     uint8_t *ptr;
2498     MemoryRegionSection *section;
2499
2500     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2501
2502     if (!memory_region_is_ram(section->mr) || section->readonly) {
2503         addr = memory_region_section_addr(section, addr);
2504         if (memory_region_is_ram(section->mr)) {
2505             section = &phys_sections[phys_section_rom];
2506         }
2507 #if defined(TARGET_WORDS_BIGENDIAN)
2508         if (endian == DEVICE_LITTLE_ENDIAN) {
2509             val = bswap16(val);
2510         }
2511 #else
2512         if (endian == DEVICE_BIG_ENDIAN) {
2513             val = bswap16(val);
2514         }
2515 #endif
2516         io_mem_write(section->mr, addr, val, 2);
2517     } else {
2518         unsigned long addr1;
2519         addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2520             + memory_region_section_addr(section, addr);
2521         /* RAM case */
2522         ptr = qemu_get_ram_ptr(addr1);
2523         switch (endian) {
2524         case DEVICE_LITTLE_ENDIAN:
2525             stw_le_p(ptr, val);
2526             break;
2527         case DEVICE_BIG_ENDIAN:
2528             stw_be_p(ptr, val);
2529             break;
2530         default:
2531             stw_p(ptr, val);
2532             break;
2533         }
2534         invalidate_and_set_dirty(addr1, 2);
2535     }
2536 }
2537
2538 void stw_phys(hwaddr addr, uint32_t val)
2539 {
2540     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2541 }
2542
2543 void stw_le_phys(hwaddr addr, uint32_t val)
2544 {
2545     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2546 }
2547
2548 void stw_be_phys(hwaddr addr, uint32_t val)
2549 {
2550     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2551 }
2552
2553 /* XXX: optimize */
2554 void stq_phys(hwaddr addr, uint64_t val)
2555 {
2556     val = tswap64(val);
2557     cpu_physical_memory_write(addr, &val, 8);
2558 }
2559
2560 void stq_le_phys(hwaddr addr, uint64_t val)
2561 {
2562     val = cpu_to_le64(val);
2563     cpu_physical_memory_write(addr, &val, 8);
2564 }
2565
2566 void stq_be_phys(hwaddr addr, uint64_t val)
2567 {
2568     val = cpu_to_be64(val);
2569     cpu_physical_memory_write(addr, &val, 8);
2570 }
2571
2572 /* virtual memory access for debug (includes writing to ROM) */
2573 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2574                         uint8_t *buf, int len, int is_write)
2575 {
2576     int l;
2577     hwaddr phys_addr;
2578     target_ulong page;
2579
2580     while (len > 0) {
2581         page = addr & TARGET_PAGE_MASK;
2582         phys_addr = cpu_get_phys_page_debug(env, page);
2583         /* if no physical page mapped, return an error */
2584         if (phys_addr == -1)
2585             return -1;
2586         l = (page + TARGET_PAGE_SIZE) - addr;
2587         if (l > len)
2588             l = len;
2589         phys_addr += (addr & ~TARGET_PAGE_MASK);
2590         if (is_write)
2591             cpu_physical_memory_write_rom(phys_addr, buf, l);
2592         else
2593             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2594         len -= l;
2595         buf += l;
2596         addr += l;
2597     }
2598     return 0;
2599 }
2600 #endif
2601
2602 #if !defined(CONFIG_USER_ONLY)
2603
2604 /*
2605  * A helper function for the _utterly broken_ virtio device model to find out if
2606  * it's running on a big endian machine. Don't do this at home kids!
2607  */
2608 bool virtio_is_big_endian(void);
2609 bool virtio_is_big_endian(void)
2610 {
2611 #if defined(TARGET_WORDS_BIGENDIAN)
2612     return true;
2613 #else
2614     return false;
2615 #endif
2616 }
2617
2618 #endif
2619
2620 #ifndef CONFIG_USER_ONLY
2621 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2622 {
2623     MemoryRegionSection *section;
2624
2625     section = phys_page_find(address_space_memory.dispatch,
2626                              phys_addr >> TARGET_PAGE_BITS);
2627
2628     return !(memory_region_is_ram(section->mr) ||
2629              memory_region_is_romd(section->mr));
2630 }
2631 #endif