exec.c

   1 /*
   2  *  Virtual page mapping
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "config.h"
  20 #ifdef _WIN32
  21 #include <windows.h>
  22 #else
  23 #include <sys/types.h>
  24 #include <sys/mman.h>
  25 #endif
  26
  27 #include "qemu-common.h"
  28 #include "cpu.h"
  29 #include "tcg.h"
  30 #include "hw/hw.h"
  31 #include "hw/qdev.h"
  32 #include "osdep.h"
  33 #include "kvm.h"
  34 #include "hw/xen.h"
  35 #include "qemu-timer.h"
  36 #include "memory.h"
  37 #include "dma.h"
  38 #include "exec-memory.h"
  39 #if defined(CONFIG_USER_ONLY)
  40 #include <qemu.h>
  41 #else /* !CONFIG_USER_ONLY */
  42 #include "xen-mapcache.h"
  43 #include "trace.h"
  44 #endif
  45
  46 #include "cputlb.h"
  47 #include "translate-all.h"
  48
  49 #include "memory-internal.h"
  50
  51 //#define DEBUG_UNASSIGNED
  52 //#define DEBUG_SUBPAGE
  53
  54 #if !defined(CONFIG_USER_ONLY)
  55 int phys_ram_fd;
  56 static int in_migration;
  57
  58 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
  59
  60 static MemoryRegion *system_memory;
  61 static MemoryRegion *system_io;
  62
  63 AddressSpace address_space_io;
  64 AddressSpace address_space_memory;
  65 DMAContext dma_context_memory;
  66
  67 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
  68 static MemoryRegion io_mem_subpage_ram;
  69
  70 #endif
  71
  72 CPUArchState *first_cpu;
  73 /* current CPU in the current thread. It is only valid inside
  74    cpu_exec() */
  75 DEFINE_TLS(CPUArchState *,cpu_single_env);
  76 /* 0 = Do not count executed instructions.
  77    1 = Precise instruction counting.
  78    2 = Adaptive rate instruction counting.  */
  79 int use_icount = 0;
  80
  81 #if !defined(CONFIG_USER_ONLY)
  82
  83 static MemoryRegionSection *phys_sections;
  84 static unsigned phys_sections_nb, phys_sections_nb_alloc;
  85 static uint16_t phys_section_unassigned;
  86 static uint16_t phys_section_notdirty;
  87 static uint16_t phys_section_rom;
  88 static uint16_t phys_section_watch;
  89
  90 /* Simple allocator for PhysPageEntry nodes */
  91 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
  92 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
  93
  94 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
  95
  96 static void io_mem_init(void);
  97 static void memory_map_init(void);
  98 static void *qemu_safe_ram_ptr(ram_addr_t addr);
  99
 100 static MemoryRegion io_mem_watch;
 101 #endif
 102
 103 #if !defined(CONFIG_USER_ONLY)
 104
 105 static void phys_map_node_reserve(unsigned nodes)
 106 {
 107     if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
 108         typedef PhysPageEntry Node[L2_SIZE];
 109         phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
 110         phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
 111                                       phys_map_nodes_nb + nodes);
 112         phys_map_nodes = g_renew(Node, phys_map_nodes,
 113                                  phys_map_nodes_nb_alloc);
 114     }
 115 }
 116
 117 static uint16_t phys_map_node_alloc(void)
 118 {
 119     unsigned i;
 120     uint16_t ret;
 121
 122     ret = phys_map_nodes_nb++;
 123     assert(ret != PHYS_MAP_NODE_NIL);
 124     assert(ret != phys_map_nodes_nb_alloc);
 125     for (i = 0; i < L2_SIZE; ++i) {
 126         phys_map_nodes[ret][i].is_leaf = 0;
 127         phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
 128     }
 129     return ret;
 130 }
 131
 132 static void phys_map_nodes_reset(void)
 133 {
 134     phys_map_nodes_nb = 0;
 135 }
 136
 137
 138 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
 139                                 hwaddr *nb, uint16_t leaf,
 140                                 int level)
 141 {
 142     PhysPageEntry *p;
 143     int i;
 144     hwaddr step = (hwaddr)1 << (level * L2_BITS);
 145
 146     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
 147         lp->ptr = phys_map_node_alloc();
 148         p = phys_map_nodes[lp->ptr];
 149         if (level == 0) {
 150             for (i = 0; i < L2_SIZE; i++) {
 151                 p[i].is_leaf = 1;
 152                 p[i].ptr = phys_section_unassigned;
 153             }
 154         }
 155     } else {
 156         p = phys_map_nodes[lp->ptr];
 157     }
 158     lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
 159
 160     while (*nb && lp < &p[L2_SIZE]) {
 161         if ((*index & (step - 1)) == 0 && *nb >= step) {
 162             lp->is_leaf = true;
 163             lp->ptr = leaf;
 164             *index += step;
 165             *nb -= step;
 166         } else {
 167             phys_page_set_level(lp, index, nb, leaf, level - 1);
 168         }
 169         ++lp;
 170     }
 171 }
 172
 173 static void phys_page_set(AddressSpaceDispatch *d,
 174                           hwaddr index, hwaddr nb,
 175                           uint16_t leaf)
 176 {
 177     /* Wildly overreserve - it doesn't matter much. */
 178     phys_map_node_reserve(3 * P_L2_LEVELS);
 179
 180     phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 181 }
 182
 183 MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
 184 {
 185     PhysPageEntry lp = d->phys_map;
 186     PhysPageEntry *p;
 187     int i;
 188     uint16_t s_index = phys_section_unassigned;
 189
 190     for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
 191         if (lp.ptr == PHYS_MAP_NODE_NIL) {
 192             goto not_found;
 193         }
 194         p = phys_map_nodes[lp.ptr];
 195         lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
 196     }
 197
 198     s_index = lp.ptr;
 199 not_found:
 200     return &phys_sections[s_index];
 201 }
 202
 203 bool memory_region_is_unassigned(MemoryRegion *mr)
 204 {
 205     return mr != &io_mem_ram && mr != &io_mem_rom
 206         && mr != &io_mem_notdirty && !mr->rom_device
 207         && mr != &io_mem_watch;
 208 }
 209 #endif
 210
 211 void cpu_exec_init_all(void)
 212 {
 213 #if !defined(CONFIG_USER_ONLY)
 214     memory_map_init();
 215     io_mem_init();
 216 #endif
 217 }
 218
 219 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 220
 221 static int cpu_common_post_load(void *opaque, int version_id)
 222 {
 223     CPUArchState *env = opaque;
 224
 225     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
 226        version_id is increased. */
 227     env->interrupt_request &= ~0x01;
 228     tlb_flush(env, 1);
 229
 230     return 0;
 231 }
 232
 233 static const VMStateDescription vmstate_cpu_common = {
 234     .name = "cpu_common",
 235     .version_id = 1,
 236     .minimum_version_id = 1,
 237     .minimum_version_id_old = 1,
 238     .post_load = cpu_common_post_load,
 239     .fields      = (VMStateField []) {
 240         VMSTATE_UINT32(halted, CPUArchState),
 241         VMSTATE_UINT32(interrupt_request, CPUArchState),
 242         VMSTATE_END_OF_LIST()
 243     }
 244 };
 245 #endif
 246
 247 CPUArchState *qemu_get_cpu(int cpu)
 248 {
 249     CPUArchState *env = first_cpu;
 250
 251     while (env) {
 252         if (env->cpu_index == cpu)
 253             break;
 254         env = env->next_cpu;
 255     }
 256
 257     return env;
 258 }
 259
 260 void cpu_exec_init(CPUArchState *env)
 261 {
 262 #ifndef CONFIG_USER_ONLY
 263     CPUState *cpu = ENV_GET_CPU(env);
 264 #endif
 265     CPUArchState **penv;
 266     int cpu_index;
 267
 268 #if defined(CONFIG_USER_ONLY)
 269     cpu_list_lock();
 270 #endif
 271     env->next_cpu = NULL;
 272     penv = &first_cpu;
 273     cpu_index = 0;
 274     while (*penv != NULL) {
 275         penv = &(*penv)->next_cpu;
 276         cpu_index++;
 277     }
 278     env->cpu_index = cpu_index;
 279     env->numa_node = 0;
 280     QTAILQ_INIT(&env->breakpoints);
 281     QTAILQ_INIT(&env->watchpoints);
 282 #ifndef CONFIG_USER_ONLY
 283     cpu->thread_id = qemu_get_thread_id();
 284 #endif
 285     *penv = env;
 286 #if defined(CONFIG_USER_ONLY)
 287     cpu_list_unlock();
 288 #endif
 289 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 290     vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
 291     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
 292                     cpu_save, cpu_load, env);
 293 #endif
 294 }
 295
 296 #if defined(TARGET_HAS_ICE)
 297 #if defined(CONFIG_USER_ONLY)
 298 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 299 {
 300     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 301 }
 302 #else
 303 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 304 {
 305     tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
 306             (pc & ~TARGET_PAGE_MASK));
 307 }
 308 #endif
 309 #endif /* TARGET_HAS_ICE */
 310
 311 #if defined(CONFIG_USER_ONLY)
 312 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 313
 314 {
 315 }
 316
 317 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 318                           int flags, CPUWatchpoint **watchpoint)
 319 {
 320     return -ENOSYS;
 321 }
 322 #else
 323 /* Add a watchpoint.  */
 324 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 325                           int flags, CPUWatchpoint **watchpoint)
 326 {
 327     target_ulong len_mask = ~(len - 1);
 328     CPUWatchpoint *wp;
 329
 330     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
 331     if ((len & (len - 1)) || (addr & ~len_mask) ||
 332             len == 0 || len > TARGET_PAGE_SIZE) {
 333         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
 334                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
 335         return -EINVAL;
 336     }
 337     wp = g_malloc(sizeof(*wp));
 338
 339     wp->vaddr = addr;
 340     wp->len_mask = len_mask;
 341     wp->flags = flags;
 342
 343     /* keep all GDB-injected watchpoints in front */
 344     if (flags & BP_GDB)
 345         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
 346     else
 347         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
 348
 349     tlb_flush_page(env, addr);
 350
 351     if (watchpoint)
 352         *watchpoint = wp;
 353     return 0;
 354 }
 355
 356 /* Remove a specific watchpoint.  */
 357 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
 358                           int flags)
 359 {
 360     target_ulong len_mask = ~(len - 1);
 361     CPUWatchpoint *wp;
 362
 363     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 364         if (addr == wp->vaddr && len_mask == wp->len_mask
 365                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
 366             cpu_watchpoint_remove_by_ref(env, wp);
 367             return 0;
 368         }
 369     }
 370     return -ENOENT;
 371 }
 372
 373 /* Remove a specific watchpoint by reference.  */
 374 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
 375 {
 376     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
 377
 378     tlb_flush_page(env, watchpoint->vaddr);
 379
 380     g_free(watchpoint);
 381 }
 382
 383 /* Remove all matching watchpoints.  */
 384 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 385 {
 386     CPUWatchpoint *wp, *next;
 387
 388     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
 389         if (wp->flags & mask)
 390             cpu_watchpoint_remove_by_ref(env, wp);
 391     }
 392 }
 393 #endif
 394
 395 /* Add a breakpoint.  */
 396 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
 397                           CPUBreakpoint **breakpoint)
 398 {
 399 #if defined(TARGET_HAS_ICE)
 400     CPUBreakpoint *bp;
 401
 402     bp = g_malloc(sizeof(*bp));
 403
 404     bp->pc = pc;
 405     bp->flags = flags;
 406
 407     /* keep all GDB-injected breakpoints in front */
 408     if (flags & BP_GDB)
 409         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
 410     else
 411         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
 412
 413     breakpoint_invalidate(env, pc);
 414
 415     if (breakpoint)
 416         *breakpoint = bp;
 417     return 0;
 418 #else
 419     return -ENOSYS;
 420 #endif
 421 }
 422
 423 /* Remove a specific breakpoint.  */
 424 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
 425 {
 426 #if defined(TARGET_HAS_ICE)
 427     CPUBreakpoint *bp;
 428
 429     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 430         if (bp->pc == pc && bp->flags == flags) {
 431             cpu_breakpoint_remove_by_ref(env, bp);
 432             return 0;
 433         }
 434     }
 435     return -ENOENT;
 436 #else
 437     return -ENOSYS;
 438 #endif
 439 }
 440
 441 /* Remove a specific breakpoint by reference.  */
 442 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
 443 {
 444 #if defined(TARGET_HAS_ICE)
 445     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
 446
 447     breakpoint_invalidate(env, breakpoint->pc);
 448
 449     g_free(breakpoint);
 450 #endif
 451 }
 452
 453 /* Remove all matching breakpoints. */
 454 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
 455 {
 456 #if defined(TARGET_HAS_ICE)
 457     CPUBreakpoint *bp, *next;
 458
 459     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
 460         if (bp->flags & mask)
 461             cpu_breakpoint_remove_by_ref(env, bp);
 462     }
 463 #endif
 464 }
 465
 466 /* enable or disable single step mode. EXCP_DEBUG is returned by the
 467    CPU loop after each instruction */
 468 void cpu_single_step(CPUArchState *env, int enabled)
 469 {
 470 #if defined(TARGET_HAS_ICE)
 471     if (env->singlestep_enabled != enabled) {
 472         env->singlestep_enabled = enabled;
 473         if (kvm_enabled())
 474             kvm_update_guest_debug(env, 0);
 475         else {
 476             /* must flush all the translated code to avoid inconsistencies */
 477             /* XXX: only flush what is necessary */
 478             tb_flush(env);
 479         }
 480     }
 481 #endif
 482 }
 483
 484 void cpu_reset_interrupt(CPUArchState *env, int mask)
 485 {
 486     env->interrupt_request &= ~mask;
 487 }
 488
 489 void cpu_exit(CPUArchState *env)
 490 {
 491     env->exit_request = 1;
 492     cpu_unlink_tb(env);
 493 }
 494
 495 void cpu_abort(CPUArchState *env, const char *fmt, ...)
 496 {
 497     va_list ap;
 498     va_list ap2;
 499
 500     va_start(ap, fmt);
 501     va_copy(ap2, ap);
 502     fprintf(stderr, "qemu: fatal: ");
 503     vfprintf(stderr, fmt, ap);
 504     fprintf(stderr, "\n");
 505     cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 506     if (qemu_log_enabled()) {
 507         qemu_log("qemu: fatal: ");
 508         qemu_log_vprintf(fmt, ap2);
 509         qemu_log("\n");
 510         log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 511         qemu_log_flush();
 512         qemu_log_close();
 513     }
 514     va_end(ap2);
 515     va_end(ap);
 516 #if defined(CONFIG_USER_ONLY)
 517     {
 518         struct sigaction act;
 519         sigfillset(&act.sa_mask);
 520         act.sa_handler = SIG_DFL;
 521         sigaction(SIGABRT, &act, NULL);
 522     }
 523 #endif
 524     abort();
 525 }
 526
 527 CPUArchState *cpu_copy(CPUArchState *env)
 528 {
 529     CPUArchState *new_env = cpu_init(env->cpu_model_str);
 530     CPUArchState *next_cpu = new_env->next_cpu;
 531     int cpu_index = new_env->cpu_index;
 532 #if defined(TARGET_HAS_ICE)
 533     CPUBreakpoint *bp;
 534     CPUWatchpoint *wp;
 535 #endif
 536
 537     memcpy(new_env, env, sizeof(CPUArchState));
 538
 539     /* Preserve chaining and index. */
 540     new_env->next_cpu = next_cpu;
 541     new_env->cpu_index = cpu_index;
 542
 543     /* Clone all break/watchpoints.
 544        Note: Once we support ptrace with hw-debug register access, make sure
 545        BP_CPU break/watchpoints are handled correctly on clone. */
 546     QTAILQ_INIT(&env->breakpoints);
 547     QTAILQ_INIT(&env->watchpoints);
 548 #if defined(TARGET_HAS_ICE)
 549     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 550         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
 551     }
 552     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 553         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
 554                               wp->flags, NULL);
 555     }
 556 #endif
 557
 558     return new_env;
 559 }
 560
 561 #if !defined(CONFIG_USER_ONLY)
 562 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
 563                                       uintptr_t length)
 564 {
 565     uintptr_t start1;
 566
 567     /* we modify the TLB cache so that the dirty bit will be set again
 568        when accessing the range */
 569     start1 = (uintptr_t)qemu_safe_ram_ptr(start);
 570     /* Check that we don't span multiple blocks - this breaks the
 571        address comparisons below.  */
 572     if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
 573             != (end - 1) - start) {
 574         abort();
 575     }
 576     cpu_tlb_reset_dirty_all(start1, length);
 577
 578 }
 579
 580 /* Note: start and end must be within the same ram block.  */
 581 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
 582                                      int dirty_flags)
 583 {
 584     uintptr_t length;
 585
 586     start &= TARGET_PAGE_MASK;
 587     end = TARGET_PAGE_ALIGN(end);
 588
 589     length = end - start;
 590     if (length == 0)
 591         return;
 592     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
 593
 594     if (tcg_enabled()) {
 595         tlb_reset_dirty_range_all(start, end, length);
 596     }
 597 }
 598
 599 static int cpu_physical_memory_set_dirty_tracking(int enable)
 600 {
 601     int ret = 0;
 602     in_migration = enable;
 603     return ret;
 604 }
 605
 606 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 607                                                    MemoryRegionSection *section,
 608                                                    target_ulong vaddr,
 609                                                    hwaddr paddr,
 610                                                    int prot,
 611                                                    target_ulong *address)
 612 {
 613     hwaddr iotlb;
 614     CPUWatchpoint *wp;
 615
 616     if (memory_region_is_ram(section->mr)) {
 617         /* Normal RAM.  */
 618         iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
 619             + memory_region_section_addr(section, paddr);
 620         if (!section->readonly) {
 621             iotlb |= phys_section_notdirty;
 622         } else {
 623             iotlb |= phys_section_rom;
 624         }
 625     } else {
 626         /* IO handlers are currently passed a physical address.
 627            It would be nice to pass an offset from the base address
 628            of that region.  This would avoid having to special case RAM,
 629            and avoid full address decoding in every device.
 630            We can't use the high bits of pd for this because
 631            IO_MEM_ROMD uses these as a ram address.  */
 632         iotlb = section - phys_sections;
 633         iotlb += memory_region_section_addr(section, paddr);
 634     }
 635
 636     /* Make accesses to pages with watchpoints go via the
 637        watchpoint trap routines.  */
 638     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 639         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
 640             /* Avoid trapping reads of pages with a write breakpoint. */
 641             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
 642                 iotlb = phys_section_watch + paddr;
 643                 *address |= TLB_MMIO;
 644                 break;
 645             }
 646         }
 647     }
 648
 649     return iotlb;
 650 }
 651 #endif /* defined(CONFIG_USER_ONLY) */
 652
 653 #if !defined(CONFIG_USER_ONLY)
 654
 655 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 656 typedef struct subpage_t {
 657     MemoryRegion iomem;
 658     hwaddr base;
 659     uint16_t sub_section[TARGET_PAGE_SIZE];
 660 } subpage_t;
 661
 662 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
 663                              uint16_t section);
 664 static subpage_t *subpage_init(hwaddr base);
 665 static void destroy_page_desc(uint16_t section_index)
 666 {
 667     MemoryRegionSection *section = &phys_sections[section_index];
 668     MemoryRegion *mr = section->mr;
 669
 670     if (mr->subpage) {
 671         subpage_t *subpage = container_of(mr, subpage_t, iomem);
 672         memory_region_destroy(&subpage->iomem);
 673         g_free(subpage);
 674     }
 675 }
 676
 677 static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
 678 {
 679     unsigned i;
 680     PhysPageEntry *p;
 681
 682     if (lp->ptr == PHYS_MAP_NODE_NIL) {
 683         return;
 684     }
 685
 686     p = phys_map_nodes[lp->ptr];
 687     for (i = 0; i < L2_SIZE; ++i) {
 688         if (!p[i].is_leaf) {
 689             destroy_l2_mapping(&p[i], level - 1);
 690         } else {
 691             destroy_page_desc(p[i].ptr);
 692         }
 693     }
 694     lp->is_leaf = 0;
 695     lp->ptr = PHYS_MAP_NODE_NIL;
 696 }
 697
 698 static void destroy_all_mappings(AddressSpaceDispatch *d)
 699 {
 700     destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
 701     phys_map_nodes_reset();
 702 }
 703
 704 static uint16_t phys_section_add(MemoryRegionSection *section)
 705 {
 706     if (phys_sections_nb == phys_sections_nb_alloc) {
 707         phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
 708         phys_sections = g_renew(MemoryRegionSection, phys_sections,
 709                                 phys_sections_nb_alloc);
 710     }
 711     phys_sections[phys_sections_nb] = *section;
 712     return phys_sections_nb++;
 713 }
 714
 715 static void phys_sections_clear(void)
 716 {
 717     phys_sections_nb = 0;
 718 }
 719
 720 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 721 {
 722     subpage_t *subpage;
 723     hwaddr base = section->offset_within_address_space
 724         & TARGET_PAGE_MASK;
 725     MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
 726     MemoryRegionSection subsection = {
 727         .offset_within_address_space = base,
 728         .size = TARGET_PAGE_SIZE,
 729     };
 730     hwaddr start, end;
 731
 732     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 733
 734     if (!(existing->mr->subpage)) {
 735         subpage = subpage_init(base);
 736         subsection.mr = &subpage->iomem;
 737         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
 738                       phys_section_add(&subsection));
 739     } else {
 740         subpage = container_of(existing->mr, subpage_t, iomem);
 741     }
 742     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
 743     end = start + section->size - 1;
 744     subpage_register(subpage, start, end, phys_section_add(section));
 745 }
 746
 747
 748 static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 749 {
 750     hwaddr start_addr = section->offset_within_address_space;
 751     ram_addr_t size = section->size;
 752     hwaddr addr;
 753     uint16_t section_index = phys_section_add(section);
 754
 755     assert(size);
 756
 757     addr = start_addr;
 758     phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
 759                   section_index);
 760 }
 761
 762 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 763 {
 764     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
 765     MemoryRegionSection now = *section, remain = *section;
 766
 767     if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
 768         || (now.size < TARGET_PAGE_SIZE)) {
 769         now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
 770                        - now.offset_within_address_space,
 771                        now.size);
 772         register_subpage(d, &now);
 773         remain.size -= now.size;
 774         remain.offset_within_address_space += now.size;
 775         remain.offset_within_region += now.size;
 776     }
 777     while (remain.size >= TARGET_PAGE_SIZE) {
 778         now = remain;
 779         if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
 780             now.size = TARGET_PAGE_SIZE;
 781             register_subpage(d, &now);
 782         } else {
 783             now.size &= TARGET_PAGE_MASK;
 784             register_multipage(d, &now);
 785         }
 786         remain.size -= now.size;
 787         remain.offset_within_address_space += now.size;
 788         remain.offset_within_region += now.size;
 789     }
 790     now = remain;
 791     if (now.size) {
 792         register_subpage(d, &now);
 793     }
 794 }
 795
 796 void qemu_flush_coalesced_mmio_buffer(void)
 797 {
 798     if (kvm_enabled())
 799         kvm_flush_coalesced_mmio_buffer();
 800 }
 801
 802 #if defined(__linux__) && !defined(TARGET_S390X)
 803
 804 #include <sys/vfs.h>
 805
 806 #define HUGETLBFS_MAGIC       0x958458f6
 807
 808 static long gethugepagesize(const char *path)
 809 {
 810     struct statfs fs;
 811     int ret;
 812
 813     do {
 814         ret = statfs(path, &fs);
 815     } while (ret != 0 && errno == EINTR);
 816
 817     if (ret != 0) {
 818         perror(path);
 819         return 0;
 820     }
 821
 822     if (fs.f_type != HUGETLBFS_MAGIC)
 823         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
 824
 825     return fs.f_bsize;
 826 }
 827
 828 static void *file_ram_alloc(RAMBlock *block,
 829                             ram_addr_t memory,
 830                             const char *path)
 831 {
 832     char *filename;
 833     void *area;
 834     int fd;
 835 #ifdef MAP_POPULATE
 836     int flags;
 837 #endif
 838     unsigned long hpagesize;
 839
 840     hpagesize = gethugepagesize(path);
 841     if (!hpagesize) {
 842         return NULL;
 843     }
 844
 845     if (memory < hpagesize) {
 846         return NULL;
 847     }
 848
 849     if (kvm_enabled() && !kvm_has_sync_mmu()) {
 850         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
 851         return NULL;
 852     }
 853
 854     if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
 855         return NULL;
 856     }
 857
 858     fd = mkstemp(filename);
 859     if (fd < 0) {
 860         perror("unable to create backing store for hugepages");
 861         free(filename);
 862         return NULL;
 863     }
 864     unlink(filename);
 865     free(filename);
 866
 867     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 868
 869     /*
 870      * ftruncate is not supported by hugetlbfs in older
 871      * hosts, so don't bother bailing out on errors.
 872      * If anything goes wrong with it under other filesystems,
 873      * mmap will fail.
 874      */
 875     if (ftruncate(fd, memory))
 876         perror("ftruncate");
 877
 878 #ifdef MAP_POPULATE
 879     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
 880      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
 881      * to sidestep this quirk.
 882      */
 883     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
 884     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
 885 #else
 886     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 887 #endif
 888     if (area == MAP_FAILED) {
 889         perror("file_ram_alloc: can't mmap RAM pages");
 890         close(fd);
 891         return (NULL);
 892     }
 893     block->fd = fd;
 894     return area;
 895 }
 896 #endif
 897
 898 static ram_addr_t find_ram_offset(ram_addr_t size)
 899 {
 900     RAMBlock *block, *next_block;
 901     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
 902
 903     if (QLIST_EMPTY(&ram_list.blocks))
 904         return 0;
 905
 906     QLIST_FOREACH(block, &ram_list.blocks, next) {
 907         ram_addr_t end, next = RAM_ADDR_MAX;
 908
 909         end = block->offset + block->length;
 910
 911         QLIST_FOREACH(next_block, &ram_list.blocks, next) {
 912             if (next_block->offset >= end) {
 913                 next = MIN(next, next_block->offset);
 914             }
 915         }
 916         if (next - end >= size && next - end < mingap) {
 917             offset = end;
 918             mingap = next - end;
 919         }
 920     }
 921
 922     if (offset == RAM_ADDR_MAX) {
 923         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
 924                 (uint64_t)size);
 925         abort();
 926     }
 927
 928     return offset;
 929 }
 930
 931 ram_addr_t last_ram_offset(void)
 932 {
 933     RAMBlock *block;
 934     ram_addr_t last = 0;
 935
 936     QLIST_FOREACH(block, &ram_list.blocks, next)
 937         last = MAX(last, block->offset + block->length);
 938
 939     return last;
 940 }
 941
 942 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
 943 {
 944     int ret;
 945     QemuOpts *machine_opts;
 946
 947     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
 948     machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
 949     if (machine_opts &&
 950         !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
 951         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
 952         if (ret) {
 953             perror("qemu_madvise");
 954             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
 955                             "but dump_guest_core=off specified\n");
 956         }
 957     }
 958 }
 959
 960 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
 961 {
 962     RAMBlock *new_block, *block;
 963
 964     new_block = NULL;
 965     QLIST_FOREACH(block, &ram_list.blocks, next) {
 966         if (block->offset == addr) {
 967             new_block = block;
 968             break;
 969         }
 970     }
 971     assert(new_block);
 972     assert(!new_block->idstr[0]);
 973
 974     if (dev) {
 975         char *id = qdev_get_dev_path(dev);
 976         if (id) {
 977             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
 978             g_free(id);
 979         }
 980     }
 981     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
 982
 983     QLIST_FOREACH(block, &ram_list.blocks, next) {
 984         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
 985             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
 986                     new_block->idstr);
 987             abort();
 988         }
 989     }
 990 }
 991
 992 static int memory_try_enable_merging(void *addr, size_t len)
 993 {
 994     QemuOpts *opts;
 995
 996     opts = qemu_opts_find(qemu_find_opts("machine"), 0);
 997     if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
 998         /* disabled by the user */
 999         return 0;
1000     }
1001
1002     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1003 }
1004
1005 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1006                                    MemoryRegion *mr)
1007 {
1008     RAMBlock *new_block;
1009
1010     size = TARGET_PAGE_ALIGN(size);
1011     new_block = g_malloc0(sizeof(*new_block));
1012
1013     new_block->mr = mr;
1014     new_block->offset = find_ram_offset(size);
1015     if (host) {
1016         new_block->host = host;
1017         new_block->flags |= RAM_PREALLOC_MASK;
1018     } else {
1019         if (mem_path) {
1020 #if defined (__linux__) && !defined(TARGET_S390X)
1021             new_block->host = file_ram_alloc(new_block, size, mem_path);
1022             if (!new_block->host) {
1023                 new_block->host = qemu_vmalloc(size);
1024                 memory_try_enable_merging(new_block->host, size);
1025             }
1026 #else
1027             fprintf(stderr, "-mem-path option unsupported\n");
1028             exit(1);
1029 #endif
1030         } else {
1031             if (xen_enabled()) {
1032                 xen_ram_alloc(new_block->offset, size, mr);
1033             } else if (kvm_enabled()) {
1034                 /* some s390/kvm configurations have special constraints */
1035                 new_block->host = kvm_vmalloc(size);
1036             } else {
1037                 new_block->host = qemu_vmalloc(size);
1038             }
1039             memory_try_enable_merging(new_block->host, size);
1040         }
1041     }
1042     new_block->length = size;
1043
1044     QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
1045
1046     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1047                                        last_ram_offset() >> TARGET_PAGE_BITS);
1048     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1049            0, size >> TARGET_PAGE_BITS);
1050     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1051
1052     qemu_ram_setup_dump(new_block->host, size);
1053     qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1054
1055     if (kvm_enabled())
1056         kvm_setup_guest_memory(new_block->host, size);
1057
1058     return new_block->offset;
1059 }
1060
1061 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1062 {
1063     return qemu_ram_alloc_from_ptr(size, NULL, mr);
1064 }
1065
1066 void qemu_ram_free_from_ptr(ram_addr_t addr)
1067 {
1068     RAMBlock *block;
1069
1070     QLIST_FOREACH(block, &ram_list.blocks, next) {
1071         if (addr == block->offset) {
1072             QLIST_REMOVE(block, next);
1073             g_free(block);
1074             return;
1075         }
1076     }
1077 }
1078
1079 void qemu_ram_free(ram_addr_t addr)
1080 {
1081     RAMBlock *block;
1082
1083     QLIST_FOREACH(block, &ram_list.blocks, next) {
1084         if (addr == block->offset) {
1085             QLIST_REMOVE(block, next);
1086             if (block->flags & RAM_PREALLOC_MASK) {
1087                 ;
1088             } else if (mem_path) {
1089 #if defined (__linux__) && !defined(TARGET_S390X)
1090                 if (block->fd) {
1091                     munmap(block->host, block->length);
1092                     close(block->fd);
1093                 } else {
1094                     qemu_vfree(block->host);
1095                 }
1096 #else
1097                 abort();
1098 #endif
1099             } else {
1100 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1101                 munmap(block->host, block->length);
1102 #else
1103                 if (xen_enabled()) {
1104                     xen_invalidate_map_cache_entry(block->host);
1105                 } else {
1106                     qemu_vfree(block->host);
1107                 }
1108 #endif
1109             }
1110             g_free(block);
1111             return;
1112         }
1113     }
1114
1115 }
1116
1117 #ifndef _WIN32
1118 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1119 {
1120     RAMBlock *block;
1121     ram_addr_t offset;
1122     int flags;
1123     void *area, *vaddr;
1124
1125     QLIST_FOREACH(block, &ram_list.blocks, next) {
1126         offset = addr - block->offset;
1127         if (offset < block->length) {
1128             vaddr = block->host + offset;
1129             if (block->flags & RAM_PREALLOC_MASK) {
1130                 ;
1131             } else {
1132                 flags = MAP_FIXED;
1133                 munmap(vaddr, length);
1134                 if (mem_path) {
1135 #if defined(__linux__) && !defined(TARGET_S390X)
1136                     if (block->fd) {
1137 #ifdef MAP_POPULATE
1138                         flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1139                             MAP_PRIVATE;
1140 #else
1141                         flags |= MAP_PRIVATE;
1142 #endif
1143                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1144                                     flags, block->fd, offset);
1145                     } else {
1146                         flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1147                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1148                                     flags, -1, 0);
1149                     }
1150 #else
1151                     abort();
1152 #endif
1153                 } else {
1154 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1155                     flags |= MAP_SHARED | MAP_ANONYMOUS;
1156                     area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1157                                 flags, -1, 0);
1158 #else
1159                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1160                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1161                                 flags, -1, 0);
1162 #endif
1163                 }
1164                 if (area != vaddr) {
1165                     fprintf(stderr, "Could not remap addr: "
1166                             RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1167                             length, addr);
1168                     exit(1);
1169                 }
1170                 memory_try_enable_merging(vaddr, length);
1171                 qemu_ram_setup_dump(vaddr, length);
1172             }
1173             return;
1174         }
1175     }
1176 }
1177 #endif /* !_WIN32 */
1178
1179 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1180    With the exception of the softmmu code in this file, this should
1181    only be used for local memory (e.g. video ram) that the device owns,
1182    and knows it isn't going to access beyond the end of the block.
1183
1184    It should not be used for general purpose DMA.
1185    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1186  */
1187 void *qemu_get_ram_ptr(ram_addr_t addr)
1188 {
1189     RAMBlock *block;
1190
1191     QLIST_FOREACH(block, &ram_list.blocks, next) {
1192         if (addr - block->offset < block->length) {
1193             /* Move this entry to to start of the list.  */
1194             if (block != QLIST_FIRST(&ram_list.blocks)) {
1195                 QLIST_REMOVE(block, next);
1196                 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
1197             }
1198             if (xen_enabled()) {
1199                 /* We need to check if the requested address is in the RAM
1200                  * because we don't want to map the entire memory in QEMU.
1201                  * In that case just map until the end of the page.
1202                  */
1203                 if (block->offset == 0) {
1204                     return xen_map_cache(addr, 0, 0);
1205                 } else if (block->host == NULL) {
1206                     block->host =
1207                         xen_map_cache(block->offset, block->length, 1);
1208                 }
1209             }
1210             return block->host + (addr - block->offset);
1211         }
1212     }
1213
1214     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1215     abort();
1216
1217     return NULL;
1218 }
1219
1220 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1221  * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
1222  */
1223 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1224 {
1225     RAMBlock *block;
1226
1227     QLIST_FOREACH(block, &ram_list.blocks, next) {
1228         if (addr - block->offset < block->length) {
1229             if (xen_enabled()) {
1230                 /* We need to check if the requested address is in the RAM
1231                  * because we don't want to map the entire memory in QEMU.
1232                  * In that case just map until the end of the page.
1233                  */
1234                 if (block->offset == 0) {
1235                     return xen_map_cache(addr, 0, 0);
1236                 } else if (block->host == NULL) {
1237                     block->host =
1238                         xen_map_cache(block->offset, block->length, 1);
1239                 }
1240             }
1241             return block->host + (addr - block->offset);
1242         }
1243     }
1244
1245     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1246     abort();
1247
1248     return NULL;
1249 }
1250
1251 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1252  * but takes a size argument */
1253 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1254 {
1255     if (*size == 0) {
1256         return NULL;
1257     }
1258     if (xen_enabled()) {
1259         return xen_map_cache(addr, *size, 1);
1260     } else {
1261         RAMBlock *block;
1262
1263         QLIST_FOREACH(block, &ram_list.blocks, next) {
1264             if (addr - block->offset < block->length) {
1265                 if (addr - block->offset + *size > block->length)
1266                     *size = block->length - addr + block->offset;
1267                 return block->host + (addr - block->offset);
1268             }
1269         }
1270
1271         fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1272         abort();
1273     }
1274 }
1275
1276 void qemu_put_ram_ptr(void *addr)
1277 {
1278     trace_qemu_put_ram_ptr(addr);
1279 }
1280
1281 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1282 {
1283     RAMBlock *block;
1284     uint8_t *host = ptr;
1285
1286     if (xen_enabled()) {
1287         *ram_addr = xen_ram_addr_from_mapcache(ptr);
1288         return 0;
1289     }
1290
1291     QLIST_FOREACH(block, &ram_list.blocks, next) {
1292         /* This case append when the block is not mapped. */
1293         if (block->host == NULL) {
1294             continue;
1295         }
1296         if (host - block->host < block->length) {
1297             *ram_addr = block->offset + (host - block->host);
1298             return 0;
1299         }
1300     }
1301
1302     return -1;
1303 }
1304
1305 /* Some of the softmmu routines need to translate from a host pointer
1306    (typically a TLB entry) back to a ram offset.  */
1307 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1308 {
1309     ram_addr_t ram_addr;
1310
1311     if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1312         fprintf(stderr, "Bad ram pointer %p\n", ptr);
1313         abort();
1314     }
1315     return ram_addr;
1316 }
1317
1318 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1319                                     unsigned size)
1320 {
1321 #ifdef DEBUG_UNASSIGNED
1322     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1323 #endif
1324 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1325     cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, size);
1326 #endif
1327     return 0;
1328 }
1329
1330 static void unassigned_mem_write(void *opaque, hwaddr addr,
1331                                  uint64_t val, unsigned size)
1332 {
1333 #ifdef DEBUG_UNASSIGNED
1334     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1335 #endif
1336 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1337     cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, size);
1338 #endif
1339 }
1340
1341 static const MemoryRegionOps unassigned_mem_ops = {
1342     .read = unassigned_mem_read,
1343     .write = unassigned_mem_write,
1344     .endianness = DEVICE_NATIVE_ENDIAN,
1345 };
1346
1347 static uint64_t error_mem_read(void *opaque, hwaddr addr,
1348                                unsigned size)
1349 {
1350     abort();
1351 }
1352
1353 static void error_mem_write(void *opaque, hwaddr addr,
1354                             uint64_t value, unsigned size)
1355 {
1356     abort();
1357 }
1358
1359 static const MemoryRegionOps error_mem_ops = {
1360     .read = error_mem_read,
1361     .write = error_mem_write,
1362     .endianness = DEVICE_NATIVE_ENDIAN,
1363 };
1364
1365 static const MemoryRegionOps rom_mem_ops = {
1366     .read = error_mem_read,
1367     .write = unassigned_mem_write,
1368     .endianness = DEVICE_NATIVE_ENDIAN,
1369 };
1370
1371 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1372                                uint64_t val, unsigned size)
1373 {
1374     int dirty_flags;
1375     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1376     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1377 #if !defined(CONFIG_USER_ONLY)
1378         tb_invalidate_phys_page_fast(ram_addr, size);
1379         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1380 #endif
1381     }
1382     switch (size) {
1383     case 1:
1384         stb_p(qemu_get_ram_ptr(ram_addr), val);
1385         break;
1386     case 2:
1387         stw_p(qemu_get_ram_ptr(ram_addr), val);
1388         break;
1389     case 4:
1390         stl_p(qemu_get_ram_ptr(ram_addr), val);
1391         break;
1392     default:
1393         abort();
1394     }
1395     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1396     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1397     /* we remove the notdirty callback only if the code has been
1398        flushed */
1399     if (dirty_flags == 0xff)
1400         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1401 }
1402
1403 static const MemoryRegionOps notdirty_mem_ops = {
1404     .read = error_mem_read,
1405     .write = notdirty_mem_write,
1406     .endianness = DEVICE_NATIVE_ENDIAN,
1407 };
1408
1409 /* Generate a debug exception if a watchpoint has been hit.  */
1410 static void check_watchpoint(int offset, int len_mask, int flags)
1411 {
1412     CPUArchState *env = cpu_single_env;
1413     target_ulong pc, cs_base;
1414     target_ulong vaddr;
1415     CPUWatchpoint *wp;
1416     int cpu_flags;
1417
1418     if (env->watchpoint_hit) {
1419         /* We re-entered the check after replacing the TB. Now raise
1420          * the debug interrupt so that is will trigger after the
1421          * current instruction. */
1422         cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
1423         return;
1424     }
1425     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1426     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1427         if ((vaddr == (wp->vaddr & len_mask) ||
1428              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1429             wp->flags |= BP_WATCHPOINT_HIT;
1430             if (!env->watchpoint_hit) {
1431                 env->watchpoint_hit = wp;
1432                 tb_check_watchpoint(env);
1433                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1434                     env->exception_index = EXCP_DEBUG;
1435                     cpu_loop_exit(env);
1436                 } else {
1437                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1438                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1439                     cpu_resume_from_signal(env, NULL);
1440                 }
1441             }
1442         } else {
1443             wp->flags &= ~BP_WATCHPOINT_HIT;
1444         }
1445     }
1446 }
1447
1448 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
1449    so these check for a hit then pass through to the normal out-of-line
1450    phys routines.  */
1451 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1452                                unsigned size)
1453 {
1454     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1455     switch (size) {
1456     case 1: return ldub_phys(addr);
1457     case 2: return lduw_phys(addr);
1458     case 4: return ldl_phys(addr);
1459     default: abort();
1460     }
1461 }
1462
1463 static void watch_mem_write(void *opaque, hwaddr addr,
1464                             uint64_t val, unsigned size)
1465 {
1466     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1467     switch (size) {
1468     case 1:
1469         stb_phys(addr, val);
1470         break;
1471     case 2:
1472         stw_phys(addr, val);
1473         break;
1474     case 4:
1475         stl_phys(addr, val);
1476         break;
1477     default: abort();
1478     }
1479 }
1480
1481 static const MemoryRegionOps watch_mem_ops = {
1482     .read = watch_mem_read,
1483     .write = watch_mem_write,
1484     .endianness = DEVICE_NATIVE_ENDIAN,
1485 };
1486
1487 static uint64_t subpage_read(void *opaque, hwaddr addr,
1488                              unsigned len)
1489 {
1490     subpage_t *mmio = opaque;
1491     unsigned int idx = SUBPAGE_IDX(addr);
1492     MemoryRegionSection *section;
1493 #if defined(DEBUG_SUBPAGE)
1494     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
1495            mmio, len, addr, idx);
1496 #endif
1497
1498     section = &phys_sections[mmio->sub_section[idx]];
1499     addr += mmio->base;
1500     addr -= section->offset_within_address_space;
1501     addr += section->offset_within_region;
1502     return io_mem_read(section->mr, addr, len);
1503 }
1504
1505 static void subpage_write(void *opaque, hwaddr addr,
1506                           uint64_t value, unsigned len)
1507 {
1508     subpage_t *mmio = opaque;
1509     unsigned int idx = SUBPAGE_IDX(addr);
1510     MemoryRegionSection *section;
1511 #if defined(DEBUG_SUBPAGE)
1512     printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1513            " idx %d value %"PRIx64"\n",
1514            __func__, mmio, len, addr, idx, value);
1515 #endif
1516
1517     section = &phys_sections[mmio->sub_section[idx]];
1518     addr += mmio->base;
1519     addr -= section->offset_within_address_space;
1520     addr += section->offset_within_region;
1521     io_mem_write(section->mr, addr, value, len);
1522 }
1523
1524 static const MemoryRegionOps subpage_ops = {
1525     .read = subpage_read,
1526     .write = subpage_write,
1527     .endianness = DEVICE_NATIVE_ENDIAN,
1528 };
1529
1530 static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
1531                                  unsigned size)
1532 {
1533     ram_addr_t raddr = addr;
1534     void *ptr = qemu_get_ram_ptr(raddr);
1535     switch (size) {
1536     case 1: return ldub_p(ptr);
1537     case 2: return lduw_p(ptr);
1538     case 4: return ldl_p(ptr);
1539     default: abort();
1540     }
1541 }
1542
1543 static void subpage_ram_write(void *opaque, hwaddr addr,
1544                               uint64_t value, unsigned size)
1545 {
1546     ram_addr_t raddr = addr;
1547     void *ptr = qemu_get_ram_ptr(raddr);
1548     switch (size) {
1549     case 1: return stb_p(ptr, value);
1550     case 2: return stw_p(ptr, value);
1551     case 4: return stl_p(ptr, value);
1552     default: abort();
1553     }
1554 }
1555
1556 static const MemoryRegionOps subpage_ram_ops = {
1557     .read = subpage_ram_read,
1558     .write = subpage_ram_write,
1559     .endianness = DEVICE_NATIVE_ENDIAN,
1560 };
1561
1562 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1563                              uint16_t section)
1564 {
1565     int idx, eidx;
1566
1567     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1568         return -1;
1569     idx = SUBPAGE_IDX(start);
1570     eidx = SUBPAGE_IDX(end);
1571 #if defined(DEBUG_SUBPAGE)
1572     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1573            mmio, start, end, idx, eidx, memory);
1574 #endif
1575     if (memory_region_is_ram(phys_sections[section].mr)) {
1576         MemoryRegionSection new_section = phys_sections[section];
1577         new_section.mr = &io_mem_subpage_ram;
1578         section = phys_section_add(&new_section);
1579     }
1580     for (; idx <= eidx; idx++) {
1581         mmio->sub_section[idx] = section;
1582     }
1583
1584     return 0;
1585 }
1586
1587 static subpage_t *subpage_init(hwaddr base)
1588 {
1589     subpage_t *mmio;
1590
1591     mmio = g_malloc0(sizeof(subpage_t));
1592
1593     mmio->base = base;
1594     memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1595                           "subpage", TARGET_PAGE_SIZE);
1596     mmio->iomem.subpage = true;
1597 #if defined(DEBUG_SUBPAGE)
1598     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1599            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1600 #endif
1601     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1602
1603     return mmio;
1604 }
1605
1606 static uint16_t dummy_section(MemoryRegion *mr)
1607 {
1608     MemoryRegionSection section = {
1609         .mr = mr,
1610         .offset_within_address_space = 0,
1611         .offset_within_region = 0,
1612         .size = UINT64_MAX,
1613     };
1614
1615     return phys_section_add(&section);
1616 }
1617
1618 MemoryRegion *iotlb_to_region(hwaddr index)
1619 {
1620     return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1621 }
1622
1623 static void io_mem_init(void)
1624 {
1625     memory_region_init_io(&io_mem_ram, &error_mem_ops, NULL, "ram", UINT64_MAX);
1626     memory_region_init_io(&io_mem_rom, &rom_mem_ops, NULL, "rom", UINT64_MAX);
1627     memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1628                           "unassigned", UINT64_MAX);
1629     memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1630                           "notdirty", UINT64_MAX);
1631     memory_region_init_io(&io_mem_subpage_ram, &subpage_ram_ops, NULL,
1632                           "subpage-ram", UINT64_MAX);
1633     memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1634                           "watch", UINT64_MAX);
1635 }
1636
1637 static void mem_begin(MemoryListener *listener)
1638 {
1639     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1640
1641     destroy_all_mappings(d);
1642     d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1643 }
1644
1645 static void core_begin(MemoryListener *listener)
1646 {
1647     phys_sections_clear();
1648     phys_section_unassigned = dummy_section(&io_mem_unassigned);
1649     phys_section_notdirty = dummy_section(&io_mem_notdirty);
1650     phys_section_rom = dummy_section(&io_mem_rom);
1651     phys_section_watch = dummy_section(&io_mem_watch);
1652 }
1653
1654 static void tcg_commit(MemoryListener *listener)
1655 {
1656     CPUArchState *env;
1657
1658     /* since each CPU stores ram addresses in its TLB cache, we must
1659        reset the modified entries */
1660     /* XXX: slow ! */
1661     for(env = first_cpu; env != NULL; env = env->next_cpu) {
1662         tlb_flush(env, 1);
1663     }
1664 }
1665
1666 static void core_log_global_start(MemoryListener *listener)
1667 {
1668     cpu_physical_memory_set_dirty_tracking(1);
1669 }
1670
1671 static void core_log_global_stop(MemoryListener *listener)
1672 {
1673     cpu_physical_memory_set_dirty_tracking(0);
1674 }
1675
1676 static void io_region_add(MemoryListener *listener,
1677                           MemoryRegionSection *section)
1678 {
1679     MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1680
1681     mrio->mr = section->mr;
1682     mrio->offset = section->offset_within_region;
1683     iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1684                  section->offset_within_address_space, section->size);
1685     ioport_register(&mrio->iorange);
1686 }
1687
1688 static void io_region_del(MemoryListener *listener,
1689                           MemoryRegionSection *section)
1690 {
1691     isa_unassign_ioport(section->offset_within_address_space, section->size);
1692 }
1693
1694 static MemoryListener core_memory_listener = {
1695     .begin = core_begin,
1696     .log_global_start = core_log_global_start,
1697     .log_global_stop = core_log_global_stop,
1698     .priority = 1,
1699 };
1700
1701 static MemoryListener io_memory_listener = {
1702     .region_add = io_region_add,
1703     .region_del = io_region_del,
1704     .priority = 0,
1705 };
1706
1707 static MemoryListener tcg_memory_listener = {
1708     .commit = tcg_commit,
1709 };
1710
1711 void address_space_init_dispatch(AddressSpace *as)
1712 {
1713     AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1714
1715     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1716     d->listener = (MemoryListener) {
1717         .begin = mem_begin,
1718         .region_add = mem_add,
1719         .region_nop = mem_add,
1720         .priority = 0,
1721     };
1722     as->dispatch = d;
1723     memory_listener_register(&d->listener, as);
1724 }
1725
1726 void address_space_destroy_dispatch(AddressSpace *as)
1727 {
1728     AddressSpaceDispatch *d = as->dispatch;
1729
1730     memory_listener_unregister(&d->listener);
1731     destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1732     g_free(d);
1733     as->dispatch = NULL;
1734 }
1735
1736 static void memory_map_init(void)
1737 {
1738     system_memory = g_malloc(sizeof(*system_memory));
1739     memory_region_init(system_memory, "system", INT64_MAX);
1740     address_space_init(&address_space_memory, system_memory);
1741     address_space_memory.name = "memory";
1742
1743     system_io = g_malloc(sizeof(*system_io));
1744     memory_region_init(system_io, "io", 65536);
1745     address_space_init(&address_space_io, system_io);
1746     address_space_io.name = "I/O";
1747
1748     memory_listener_register(&core_memory_listener, &address_space_memory);
1749     memory_listener_register(&io_memory_listener, &address_space_io);
1750     memory_listener_register(&tcg_memory_listener, &address_space_memory);
1751
1752     dma_context_init(&dma_context_memory, &address_space_memory,
1753                      NULL, NULL, NULL);
1754 }
1755
1756 MemoryRegion *get_system_memory(void)
1757 {
1758     return system_memory;
1759 }
1760
1761 MemoryRegion *get_system_io(void)
1762 {
1763     return system_io;
1764 }
1765
1766 #endif /* !defined(CONFIG_USER_ONLY) */
1767
1768 /* physical memory access (slow version, mainly for debug) */
1769 #if defined(CONFIG_USER_ONLY)
1770 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1771                         uint8_t *buf, int len, int is_write)
1772 {
1773     int l, flags;
1774     target_ulong page;
1775     void * p;
1776
1777     while (len > 0) {
1778         page = addr & TARGET_PAGE_MASK;
1779         l = (page + TARGET_PAGE_SIZE) - addr;
1780         if (l > len)
1781             l = len;
1782         flags = page_get_flags(page);
1783         if (!(flags & PAGE_VALID))
1784             return -1;
1785         if (is_write) {
1786             if (!(flags & PAGE_WRITE))
1787                 return -1;
1788             /* XXX: this code should not depend on lock_user */
1789             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1790                 return -1;
1791             memcpy(p, buf, l);
1792             unlock_user(p, addr, l);
1793         } else {
1794             if (!(flags & PAGE_READ))
1795                 return -1;
1796             /* XXX: this code should not depend on lock_user */
1797             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1798                 return -1;
1799             memcpy(buf, p, l);
1800             unlock_user(p, addr, 0);
1801         }
1802         len -= l;
1803         buf += l;
1804         addr += l;
1805     }
1806     return 0;
1807 }
1808
1809 #else
1810
1811 static void invalidate_and_set_dirty(hwaddr addr,
1812                                      hwaddr length)
1813 {
1814     if (!cpu_physical_memory_is_dirty(addr)) {
1815         /* invalidate code */
1816         tb_invalidate_phys_page_range(addr, addr + length, 0);
1817         /* set dirty bit */
1818         cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1819     }
1820     xen_modified_memory(addr, length);
1821 }
1822
1823 void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1824                       int len, bool is_write)
1825 {
1826     AddressSpaceDispatch *d = as->dispatch;
1827     int l;
1828     uint8_t *ptr;
1829     uint32_t val;
1830     hwaddr page;
1831     MemoryRegionSection *section;
1832
1833     while (len > 0) {
1834         page = addr & TARGET_PAGE_MASK;
1835         l = (page + TARGET_PAGE_SIZE) - addr;
1836         if (l > len)
1837             l = len;
1838         section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1839
1840         if (is_write) {
1841             if (!memory_region_is_ram(section->mr)) {
1842                 hwaddr addr1;
1843                 addr1 = memory_region_section_addr(section, addr);
1844                 /* XXX: could force cpu_single_env to NULL to avoid
1845                    potential bugs */
1846                 if (l >= 4 && ((addr1 & 3) == 0)) {
1847                     /* 32 bit write access */
1848                     val = ldl_p(buf);
1849                     io_mem_write(section->mr, addr1, val, 4);
1850                     l = 4;
1851                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1852                     /* 16 bit write access */
1853                     val = lduw_p(buf);
1854                     io_mem_write(section->mr, addr1, val, 2);
1855                     l = 2;
1856                 } else {
1857                     /* 8 bit write access */
1858                     val = ldub_p(buf);
1859                     io_mem_write(section->mr, addr1, val, 1);
1860                     l = 1;
1861                 }
1862             } else if (!section->readonly) {
1863                 ram_addr_t addr1;
1864                 addr1 = memory_region_get_ram_addr(section->mr)
1865                     + memory_region_section_addr(section, addr);
1866                 /* RAM case */
1867                 ptr = qemu_get_ram_ptr(addr1);
1868                 memcpy(ptr, buf, l);
1869                 invalidate_and_set_dirty(addr1, l);
1870                 qemu_put_ram_ptr(ptr);
1871             }
1872         } else {
1873             if (!(memory_region_is_ram(section->mr) ||
1874                   memory_region_is_romd(section->mr))) {
1875                 hwaddr addr1;
1876                 /* I/O case */
1877                 addr1 = memory_region_section_addr(section, addr);
1878                 if (l >= 4 && ((addr1 & 3) == 0)) {
1879                     /* 32 bit read access */
1880                     val = io_mem_read(section->mr, addr1, 4);
1881                     stl_p(buf, val);
1882                     l = 4;
1883                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1884                     /* 16 bit read access */
1885                     val = io_mem_read(section->mr, addr1, 2);
1886                     stw_p(buf, val);
1887                     l = 2;
1888                 } else {
1889                     /* 8 bit read access */
1890                     val = io_mem_read(section->mr, addr1, 1);
1891                     stb_p(buf, val);
1892                     l = 1;
1893                 }
1894             } else {
1895                 /* RAM case */
1896                 ptr = qemu_get_ram_ptr(section->mr->ram_addr
1897                                        + memory_region_section_addr(section,
1898                                                                     addr));
1899                 memcpy(buf, ptr, l);
1900                 qemu_put_ram_ptr(ptr);
1901             }
1902         }
1903         len -= l;
1904         buf += l;
1905         addr += l;
1906     }
1907 }
1908
1909 void address_space_write(AddressSpace *as, hwaddr addr,
1910                          const uint8_t *buf, int len)
1911 {
1912     address_space_rw(as, addr, (uint8_t *)buf, len, true);
1913 }
1914
1915 /**
1916  * address_space_read: read from an address space.
1917  *
1918  * @as: #AddressSpace to be accessed
1919  * @addr: address within that address space
1920  * @buf: buffer with the data transferred
1921  */
1922 void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1923 {
1924     address_space_rw(as, addr, buf, len, false);
1925 }
1926
1927
1928 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1929                             int len, int is_write)
1930 {
1931     return address_space_rw(&address_space_memory, addr, buf, len, is_write);
1932 }
1933
1934 /* used for ROM loading : can write in RAM and ROM */
1935 void cpu_physical_memory_write_rom(hwaddr addr,
1936                                    const uint8_t *buf, int len)
1937 {
1938     AddressSpaceDispatch *d = address_space_memory.dispatch;
1939     int l;
1940     uint8_t *ptr;
1941     hwaddr page;
1942     MemoryRegionSection *section;
1943
1944     while (len > 0) {
1945         page = addr & TARGET_PAGE_MASK;
1946         l = (page + TARGET_PAGE_SIZE) - addr;
1947         if (l > len)
1948             l = len;
1949         section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1950
1951         if (!(memory_region_is_ram(section->mr) ||
1952               memory_region_is_romd(section->mr))) {
1953             /* do nothing */
1954         } else {
1955             unsigned long addr1;
1956             addr1 = memory_region_get_ram_addr(section->mr)
1957                 + memory_region_section_addr(section, addr);
1958             /* ROM/RAM case */
1959             ptr = qemu_get_ram_ptr(addr1);
1960             memcpy(ptr, buf, l);
1961             invalidate_and_set_dirty(addr1, l);
1962             qemu_put_ram_ptr(ptr);
1963         }
1964         len -= l;
1965         buf += l;
1966         addr += l;
1967     }
1968 }
1969
1970 typedef struct {
1971     void *buffer;
1972     hwaddr addr;
1973     hwaddr len;
1974 } BounceBuffer;
1975
1976 static BounceBuffer bounce;
1977
1978 typedef struct MapClient {
1979     void *opaque;
1980     void (*callback)(void *opaque);
1981     QLIST_ENTRY(MapClient) link;
1982 } MapClient;
1983
1984 static QLIST_HEAD(map_client_list, MapClient) map_client_list
1985     = QLIST_HEAD_INITIALIZER(map_client_list);
1986
1987 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
1988 {
1989     MapClient *client = g_malloc(sizeof(*client));
1990
1991     client->opaque = opaque;
1992     client->callback = callback;
1993     QLIST_INSERT_HEAD(&map_client_list, client, link);
1994     return client;
1995 }
1996
1997 static void cpu_unregister_map_client(void *_client)
1998 {
1999     MapClient *client = (MapClient *)_client;
2000
2001     QLIST_REMOVE(client, link);
2002     g_free(client);
2003 }
2004
2005 static void cpu_notify_map_clients(void)
2006 {
2007     MapClient *client;
2008
2009     while (!QLIST_EMPTY(&map_client_list)) {
2010         client = QLIST_FIRST(&map_client_list);
2011         client->callback(client->opaque);
2012         cpu_unregister_map_client(client);
2013     }
2014 }
2015
2016 /* Map a physical memory region into a host virtual address.
2017  * May map a subset of the requested range, given by and returned in *plen.
2018  * May return NULL if resources needed to perform the mapping are exhausted.
2019  * Use only for reads OR writes - not for read-modify-write operations.
2020  * Use cpu_register_map_client() to know when retrying the map operation is
2021  * likely to succeed.
2022  */
2023 void *address_space_map(AddressSpace *as,
2024                         hwaddr addr,
2025                         hwaddr *plen,
2026                         bool is_write)
2027 {
2028     AddressSpaceDispatch *d = as->dispatch;
2029     hwaddr len = *plen;
2030     hwaddr todo = 0;
2031     int l;
2032     hwaddr page;
2033     MemoryRegionSection *section;
2034     ram_addr_t raddr = RAM_ADDR_MAX;
2035     ram_addr_t rlen;
2036     void *ret;
2037
2038     while (len > 0) {
2039         page = addr & TARGET_PAGE_MASK;
2040         l = (page + TARGET_PAGE_SIZE) - addr;
2041         if (l > len)
2042             l = len;
2043         section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2044
2045         if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
2046             if (todo || bounce.buffer) {
2047                 break;
2048             }
2049             bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2050             bounce.addr = addr;
2051             bounce.len = l;
2052             if (!is_write) {
2053                 address_space_read(as, addr, bounce.buffer, l);
2054             }
2055
2056             *plen = l;
2057             return bounce.buffer;
2058         }
2059         if (!todo) {
2060             raddr = memory_region_get_ram_addr(section->mr)
2061                 + memory_region_section_addr(section, addr);
2062         }
2063
2064         len -= l;
2065         addr += l;
2066         todo += l;
2067     }
2068     rlen = todo;
2069     ret = qemu_ram_ptr_length(raddr, &rlen);
2070     *plen = rlen;
2071     return ret;
2072 }
2073
2074 /* Unmaps a memory region previously mapped by address_space_map().
2075  * Will also mark the memory as dirty if is_write == 1.  access_len gives
2076  * the amount of memory that was actually read or written by the caller.
2077  */
2078 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2079                          int is_write, hwaddr access_len)
2080 {
2081     if (buffer != bounce.buffer) {
2082         if (is_write) {
2083             ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2084             while (access_len) {
2085                 unsigned l;
2086                 l = TARGET_PAGE_SIZE;
2087                 if (l > access_len)
2088                     l = access_len;
2089                 invalidate_and_set_dirty(addr1, l);
2090                 addr1 += l;
2091                 access_len -= l;
2092             }
2093         }
2094         if (xen_enabled()) {
2095             xen_invalidate_map_cache_entry(buffer);
2096         }
2097         return;
2098     }
2099     if (is_write) {
2100         address_space_write(as, bounce.addr, bounce.buffer, access_len);
2101     }
2102     qemu_vfree(bounce.buffer);
2103     bounce.buffer = NULL;
2104     cpu_notify_map_clients();
2105 }
2106
2107 void *cpu_physical_memory_map(hwaddr addr,
2108                               hwaddr *plen,
2109                               int is_write)
2110 {
2111     return address_space_map(&address_space_memory, addr, plen, is_write);
2112 }
2113
2114 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2115                                int is_write, hwaddr access_len)
2116 {
2117     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2118 }
2119
2120 /* warning: addr must be aligned */
2121 static inline uint32_t ldl_phys_internal(hwaddr addr,
2122                                          enum device_endian endian)
2123 {
2124     uint8_t *ptr;
2125     uint32_t val;
2126     MemoryRegionSection *section;
2127
2128     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2129
2130     if (!(memory_region_is_ram(section->mr) ||
2131           memory_region_is_romd(section->mr))) {
2132         /* I/O case */
2133         addr = memory_region_section_addr(section, addr);
2134         val = io_mem_read(section->mr, addr, 4);
2135 #if defined(TARGET_WORDS_BIGENDIAN)
2136         if (endian == DEVICE_LITTLE_ENDIAN) {
2137             val = bswap32(val);
2138         }
2139 #else
2140         if (endian == DEVICE_BIG_ENDIAN) {
2141             val = bswap32(val);
2142         }
2143 #endif
2144     } else {
2145         /* RAM case */
2146         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2147                                 & TARGET_PAGE_MASK)
2148                                + memory_region_section_addr(section, addr));
2149         switch (endian) {
2150         case DEVICE_LITTLE_ENDIAN:
2151             val = ldl_le_p(ptr);
2152             break;
2153         case DEVICE_BIG_ENDIAN:
2154             val = ldl_be_p(ptr);
2155             break;
2156         default:
2157             val = ldl_p(ptr);
2158             break;
2159         }
2160     }
2161     return val;
2162 }
2163
2164 uint32_t ldl_phys(hwaddr addr)
2165 {
2166     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2167 }
2168
2169 uint32_t ldl_le_phys(hwaddr addr)
2170 {
2171     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2172 }
2173
2174 uint32_t ldl_be_phys(hwaddr addr)
2175 {
2176     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2177 }
2178
2179 /* warning: addr must be aligned */
2180 static inline uint64_t ldq_phys_internal(hwaddr addr,
2181                                          enum device_endian endian)
2182 {
2183     uint8_t *ptr;
2184     uint64_t val;
2185     MemoryRegionSection *section;
2186
2187     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2188
2189     if (!(memory_region_is_ram(section->mr) ||
2190           memory_region_is_romd(section->mr))) {
2191         /* I/O case */
2192         addr = memory_region_section_addr(section, addr);
2193
2194         /* XXX This is broken when device endian != cpu endian.
2195                Fix and add "endian" variable check */
2196 #ifdef TARGET_WORDS_BIGENDIAN
2197         val = io_mem_read(section->mr, addr, 4) << 32;
2198         val |= io_mem_read(section->mr, addr + 4, 4);
2199 #else
2200         val = io_mem_read(section->mr, addr, 4);
2201         val |= io_mem_read(section->mr, addr + 4, 4) << 32;
2202 #endif
2203     } else {
2204         /* RAM case */
2205         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2206                                 & TARGET_PAGE_MASK)
2207                                + memory_region_section_addr(section, addr));
2208         switch (endian) {
2209         case DEVICE_LITTLE_ENDIAN:
2210             val = ldq_le_p(ptr);
2211             break;
2212         case DEVICE_BIG_ENDIAN:
2213             val = ldq_be_p(ptr);
2214             break;
2215         default:
2216             val = ldq_p(ptr);
2217             break;
2218         }
2219     }
2220     return val;
2221 }
2222
2223 uint64_t ldq_phys(hwaddr addr)
2224 {
2225     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2226 }
2227
2228 uint64_t ldq_le_phys(hwaddr addr)
2229 {
2230     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2231 }
2232
2233 uint64_t ldq_be_phys(hwaddr addr)
2234 {
2235     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2236 }
2237
2238 /* XXX: optimize */
2239 uint32_t ldub_phys(hwaddr addr)
2240 {
2241     uint8_t val;
2242     cpu_physical_memory_read(addr, &val, 1);
2243     return val;
2244 }
2245
2246 /* warning: addr must be aligned */
2247 static inline uint32_t lduw_phys_internal(hwaddr addr,
2248                                           enum device_endian endian)
2249 {
2250     uint8_t *ptr;
2251     uint64_t val;
2252     MemoryRegionSection *section;
2253
2254     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2255
2256     if (!(memory_region_is_ram(section->mr) ||
2257           memory_region_is_romd(section->mr))) {
2258         /* I/O case */
2259         addr = memory_region_section_addr(section, addr);
2260         val = io_mem_read(section->mr, addr, 2);
2261 #if defined(TARGET_WORDS_BIGENDIAN)
2262         if (endian == DEVICE_LITTLE_ENDIAN) {
2263             val = bswap16(val);
2264         }
2265 #else
2266         if (endian == DEVICE_BIG_ENDIAN) {
2267             val = bswap16(val);
2268         }
2269 #endif
2270     } else {
2271         /* RAM case */
2272         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2273                                 & TARGET_PAGE_MASK)
2274                                + memory_region_section_addr(section, addr));
2275         switch (endian) {
2276         case DEVICE_LITTLE_ENDIAN:
2277             val = lduw_le_p(ptr);
2278             break;
2279         case DEVICE_BIG_ENDIAN:
2280             val = lduw_be_p(ptr);
2281             break;
2282         default:
2283             val = lduw_p(ptr);
2284             break;
2285         }
2286     }
2287     return val;
2288 }
2289
2290 uint32_t lduw_phys(hwaddr addr)
2291 {
2292     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2293 }
2294
2295 uint32_t lduw_le_phys(hwaddr addr)
2296 {
2297     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2298 }
2299
2300 uint32_t lduw_be_phys(hwaddr addr)
2301 {
2302     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2303 }
2304
2305 /* warning: addr must be aligned. The ram page is not masked as dirty
2306    and the code inside is not invalidated. It is useful if the dirty
2307    bits are used to track modified PTEs */
2308 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2309 {
2310     uint8_t *ptr;
2311     MemoryRegionSection *section;
2312
2313     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2314
2315     if (!memory_region_is_ram(section->mr) || section->readonly) {
2316         addr = memory_region_section_addr(section, addr);
2317         if (memory_region_is_ram(section->mr)) {
2318             section = &phys_sections[phys_section_rom];
2319         }
2320         io_mem_write(section->mr, addr, val, 4);
2321     } else {
2322         unsigned long addr1 = (memory_region_get_ram_addr(section->mr)
2323                                & TARGET_PAGE_MASK)
2324             + memory_region_section_addr(section, addr);
2325         ptr = qemu_get_ram_ptr(addr1);
2326         stl_p(ptr, val);
2327
2328         if (unlikely(in_migration)) {
2329             if (!cpu_physical_memory_is_dirty(addr1)) {
2330                 /* invalidate code */
2331                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2332                 /* set dirty bit */
2333                 cpu_physical_memory_set_dirty_flags(
2334                     addr1, (0xff & ~CODE_DIRTY_FLAG));
2335             }
2336         }
2337     }
2338 }
2339
2340 void stq_phys_notdirty(hwaddr addr, uint64_t val)
2341 {
2342     uint8_t *ptr;
2343     MemoryRegionSection *section;
2344
2345     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2346
2347     if (!memory_region_is_ram(section->mr) || section->readonly) {
2348         addr = memory_region_section_addr(section, addr);
2349         if (memory_region_is_ram(section->mr)) {
2350             section = &phys_sections[phys_section_rom];
2351         }
2352 #ifdef TARGET_WORDS_BIGENDIAN
2353         io_mem_write(section->mr, addr, val >> 32, 4);
2354         io_mem_write(section->mr, addr + 4, (uint32_t)val, 4);
2355 #else
2356         io_mem_write(section->mr, addr, (uint32_t)val, 4);
2357         io_mem_write(section->mr, addr + 4, val >> 32, 4);
2358 #endif
2359     } else {
2360         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2361                                 & TARGET_PAGE_MASK)
2362                                + memory_region_section_addr(section, addr));
2363         stq_p(ptr, val);
2364     }
2365 }
2366
2367 /* warning: addr must be aligned */
2368 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2369                                      enum device_endian endian)
2370 {
2371     uint8_t *ptr;
2372     MemoryRegionSection *section;
2373
2374     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2375
2376     if (!memory_region_is_ram(section->mr) || section->readonly) {
2377         addr = memory_region_section_addr(section, addr);
2378         if (memory_region_is_ram(section->mr)) {
2379             section = &phys_sections[phys_section_rom];
2380         }
2381 #if defined(TARGET_WORDS_BIGENDIAN)
2382         if (endian == DEVICE_LITTLE_ENDIAN) {
2383             val = bswap32(val);
2384         }
2385 #else
2386         if (endian == DEVICE_BIG_ENDIAN) {
2387             val = bswap32(val);
2388         }
2389 #endif
2390         io_mem_write(section->mr, addr, val, 4);
2391     } else {
2392         unsigned long addr1;
2393         addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2394             + memory_region_section_addr(section, addr);
2395         /* RAM case */
2396         ptr = qemu_get_ram_ptr(addr1);
2397         switch (endian) {
2398         case DEVICE_LITTLE_ENDIAN:
2399             stl_le_p(ptr, val);
2400             break;
2401         case DEVICE_BIG_ENDIAN:
2402             stl_be_p(ptr, val);
2403             break;
2404         default:
2405             stl_p(ptr, val);
2406             break;
2407         }
2408         invalidate_and_set_dirty(addr1, 4);
2409     }
2410 }
2411
2412 void stl_phys(hwaddr addr, uint32_t val)
2413 {
2414     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2415 }
2416
2417 void stl_le_phys(hwaddr addr, uint32_t val)
2418 {
2419     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2420 }
2421
2422 void stl_be_phys(hwaddr addr, uint32_t val)
2423 {
2424     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2425 }
2426
2427 /* XXX: optimize */
2428 void stb_phys(hwaddr addr, uint32_t val)
2429 {
2430     uint8_t v = val;
2431     cpu_physical_memory_write(addr, &v, 1);
2432 }
2433
2434 /* warning: addr must be aligned */
2435 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2436                                      enum device_endian endian)
2437 {
2438     uint8_t *ptr;
2439     MemoryRegionSection *section;
2440
2441     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2442
2443     if (!memory_region_is_ram(section->mr) || section->readonly) {
2444         addr = memory_region_section_addr(section, addr);
2445         if (memory_region_is_ram(section->mr)) {
2446             section = &phys_sections[phys_section_rom];
2447         }
2448 #if defined(TARGET_WORDS_BIGENDIAN)
2449         if (endian == DEVICE_LITTLE_ENDIAN) {
2450             val = bswap16(val);
2451         }
2452 #else
2453         if (endian == DEVICE_BIG_ENDIAN) {
2454             val = bswap16(val);
2455         }
2456 #endif
2457         io_mem_write(section->mr, addr, val, 2);
2458     } else {
2459         unsigned long addr1;
2460         addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2461             + memory_region_section_addr(section, addr);
2462         /* RAM case */
2463         ptr = qemu_get_ram_ptr(addr1);
2464         switch (endian) {
2465         case DEVICE_LITTLE_ENDIAN:
2466             stw_le_p(ptr, val);
2467             break;
2468         case DEVICE_BIG_ENDIAN:
2469             stw_be_p(ptr, val);
2470             break;
2471         default:
2472             stw_p(ptr, val);
2473             break;
2474         }
2475         invalidate_and_set_dirty(addr1, 2);
2476     }
2477 }
2478
2479 void stw_phys(hwaddr addr, uint32_t val)
2480 {
2481     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2482 }
2483
2484 void stw_le_phys(hwaddr addr, uint32_t val)
2485 {
2486     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2487 }
2488
2489 void stw_be_phys(hwaddr addr, uint32_t val)
2490 {
2491     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2492 }
2493
2494 /* XXX: optimize */
2495 void stq_phys(hwaddr addr, uint64_t val)
2496 {
2497     val = tswap64(val);
2498     cpu_physical_memory_write(addr, &val, 8);
2499 }
2500
2501 void stq_le_phys(hwaddr addr, uint64_t val)
2502 {
2503     val = cpu_to_le64(val);
2504     cpu_physical_memory_write(addr, &val, 8);
2505 }
2506
2507 void stq_be_phys(hwaddr addr, uint64_t val)
2508 {
2509     val = cpu_to_be64(val);
2510     cpu_physical_memory_write(addr, &val, 8);
2511 }
2512
2513 /* virtual memory access for debug (includes writing to ROM) */
2514 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2515                         uint8_t *buf, int len, int is_write)
2516 {
2517     int l;
2518     hwaddr phys_addr;
2519     target_ulong page;
2520
2521     while (len > 0) {
2522         page = addr & TARGET_PAGE_MASK;
2523         phys_addr = cpu_get_phys_page_debug(env, page);
2524         /* if no physical page mapped, return an error */
2525         if (phys_addr == -1)
2526             return -1;
2527         l = (page + TARGET_PAGE_SIZE) - addr;
2528         if (l > len)
2529             l = len;
2530         phys_addr += (addr & ~TARGET_PAGE_MASK);
2531         if (is_write)
2532             cpu_physical_memory_write_rom(phys_addr, buf, l);
2533         else
2534             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2535         len -= l;
2536         buf += l;
2537         addr += l;
2538     }
2539     return 0;
2540 }
2541 #endif
2542
2543 #if !defined(CONFIG_USER_ONLY)
2544
2545 /*
2546  * A helper function for the _utterly broken_ virtio device model to find out if
2547  * it's running on a big endian machine. Don't do this at home kids!
2548  */
2549 bool virtio_is_big_endian(void);
2550 bool virtio_is_big_endian(void)
2551 {
2552 #if defined(TARGET_WORDS_BIGENDIAN)
2553     return true;
2554 #else
2555     return false;
2556 #endif
2557 }
2558
2559 #endif
2560
2561 #ifndef CONFIG_USER_ONLY
2562 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2563 {
2564     MemoryRegionSection *section;
2565
2566     section = phys_page_find(address_space_memory.dispatch,
2567                              phys_addr >> TARGET_PAGE_BITS);
2568
2569     return !(memory_region_is_ram(section->mr) ||
2570              memory_region_is_romd(section->mr));
2571 }
2572 #endif