exec.c

   1 /*
   2  *  Virtual page mapping
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "config.h"
  20 #ifdef _WIN32
  21 #include <windows.h>
  22 #else
  23 #include <sys/types.h>
  24 #include <sys/mman.h>
  25 #endif
  26
  27 #include "qemu-common.h"
  28 #include "cpu.h"
  29 #include "tcg.h"
  30 #include "hw/hw.h"
  31 #include "hw/qdev.h"
  32 #include "qemu/osdep.h"
  33 #include "sysemu/kvm.h"
  34 #include "hw/xen.h"
  35 #include "qemu/timer.h"
  36 #include "qemu/config-file.h"
  37 #include "exec/memory.h"
  38 #include "sysemu/dma.h"
  39 #include "exec/address-spaces.h"
  40 #if defined(CONFIG_USER_ONLY)
  41 #include <qemu.h>
  42 #else /* !CONFIG_USER_ONLY */
  43 #include "sysemu/xen-mapcache.h"
  44 #include "trace.h"
  45 #endif
  46 #include "exec/cpu-all.h"
  47
  48 #include "exec/cputlb.h"
  49 #include "translate-all.h"
  50
  51 #include "exec/memory-internal.h"
  52
  53 //#define DEBUG_UNASSIGNED
  54 //#define DEBUG_SUBPAGE
  55
  56 #if !defined(CONFIG_USER_ONLY)
  57 int phys_ram_fd;
  58 static int in_migration;
  59
  60 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
  61
  62 static MemoryRegion *system_memory;
  63 static MemoryRegion *system_io;
  64
  65 AddressSpace address_space_io;
  66 AddressSpace address_space_memory;
  67 DMAContext dma_context_memory;
  68
  69 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
  70 static MemoryRegion io_mem_subpage_ram;
  71
  72 #endif
  73
  74 CPUArchState *first_cpu;
  75 /* current CPU in the current thread. It is only valid inside
  76    cpu_exec() */
  77 DEFINE_TLS(CPUArchState *,cpu_single_env);
  78 /* 0 = Do not count executed instructions.
  79    1 = Precise instruction counting.
  80    2 = Adaptive rate instruction counting.  */
  81 int use_icount = 0;
  82
  83 #if !defined(CONFIG_USER_ONLY)
  84
  85 static MemoryRegionSection *phys_sections;
  86 static unsigned phys_sections_nb, phys_sections_nb_alloc;
  87 static uint16_t phys_section_unassigned;
  88 static uint16_t phys_section_notdirty;
  89 static uint16_t phys_section_rom;
  90 static uint16_t phys_section_watch;
  91
  92 /* Simple allocator for PhysPageEntry nodes */
  93 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
  94 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
  95
  96 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
  97
  98 static void io_mem_init(void);
  99 static void memory_map_init(void);
 100 static void *qemu_safe_ram_ptr(ram_addr_t addr);
 101
 102 static MemoryRegion io_mem_watch;
 103 #endif
 104
 105 #if !defined(CONFIG_USER_ONLY)
 106
 107 static void phys_map_node_reserve(unsigned nodes)
 108 {
 109     if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
 110         typedef PhysPageEntry Node[L2_SIZE];
 111         phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
 112         phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
 113                                       phys_map_nodes_nb + nodes);
 114         phys_map_nodes = g_renew(Node, phys_map_nodes,
 115                                  phys_map_nodes_nb_alloc);
 116     }
 117 }
 118
 119 static uint16_t phys_map_node_alloc(void)
 120 {
 121     unsigned i;
 122     uint16_t ret;
 123
 124     ret = phys_map_nodes_nb++;
 125     assert(ret != PHYS_MAP_NODE_NIL);
 126     assert(ret != phys_map_nodes_nb_alloc);
 127     for (i = 0; i < L2_SIZE; ++i) {
 128         phys_map_nodes[ret][i].is_leaf = 0;
 129         phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
 130     }
 131     return ret;
 132 }
 133
 134 static void phys_map_nodes_reset(void)
 135 {
 136     phys_map_nodes_nb = 0;
 137 }
 138
 139
 140 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
 141                                 hwaddr *nb, uint16_t leaf,
 142                                 int level)
 143 {
 144     PhysPageEntry *p;
 145     int i;
 146     hwaddr step = (hwaddr)1 << (level * L2_BITS);
 147
 148     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
 149         lp->ptr = phys_map_node_alloc();
 150         p = phys_map_nodes[lp->ptr];
 151         if (level == 0) {
 152             for (i = 0; i < L2_SIZE; i++) {
 153                 p[i].is_leaf = 1;
 154                 p[i].ptr = phys_section_unassigned;
 155             }
 156         }
 157     } else {
 158         p = phys_map_nodes[lp->ptr];
 159     }
 160     lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
 161
 162     while (*nb && lp < &p[L2_SIZE]) {
 163         if ((*index & (step - 1)) == 0 && *nb >= step) {
 164             lp->is_leaf = true;
 165             lp->ptr = leaf;
 166             *index += step;
 167             *nb -= step;
 168         } else {
 169             phys_page_set_level(lp, index, nb, leaf, level - 1);
 170         }
 171         ++lp;
 172     }
 173 }
 174
 175 static void phys_page_set(AddressSpaceDispatch *d,
 176                           hwaddr index, hwaddr nb,
 177                           uint16_t leaf)
 178 {
 179     /* Wildly overreserve - it doesn't matter much. */
 180     phys_map_node_reserve(3 * P_L2_LEVELS);
 181
 182     phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 183 }
 184
 185 MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
 186 {
 187     PhysPageEntry lp = d->phys_map;
 188     PhysPageEntry *p;
 189     int i;
 190     uint16_t s_index = phys_section_unassigned;
 191
 192     for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
 193         if (lp.ptr == PHYS_MAP_NODE_NIL) {
 194             goto not_found;
 195         }
 196         p = phys_map_nodes[lp.ptr];
 197         lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
 198     }
 199
 200     s_index = lp.ptr;
 201 not_found:
 202     return &phys_sections[s_index];
 203 }
 204
 205 bool memory_region_is_unassigned(MemoryRegion *mr)
 206 {
 207     return mr != &io_mem_ram && mr != &io_mem_rom
 208         && mr != &io_mem_notdirty && !mr->rom_device
 209         && mr != &io_mem_watch;
 210 }
 211 #endif
 212
 213 void cpu_exec_init_all(void)
 214 {
 215 #if !defined(CONFIG_USER_ONLY)
 216     memory_map_init();
 217     io_mem_init();
 218 #endif
 219 }
 220
 221 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 222
 223 static int cpu_common_post_load(void *opaque, int version_id)
 224 {
 225     CPUArchState *env = opaque;
 226
 227     /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
 228        version_id is increased. */
 229     env->interrupt_request &= ~0x01;
 230     tlb_flush(env, 1);
 231
 232     return 0;
 233 }
 234
 235 static const VMStateDescription vmstate_cpu_common = {
 236     .name = "cpu_common",
 237     .version_id = 1,
 238     .minimum_version_id = 1,
 239     .minimum_version_id_old = 1,
 240     .post_load = cpu_common_post_load,
 241     .fields      = (VMStateField []) {
 242         VMSTATE_UINT32(halted, CPUArchState),
 243         VMSTATE_UINT32(interrupt_request, CPUArchState),
 244         VMSTATE_END_OF_LIST()
 245     }
 246 };
 247 #endif
 248
 249 CPUArchState *qemu_get_cpu(int cpu)
 250 {
 251     CPUArchState *env = first_cpu;
 252
 253     while (env) {
 254         if (env->cpu_index == cpu)
 255             break;
 256         env = env->next_cpu;
 257     }
 258
 259     return env;
 260 }
 261
 262 void cpu_exec_init(CPUArchState *env)
 263 {
 264 #ifndef CONFIG_USER_ONLY
 265     CPUState *cpu = ENV_GET_CPU(env);
 266 #endif
 267     CPUArchState **penv;
 268     int cpu_index;
 269
 270 #if defined(CONFIG_USER_ONLY)
 271     cpu_list_lock();
 272 #endif
 273     env->next_cpu = NULL;
 274     penv = &first_cpu;
 275     cpu_index = 0;
 276     while (*penv != NULL) {
 277         penv = &(*penv)->next_cpu;
 278         cpu_index++;
 279     }
 280     env->cpu_index = cpu_index;
 281     env->numa_node = 0;
 282     QTAILQ_INIT(&env->breakpoints);
 283     QTAILQ_INIT(&env->watchpoints);
 284 #ifndef CONFIG_USER_ONLY
 285     cpu->thread_id = qemu_get_thread_id();
 286 #endif
 287     *penv = env;
 288 #if defined(CONFIG_USER_ONLY)
 289     cpu_list_unlock();
 290 #endif
 291 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 292     vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
 293     register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
 294                     cpu_save, cpu_load, env);
 295 #endif
 296 }
 297
 298 #if defined(TARGET_HAS_ICE)
 299 #if defined(CONFIG_USER_ONLY)
 300 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 301 {
 302     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 303 }
 304 #else
 305 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
 306 {
 307     tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
 308             (pc & ~TARGET_PAGE_MASK));
 309 }
 310 #endif
 311 #endif /* TARGET_HAS_ICE */
 312
 313 #if defined(CONFIG_USER_ONLY)
 314 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 315
 316 {
 317 }
 318
 319 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 320                           int flags, CPUWatchpoint **watchpoint)
 321 {
 322     return -ENOSYS;
 323 }
 324 #else
 325 /* Add a watchpoint.  */
 326 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
 327                           int flags, CPUWatchpoint **watchpoint)
 328 {
 329     target_ulong len_mask = ~(len - 1);
 330     CPUWatchpoint *wp;
 331
 332     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
 333     if ((len & (len - 1)) || (addr & ~len_mask) ||
 334             len == 0 || len > TARGET_PAGE_SIZE) {
 335         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
 336                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
 337         return -EINVAL;
 338     }
 339     wp = g_malloc(sizeof(*wp));
 340
 341     wp->vaddr = addr;
 342     wp->len_mask = len_mask;
 343     wp->flags = flags;
 344
 345     /* keep all GDB-injected watchpoints in front */
 346     if (flags & BP_GDB)
 347         QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
 348     else
 349         QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
 350
 351     tlb_flush_page(env, addr);
 352
 353     if (watchpoint)
 354         *watchpoint = wp;
 355     return 0;
 356 }
 357
 358 /* Remove a specific watchpoint.  */
 359 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
 360                           int flags)
 361 {
 362     target_ulong len_mask = ~(len - 1);
 363     CPUWatchpoint *wp;
 364
 365     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 366         if (addr == wp->vaddr && len_mask == wp->len_mask
 367                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
 368             cpu_watchpoint_remove_by_ref(env, wp);
 369             return 0;
 370         }
 371     }
 372     return -ENOENT;
 373 }
 374
 375 /* Remove a specific watchpoint by reference.  */
 376 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
 377 {
 378     QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
 379
 380     tlb_flush_page(env, watchpoint->vaddr);
 381
 382     g_free(watchpoint);
 383 }
 384
 385 /* Remove all matching watchpoints.  */
 386 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
 387 {
 388     CPUWatchpoint *wp, *next;
 389
 390     QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
 391         if (wp->flags & mask)
 392             cpu_watchpoint_remove_by_ref(env, wp);
 393     }
 394 }
 395 #endif
 396
 397 /* Add a breakpoint.  */
 398 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
 399                           CPUBreakpoint **breakpoint)
 400 {
 401 #if defined(TARGET_HAS_ICE)
 402     CPUBreakpoint *bp;
 403
 404     bp = g_malloc(sizeof(*bp));
 405
 406     bp->pc = pc;
 407     bp->flags = flags;
 408
 409     /* keep all GDB-injected breakpoints in front */
 410     if (flags & BP_GDB)
 411         QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
 412     else
 413         QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
 414
 415     breakpoint_invalidate(env, pc);
 416
 417     if (breakpoint)
 418         *breakpoint = bp;
 419     return 0;
 420 #else
 421     return -ENOSYS;
 422 #endif
 423 }
 424
 425 /* Remove a specific breakpoint.  */
 426 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
 427 {
 428 #if defined(TARGET_HAS_ICE)
 429     CPUBreakpoint *bp;
 430
 431     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 432         if (bp->pc == pc && bp->flags == flags) {
 433             cpu_breakpoint_remove_by_ref(env, bp);
 434             return 0;
 435         }
 436     }
 437     return -ENOENT;
 438 #else
 439     return -ENOSYS;
 440 #endif
 441 }
 442
 443 /* Remove a specific breakpoint by reference.  */
 444 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
 445 {
 446 #if defined(TARGET_HAS_ICE)
 447     QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
 448
 449     breakpoint_invalidate(env, breakpoint->pc);
 450
 451     g_free(breakpoint);
 452 #endif
 453 }
 454
 455 /* Remove all matching breakpoints. */
 456 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
 457 {
 458 #if defined(TARGET_HAS_ICE)
 459     CPUBreakpoint *bp, *next;
 460
 461     QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
 462         if (bp->flags & mask)
 463             cpu_breakpoint_remove_by_ref(env, bp);
 464     }
 465 #endif
 466 }
 467
 468 /* enable or disable single step mode. EXCP_DEBUG is returned by the
 469    CPU loop after each instruction */
 470 void cpu_single_step(CPUArchState *env, int enabled)
 471 {
 472 #if defined(TARGET_HAS_ICE)
 473     if (env->singlestep_enabled != enabled) {
 474         env->singlestep_enabled = enabled;
 475         if (kvm_enabled())
 476             kvm_update_guest_debug(env, 0);
 477         else {
 478             /* must flush all the translated code to avoid inconsistencies */
 479             /* XXX: only flush what is necessary */
 480             tb_flush(env);
 481         }
 482     }
 483 #endif
 484 }
 485
 486 void cpu_reset_interrupt(CPUArchState *env, int mask)
 487 {
 488     env->interrupt_request &= ~mask;
 489 }
 490
 491 void cpu_exit(CPUArchState *env)
 492 {
 493     env->exit_request = 1;
 494     cpu_unlink_tb(env);
 495 }
 496
 497 void cpu_abort(CPUArchState *env, const char *fmt, ...)
 498 {
 499     va_list ap;
 500     va_list ap2;
 501
 502     va_start(ap, fmt);
 503     va_copy(ap2, ap);
 504     fprintf(stderr, "qemu: fatal: ");
 505     vfprintf(stderr, fmt, ap);
 506     fprintf(stderr, "\n");
 507     cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 508     if (qemu_log_enabled()) {
 509         qemu_log("qemu: fatal: ");
 510         qemu_log_vprintf(fmt, ap2);
 511         qemu_log("\n");
 512         log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
 513         qemu_log_flush();
 514         qemu_log_close();
 515     }
 516     va_end(ap2);
 517     va_end(ap);
 518 #if defined(CONFIG_USER_ONLY)
 519     {
 520         struct sigaction act;
 521         sigfillset(&act.sa_mask);
 522         act.sa_handler = SIG_DFL;
 523         sigaction(SIGABRT, &act, NULL);
 524     }
 525 #endif
 526     abort();
 527 }
 528
 529 CPUArchState *cpu_copy(CPUArchState *env)
 530 {
 531     CPUArchState *new_env = cpu_init(env->cpu_model_str);
 532     CPUArchState *next_cpu = new_env->next_cpu;
 533     int cpu_index = new_env->cpu_index;
 534 #if defined(TARGET_HAS_ICE)
 535     CPUBreakpoint *bp;
 536     CPUWatchpoint *wp;
 537 #endif
 538
 539     memcpy(new_env, env, sizeof(CPUArchState));
 540
 541     /* Preserve chaining and index. */
 542     new_env->next_cpu = next_cpu;
 543     new_env->cpu_index = cpu_index;
 544
 545     /* Clone all break/watchpoints.
 546        Note: Once we support ptrace with hw-debug register access, make sure
 547        BP_CPU break/watchpoints are handled correctly on clone. */
 548     QTAILQ_INIT(&env->breakpoints);
 549     QTAILQ_INIT(&env->watchpoints);
 550 #if defined(TARGET_HAS_ICE)
 551     QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
 552         cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
 553     }
 554     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 555         cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
 556                               wp->flags, NULL);
 557     }
 558 #endif
 559
 560     return new_env;
 561 }
 562
 563 #if !defined(CONFIG_USER_ONLY)
 564 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
 565                                       uintptr_t length)
 566 {
 567     uintptr_t start1;
 568
 569     /* we modify the TLB cache so that the dirty bit will be set again
 570        when accessing the range */
 571     start1 = (uintptr_t)qemu_safe_ram_ptr(start);
 572     /* Check that we don't span multiple blocks - this breaks the
 573        address comparisons below.  */
 574     if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
 575             != (end - 1) - start) {
 576         abort();
 577     }
 578     cpu_tlb_reset_dirty_all(start1, length);
 579
 580 }
 581
 582 /* Note: start and end must be within the same ram block.  */
 583 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
 584                                      int dirty_flags)
 585 {
 586     uintptr_t length;
 587
 588     start &= TARGET_PAGE_MASK;
 589     end = TARGET_PAGE_ALIGN(end);
 590
 591     length = end - start;
 592     if (length == 0)
 593         return;
 594     cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
 595
 596     if (tcg_enabled()) {
 597         tlb_reset_dirty_range_all(start, end, length);
 598     }
 599 }
 600
 601 static int cpu_physical_memory_set_dirty_tracking(int enable)
 602 {
 603     int ret = 0;
 604     in_migration = enable;
 605     return ret;
 606 }
 607
 608 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
 609                                                    MemoryRegionSection *section,
 610                                                    target_ulong vaddr,
 611                                                    hwaddr paddr,
 612                                                    int prot,
 613                                                    target_ulong *address)
 614 {
 615     hwaddr iotlb;
 616     CPUWatchpoint *wp;
 617
 618     if (memory_region_is_ram(section->mr)) {
 619         /* Normal RAM.  */
 620         iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
 621             + memory_region_section_addr(section, paddr);
 622         if (!section->readonly) {
 623             iotlb |= phys_section_notdirty;
 624         } else {
 625             iotlb |= phys_section_rom;
 626         }
 627     } else {
 628         /* IO handlers are currently passed a physical address.
 629            It would be nice to pass an offset from the base address
 630            of that region.  This would avoid having to special case RAM,
 631            and avoid full address decoding in every device.
 632            We can't use the high bits of pd for this because
 633            IO_MEM_ROMD uses these as a ram address.  */
 634         iotlb = section - phys_sections;
 635         iotlb += memory_region_section_addr(section, paddr);
 636     }
 637
 638     /* Make accesses to pages with watchpoints go via the
 639        watchpoint trap routines.  */
 640     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
 641         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
 642             /* Avoid trapping reads of pages with a write breakpoint. */
 643             if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
 644                 iotlb = phys_section_watch + paddr;
 645                 *address |= TLB_MMIO;
 646                 break;
 647             }
 648         }
 649     }
 650
 651     return iotlb;
 652 }
 653 #endif /* defined(CONFIG_USER_ONLY) */
 654
 655 #if !defined(CONFIG_USER_ONLY)
 656
 657 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 658 typedef struct subpage_t {
 659     MemoryRegion iomem;
 660     hwaddr base;
 661     uint16_t sub_section[TARGET_PAGE_SIZE];
 662 } subpage_t;
 663
 664 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
 665                              uint16_t section);
 666 static subpage_t *subpage_init(hwaddr base);
 667 static void destroy_page_desc(uint16_t section_index)
 668 {
 669     MemoryRegionSection *section = &phys_sections[section_index];
 670     MemoryRegion *mr = section->mr;
 671
 672     if (mr->subpage) {
 673         subpage_t *subpage = container_of(mr, subpage_t, iomem);
 674         memory_region_destroy(&subpage->iomem);
 675         g_free(subpage);
 676     }
 677 }
 678
 679 static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
 680 {
 681     unsigned i;
 682     PhysPageEntry *p;
 683
 684     if (lp->ptr == PHYS_MAP_NODE_NIL) {
 685         return;
 686     }
 687
 688     p = phys_map_nodes[lp->ptr];
 689     for (i = 0; i < L2_SIZE; ++i) {
 690         if (!p[i].is_leaf) {
 691             destroy_l2_mapping(&p[i], level - 1);
 692         } else {
 693             destroy_page_desc(p[i].ptr);
 694         }
 695     }
 696     lp->is_leaf = 0;
 697     lp->ptr = PHYS_MAP_NODE_NIL;
 698 }
 699
 700 static void destroy_all_mappings(AddressSpaceDispatch *d)
 701 {
 702     destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
 703     phys_map_nodes_reset();
 704 }
 705
 706 static uint16_t phys_section_add(MemoryRegionSection *section)
 707 {
 708     if (phys_sections_nb == phys_sections_nb_alloc) {
 709         phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
 710         phys_sections = g_renew(MemoryRegionSection, phys_sections,
 711                                 phys_sections_nb_alloc);
 712     }
 713     phys_sections[phys_sections_nb] = *section;
 714     return phys_sections_nb++;
 715 }
 716
 717 static void phys_sections_clear(void)
 718 {
 719     phys_sections_nb = 0;
 720 }
 721
 722 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 723 {
 724     subpage_t *subpage;
 725     hwaddr base = section->offset_within_address_space
 726         & TARGET_PAGE_MASK;
 727     MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
 728     MemoryRegionSection subsection = {
 729         .offset_within_address_space = base,
 730         .size = TARGET_PAGE_SIZE,
 731     };
 732     hwaddr start, end;
 733
 734     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 735
 736     if (!(existing->mr->subpage)) {
 737         subpage = subpage_init(base);
 738         subsection.mr = &subpage->iomem;
 739         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
 740                       phys_section_add(&subsection));
 741     } else {
 742         subpage = container_of(existing->mr, subpage_t, iomem);
 743     }
 744     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
 745     end = start + section->size - 1;
 746     subpage_register(subpage, start, end, phys_section_add(section));
 747 }
 748
 749
 750 static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 751 {
 752     hwaddr start_addr = section->offset_within_address_space;
 753     ram_addr_t size = section->size;
 754     hwaddr addr;
 755     uint16_t section_index = phys_section_add(section);
 756
 757     assert(size);
 758
 759     addr = start_addr;
 760     phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
 761                   section_index);
 762 }
 763
 764 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 765 {
 766     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
 767     MemoryRegionSection now = *section, remain = *section;
 768
 769     if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
 770         || (now.size < TARGET_PAGE_SIZE)) {
 771         now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
 772                        - now.offset_within_address_space,
 773                        now.size);
 774         register_subpage(d, &now);
 775         remain.size -= now.size;
 776         remain.offset_within_address_space += now.size;
 777         remain.offset_within_region += now.size;
 778     }
 779     while (remain.size >= TARGET_PAGE_SIZE) {
 780         now = remain;
 781         if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
 782             now.size = TARGET_PAGE_SIZE;
 783             register_subpage(d, &now);
 784         } else {
 785             now.size &= TARGET_PAGE_MASK;
 786             register_multipage(d, &now);
 787         }
 788         remain.size -= now.size;
 789         remain.offset_within_address_space += now.size;
 790         remain.offset_within_region += now.size;
 791     }
 792     now = remain;
 793     if (now.size) {
 794         register_subpage(d, &now);
 795     }
 796 }
 797
 798 void qemu_flush_coalesced_mmio_buffer(void)
 799 {
 800     if (kvm_enabled())
 801         kvm_flush_coalesced_mmio_buffer();
 802 }
 803
 804 #if defined(__linux__) && !defined(TARGET_S390X)
 805
 806 #include <sys/vfs.h>
 807
 808 #define HUGETLBFS_MAGIC       0x958458f6
 809
 810 static long gethugepagesize(const char *path)
 811 {
 812     struct statfs fs;
 813     int ret;
 814
 815     do {
 816         ret = statfs(path, &fs);
 817     } while (ret != 0 && errno == EINTR);
 818
 819     if (ret != 0) {
 820         perror(path);
 821         return 0;
 822     }
 823
 824     if (fs.f_type != HUGETLBFS_MAGIC)
 825         fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
 826
 827     return fs.f_bsize;
 828 }
 829
 830 static void *file_ram_alloc(RAMBlock *block,
 831                             ram_addr_t memory,
 832                             const char *path)
 833 {
 834     char *filename;
 835     void *area;
 836     int fd;
 837 #ifdef MAP_POPULATE
 838     int flags;
 839 #endif
 840     unsigned long hpagesize;
 841
 842     hpagesize = gethugepagesize(path);
 843     if (!hpagesize) {
 844         return NULL;
 845     }
 846
 847     if (memory < hpagesize) {
 848         return NULL;
 849     }
 850
 851     if (kvm_enabled() && !kvm_has_sync_mmu()) {
 852         fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
 853         return NULL;
 854     }
 855
 856     if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
 857         return NULL;
 858     }
 859
 860     fd = mkstemp(filename);
 861     if (fd < 0) {
 862         perror("unable to create backing store for hugepages");
 863         free(filename);
 864         return NULL;
 865     }
 866     unlink(filename);
 867     free(filename);
 868
 869     memory = (memory+hpagesize-1) & ~(hpagesize-1);
 870
 871     /*
 872      * ftruncate is not supported by hugetlbfs in older
 873      * hosts, so don't bother bailing out on errors.
 874      * If anything goes wrong with it under other filesystems,
 875      * mmap will fail.
 876      */
 877     if (ftruncate(fd, memory))
 878         perror("ftruncate");
 879
 880 #ifdef MAP_POPULATE
 881     /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
 882      * MAP_PRIVATE is requested.  For mem_prealloc we mmap as MAP_SHARED
 883      * to sidestep this quirk.
 884      */
 885     flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
 886     area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
 887 #else
 888     area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
 889 #endif
 890     if (area == MAP_FAILED) {
 891         perror("file_ram_alloc: can't mmap RAM pages");
 892         close(fd);
 893         return (NULL);
 894     }
 895     block->fd = fd;
 896     return area;
 897 }
 898 #endif
 899
 900 static ram_addr_t find_ram_offset(ram_addr_t size)
 901 {
 902     RAMBlock *block, *next_block;
 903     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
 904
 905     if (QTAILQ_EMPTY(&ram_list.blocks))
 906         return 0;
 907
 908     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
 909         ram_addr_t end, next = RAM_ADDR_MAX;
 910
 911         end = block->offset + block->length;
 912
 913         QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
 914             if (next_block->offset >= end) {
 915                 next = MIN(next, next_block->offset);
 916             }
 917         }
 918         if (next - end >= size && next - end < mingap) {
 919             offset = end;
 920             mingap = next - end;
 921         }
 922     }
 923
 924     if (offset == RAM_ADDR_MAX) {
 925         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
 926                 (uint64_t)size);
 927         abort();
 928     }
 929
 930     return offset;
 931 }
 932
 933 ram_addr_t last_ram_offset(void)
 934 {
 935     RAMBlock *block;
 936     ram_addr_t last = 0;
 937
 938     QTAILQ_FOREACH(block, &ram_list.blocks, next)
 939         last = MAX(last, block->offset + block->length);
 940
 941     return last;
 942 }
 943
 944 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
 945 {
 946     int ret;
 947     QemuOpts *machine_opts;
 948
 949     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
 950     machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
 951     if (machine_opts &&
 952         !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
 953         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
 954         if (ret) {
 955             perror("qemu_madvise");
 956             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
 957                             "but dump_guest_core=off specified\n");
 958         }
 959     }
 960 }
 961
 962 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
 963 {
 964     RAMBlock *new_block, *block;
 965
 966     new_block = NULL;
 967     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
 968         if (block->offset == addr) {
 969             new_block = block;
 970             break;
 971         }
 972     }
 973     assert(new_block);
 974     assert(!new_block->idstr[0]);
 975
 976     if (dev) {
 977         char *id = qdev_get_dev_path(dev);
 978         if (id) {
 979             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
 980             g_free(id);
 981         }
 982     }
 983     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
 984
 985     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
 986         if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
 987             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
 988                     new_block->idstr);
 989             abort();
 990         }
 991     }
 992 }
 993
 994 static int memory_try_enable_merging(void *addr, size_t len)
 995 {
 996     QemuOpts *opts;
 997
 998     opts = qemu_opts_find(qemu_find_opts("machine"), 0);
 999     if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1000         /* disabled by the user */
1001         return 0;
1002     }
1003
1004     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1005 }
1006
1007 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1008                                    MemoryRegion *mr)
1009 {
1010     RAMBlock *block, *new_block;
1011
1012     size = TARGET_PAGE_ALIGN(size);
1013     new_block = g_malloc0(sizeof(*new_block));
1014
1015     new_block->mr = mr;
1016     new_block->offset = find_ram_offset(size);
1017     if (host) {
1018         new_block->host = host;
1019         new_block->flags |= RAM_PREALLOC_MASK;
1020     } else {
1021         if (mem_path) {
1022 #if defined (__linux__) && !defined(TARGET_S390X)
1023             new_block->host = file_ram_alloc(new_block, size, mem_path);
1024             if (!new_block->host) {
1025                 new_block->host = qemu_vmalloc(size);
1026                 memory_try_enable_merging(new_block->host, size);
1027             }
1028 #else
1029             fprintf(stderr, "-mem-path option unsupported\n");
1030             exit(1);
1031 #endif
1032         } else {
1033             if (xen_enabled()) {
1034                 xen_ram_alloc(new_block->offset, size, mr);
1035             } else if (kvm_enabled()) {
1036                 /* some s390/kvm configurations have special constraints */
1037                 new_block->host = kvm_vmalloc(size);
1038             } else {
1039                 new_block->host = qemu_vmalloc(size);
1040             }
1041             memory_try_enable_merging(new_block->host, size);
1042         }
1043     }
1044     new_block->length = size;
1045
1046     /* Keep the list sorted from biggest to smallest block.  */
1047     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1048         if (block->length < new_block->length) {
1049             break;
1050         }
1051     }
1052     if (block) {
1053         QTAILQ_INSERT_BEFORE(block, new_block, next);
1054     } else {
1055         QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1056     }
1057     ram_list.mru_block = NULL;
1058
1059     ram_list.version++;
1060
1061     ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1062                                        last_ram_offset() >> TARGET_PAGE_BITS);
1063     memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1064            0, size >> TARGET_PAGE_BITS);
1065     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1066
1067     qemu_ram_setup_dump(new_block->host, size);
1068     qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1069
1070     if (kvm_enabled())
1071         kvm_setup_guest_memory(new_block->host, size);
1072
1073     return new_block->offset;
1074 }
1075
1076 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1077 {
1078     return qemu_ram_alloc_from_ptr(size, NULL, mr);
1079 }
1080
1081 void qemu_ram_free_from_ptr(ram_addr_t addr)
1082 {
1083     RAMBlock *block;
1084
1085     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1086         if (addr == block->offset) {
1087             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1088             ram_list.mru_block = NULL;
1089             ram_list.version++;
1090             g_free(block);
1091             return;
1092         }
1093     }
1094 }
1095
1096 void qemu_ram_free(ram_addr_t addr)
1097 {
1098     RAMBlock *block;
1099
1100     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1101         if (addr == block->offset) {
1102             QTAILQ_REMOVE(&ram_list.blocks, block, next);
1103             ram_list.mru_block = NULL;
1104             ram_list.version++;
1105             if (block->flags & RAM_PREALLOC_MASK) {
1106                 ;
1107             } else if (mem_path) {
1108 #if defined (__linux__) && !defined(TARGET_S390X)
1109                 if (block->fd) {
1110                     munmap(block->host, block->length);
1111                     close(block->fd);
1112                 } else {
1113                     qemu_vfree(block->host);
1114                 }
1115 #else
1116                 abort();
1117 #endif
1118             } else {
1119 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1120                 munmap(block->host, block->length);
1121 #else
1122                 if (xen_enabled()) {
1123                     xen_invalidate_map_cache_entry(block->host);
1124                 } else {
1125                     qemu_vfree(block->host);
1126                 }
1127 #endif
1128             }
1129             g_free(block);
1130             return;
1131         }
1132     }
1133
1134 }
1135
1136 #ifndef _WIN32
1137 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1138 {
1139     RAMBlock *block;
1140     ram_addr_t offset;
1141     int flags;
1142     void *area, *vaddr;
1143
1144     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1145         offset = addr - block->offset;
1146         if (offset < block->length) {
1147             vaddr = block->host + offset;
1148             if (block->flags & RAM_PREALLOC_MASK) {
1149                 ;
1150             } else {
1151                 flags = MAP_FIXED;
1152                 munmap(vaddr, length);
1153                 if (mem_path) {
1154 #if defined(__linux__) && !defined(TARGET_S390X)
1155                     if (block->fd) {
1156 #ifdef MAP_POPULATE
1157                         flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1158                             MAP_PRIVATE;
1159 #else
1160                         flags |= MAP_PRIVATE;
1161 #endif
1162                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1163                                     flags, block->fd, offset);
1164                     } else {
1165                         flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1166                         area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1167                                     flags, -1, 0);
1168                     }
1169 #else
1170                     abort();
1171 #endif
1172                 } else {
1173 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1174                     flags |= MAP_SHARED | MAP_ANONYMOUS;
1175                     area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1176                                 flags, -1, 0);
1177 #else
1178                     flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1179                     area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1180                                 flags, -1, 0);
1181 #endif
1182                 }
1183                 if (area != vaddr) {
1184                     fprintf(stderr, "Could not remap addr: "
1185                             RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1186                             length, addr);
1187                     exit(1);
1188                 }
1189                 memory_try_enable_merging(vaddr, length);
1190                 qemu_ram_setup_dump(vaddr, length);
1191             }
1192             return;
1193         }
1194     }
1195 }
1196 #endif /* !_WIN32 */
1197
1198 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1199    With the exception of the softmmu code in this file, this should
1200    only be used for local memory (e.g. video ram) that the device owns,
1201    and knows it isn't going to access beyond the end of the block.
1202
1203    It should not be used for general purpose DMA.
1204    Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1205  */
1206 void *qemu_get_ram_ptr(ram_addr_t addr)
1207 {
1208     RAMBlock *block;
1209
1210     block = ram_list.mru_block;
1211     if (block && addr - block->offset < block->length) {
1212         goto found;
1213     }
1214     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1215         if (addr - block->offset < block->length) {
1216             goto found;
1217         }
1218     }
1219
1220     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1221     abort();
1222
1223 found:
1224     ram_list.mru_block = block;
1225     if (xen_enabled()) {
1226         /* We need to check if the requested address is in the RAM
1227          * because we don't want to map the entire memory in QEMU.
1228          * In that case just map until the end of the page.
1229          */
1230         if (block->offset == 0) {
1231             return xen_map_cache(addr, 0, 0);
1232         } else if (block->host == NULL) {
1233             block->host =
1234                 xen_map_cache(block->offset, block->length, 1);
1235         }
1236     }
1237     return block->host + (addr - block->offset);
1238 }
1239
1240 /* Return a host pointer to ram allocated with qemu_ram_alloc.  Same as
1241  * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1242  *
1243  * ??? Is this still necessary?
1244  */
1245 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1246 {
1247     RAMBlock *block;
1248
1249     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1250         if (addr - block->offset < block->length) {
1251             if (xen_enabled()) {
1252                 /* We need to check if the requested address is in the RAM
1253                  * because we don't want to map the entire memory in QEMU.
1254                  * In that case just map until the end of the page.
1255                  */
1256                 if (block->offset == 0) {
1257                     return xen_map_cache(addr, 0, 0);
1258                 } else if (block->host == NULL) {
1259                     block->host =
1260                         xen_map_cache(block->offset, block->length, 1);
1261                 }
1262             }
1263             return block->host + (addr - block->offset);
1264         }
1265     }
1266
1267     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1268     abort();
1269
1270     return NULL;
1271 }
1272
1273 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1274  * but takes a size argument */
1275 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1276 {
1277     if (*size == 0) {
1278         return NULL;
1279     }
1280     if (xen_enabled()) {
1281         return xen_map_cache(addr, *size, 1);
1282     } else {
1283         RAMBlock *block;
1284
1285         QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1286             if (addr - block->offset < block->length) {
1287                 if (addr - block->offset + *size > block->length)
1288                     *size = block->length - addr + block->offset;
1289                 return block->host + (addr - block->offset);
1290             }
1291         }
1292
1293         fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1294         abort();
1295     }
1296 }
1297
1298 void qemu_put_ram_ptr(void *addr)
1299 {
1300     trace_qemu_put_ram_ptr(addr);
1301 }
1302
1303 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1304 {
1305     RAMBlock *block;
1306     uint8_t *host = ptr;
1307
1308     if (xen_enabled()) {
1309         *ram_addr = xen_ram_addr_from_mapcache(ptr);
1310         return 0;
1311     }
1312
1313     QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1314         /* This case append when the block is not mapped. */
1315         if (block->host == NULL) {
1316             continue;
1317         }
1318         if (host - block->host < block->length) {
1319             *ram_addr = block->offset + (host - block->host);
1320             return 0;
1321         }
1322     }
1323
1324     return -1;
1325 }
1326
1327 /* Some of the softmmu routines need to translate from a host pointer
1328    (typically a TLB entry) back to a ram offset.  */
1329 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1330 {
1331     ram_addr_t ram_addr;
1332
1333     if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1334         fprintf(stderr, "Bad ram pointer %p\n", ptr);
1335         abort();
1336     }
1337     return ram_addr;
1338 }
1339
1340 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1341                                     unsigned size)
1342 {
1343 #ifdef DEBUG_UNASSIGNED
1344     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1345 #endif
1346 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1347     cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, size);
1348 #endif
1349     return 0;
1350 }
1351
1352 static void unassigned_mem_write(void *opaque, hwaddr addr,
1353                                  uint64_t val, unsigned size)
1354 {
1355 #ifdef DEBUG_UNASSIGNED
1356     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1357 #endif
1358 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1359     cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, size);
1360 #endif
1361 }
1362
1363 static const MemoryRegionOps unassigned_mem_ops = {
1364     .read = unassigned_mem_read,
1365     .write = unassigned_mem_write,
1366     .endianness = DEVICE_NATIVE_ENDIAN,
1367 };
1368
1369 static uint64_t error_mem_read(void *opaque, hwaddr addr,
1370                                unsigned size)
1371 {
1372     abort();
1373 }
1374
1375 static void error_mem_write(void *opaque, hwaddr addr,
1376                             uint64_t value, unsigned size)
1377 {
1378     abort();
1379 }
1380
1381 static const MemoryRegionOps error_mem_ops = {
1382     .read = error_mem_read,
1383     .write = error_mem_write,
1384     .endianness = DEVICE_NATIVE_ENDIAN,
1385 };
1386
1387 static const MemoryRegionOps rom_mem_ops = {
1388     .read = error_mem_read,
1389     .write = unassigned_mem_write,
1390     .endianness = DEVICE_NATIVE_ENDIAN,
1391 };
1392
1393 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1394                                uint64_t val, unsigned size)
1395 {
1396     int dirty_flags;
1397     dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1398     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1399 #if !defined(CONFIG_USER_ONLY)
1400         tb_invalidate_phys_page_fast(ram_addr, size);
1401         dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1402 #endif
1403     }
1404     switch (size) {
1405     case 1:
1406         stb_p(qemu_get_ram_ptr(ram_addr), val);
1407         break;
1408     case 2:
1409         stw_p(qemu_get_ram_ptr(ram_addr), val);
1410         break;
1411     case 4:
1412         stl_p(qemu_get_ram_ptr(ram_addr), val);
1413         break;
1414     default:
1415         abort();
1416     }
1417     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1418     cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1419     /* we remove the notdirty callback only if the code has been
1420        flushed */
1421     if (dirty_flags == 0xff)
1422         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1423 }
1424
1425 static const MemoryRegionOps notdirty_mem_ops = {
1426     .read = error_mem_read,
1427     .write = notdirty_mem_write,
1428     .endianness = DEVICE_NATIVE_ENDIAN,
1429 };
1430
1431 /* Generate a debug exception if a watchpoint has been hit.  */
1432 static void check_watchpoint(int offset, int len_mask, int flags)
1433 {
1434     CPUArchState *env = cpu_single_env;
1435     target_ulong pc, cs_base;
1436     target_ulong vaddr;
1437     CPUWatchpoint *wp;
1438     int cpu_flags;
1439
1440     if (env->watchpoint_hit) {
1441         /* We re-entered the check after replacing the TB. Now raise
1442          * the debug interrupt so that is will trigger after the
1443          * current instruction. */
1444         cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
1445         return;
1446     }
1447     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1448     QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1449         if ((vaddr == (wp->vaddr & len_mask) ||
1450              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1451             wp->flags |= BP_WATCHPOINT_HIT;
1452             if (!env->watchpoint_hit) {
1453                 env->watchpoint_hit = wp;
1454                 tb_check_watchpoint(env);
1455                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1456                     env->exception_index = EXCP_DEBUG;
1457                     cpu_loop_exit(env);
1458                 } else {
1459                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1460                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1461                     cpu_resume_from_signal(env, NULL);
1462                 }
1463             }
1464         } else {
1465             wp->flags &= ~BP_WATCHPOINT_HIT;
1466         }
1467     }
1468 }
1469
1470 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
1471    so these check for a hit then pass through to the normal out-of-line
1472    phys routines.  */
1473 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1474                                unsigned size)
1475 {
1476     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1477     switch (size) {
1478     case 1: return ldub_phys(addr);
1479     case 2: return lduw_phys(addr);
1480     case 4: return ldl_phys(addr);
1481     default: abort();
1482     }
1483 }
1484
1485 static void watch_mem_write(void *opaque, hwaddr addr,
1486                             uint64_t val, unsigned size)
1487 {
1488     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1489     switch (size) {
1490     case 1:
1491         stb_phys(addr, val);
1492         break;
1493     case 2:
1494         stw_phys(addr, val);
1495         break;
1496     case 4:
1497         stl_phys(addr, val);
1498         break;
1499     default: abort();
1500     }
1501 }
1502
1503 static const MemoryRegionOps watch_mem_ops = {
1504     .read = watch_mem_read,
1505     .write = watch_mem_write,
1506     .endianness = DEVICE_NATIVE_ENDIAN,
1507 };
1508
1509 static uint64_t subpage_read(void *opaque, hwaddr addr,
1510                              unsigned len)
1511 {
1512     subpage_t *mmio = opaque;
1513     unsigned int idx = SUBPAGE_IDX(addr);
1514     MemoryRegionSection *section;
1515 #if defined(DEBUG_SUBPAGE)
1516     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
1517            mmio, len, addr, idx);
1518 #endif
1519
1520     section = &phys_sections[mmio->sub_section[idx]];
1521     addr += mmio->base;
1522     addr -= section->offset_within_address_space;
1523     addr += section->offset_within_region;
1524     return io_mem_read(section->mr, addr, len);
1525 }
1526
1527 static void subpage_write(void *opaque, hwaddr addr,
1528                           uint64_t value, unsigned len)
1529 {
1530     subpage_t *mmio = opaque;
1531     unsigned int idx = SUBPAGE_IDX(addr);
1532     MemoryRegionSection *section;
1533 #if defined(DEBUG_SUBPAGE)
1534     printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1535            " idx %d value %"PRIx64"\n",
1536            __func__, mmio, len, addr, idx, value);
1537 #endif
1538
1539     section = &phys_sections[mmio->sub_section[idx]];
1540     addr += mmio->base;
1541     addr -= section->offset_within_address_space;
1542     addr += section->offset_within_region;
1543     io_mem_write(section->mr, addr, value, len);
1544 }
1545
1546 static const MemoryRegionOps subpage_ops = {
1547     .read = subpage_read,
1548     .write = subpage_write,
1549     .endianness = DEVICE_NATIVE_ENDIAN,
1550 };
1551
1552 static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
1553                                  unsigned size)
1554 {
1555     ram_addr_t raddr = addr;
1556     void *ptr = qemu_get_ram_ptr(raddr);
1557     switch (size) {
1558     case 1: return ldub_p(ptr);
1559     case 2: return lduw_p(ptr);
1560     case 4: return ldl_p(ptr);
1561     default: abort();
1562     }
1563 }
1564
1565 static void subpage_ram_write(void *opaque, hwaddr addr,
1566                               uint64_t value, unsigned size)
1567 {
1568     ram_addr_t raddr = addr;
1569     void *ptr = qemu_get_ram_ptr(raddr);
1570     switch (size) {
1571     case 1: return stb_p(ptr, value);
1572     case 2: return stw_p(ptr, value);
1573     case 4: return stl_p(ptr, value);
1574     default: abort();
1575     }
1576 }
1577
1578 static const MemoryRegionOps subpage_ram_ops = {
1579     .read = subpage_ram_read,
1580     .write = subpage_ram_write,
1581     .endianness = DEVICE_NATIVE_ENDIAN,
1582 };
1583
1584 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1585                              uint16_t section)
1586 {
1587     int idx, eidx;
1588
1589     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1590         return -1;
1591     idx = SUBPAGE_IDX(start);
1592     eidx = SUBPAGE_IDX(end);
1593 #if defined(DEBUG_SUBPAGE)
1594     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1595            mmio, start, end, idx, eidx, memory);
1596 #endif
1597     if (memory_region_is_ram(phys_sections[section].mr)) {
1598         MemoryRegionSection new_section = phys_sections[section];
1599         new_section.mr = &io_mem_subpage_ram;
1600         section = phys_section_add(&new_section);
1601     }
1602     for (; idx <= eidx; idx++) {
1603         mmio->sub_section[idx] = section;
1604     }
1605
1606     return 0;
1607 }
1608
1609 static subpage_t *subpage_init(hwaddr base)
1610 {
1611     subpage_t *mmio;
1612
1613     mmio = g_malloc0(sizeof(subpage_t));
1614
1615     mmio->base = base;
1616     memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1617                           "subpage", TARGET_PAGE_SIZE);
1618     mmio->iomem.subpage = true;
1619 #if defined(DEBUG_SUBPAGE)
1620     printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1621            mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1622 #endif
1623     subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1624
1625     return mmio;
1626 }
1627
1628 static uint16_t dummy_section(MemoryRegion *mr)
1629 {
1630     MemoryRegionSection section = {
1631         .mr = mr,
1632         .offset_within_address_space = 0,
1633         .offset_within_region = 0,
1634         .size = UINT64_MAX,
1635     };
1636
1637     return phys_section_add(&section);
1638 }
1639
1640 MemoryRegion *iotlb_to_region(hwaddr index)
1641 {
1642     return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1643 }
1644
1645 static void io_mem_init(void)
1646 {
1647     memory_region_init_io(&io_mem_ram, &error_mem_ops, NULL, "ram", UINT64_MAX);
1648     memory_region_init_io(&io_mem_rom, &rom_mem_ops, NULL, "rom", UINT64_MAX);
1649     memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1650                           "unassigned", UINT64_MAX);
1651     memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1652                           "notdirty", UINT64_MAX);
1653     memory_region_init_io(&io_mem_subpage_ram, &subpage_ram_ops, NULL,
1654                           "subpage-ram", UINT64_MAX);
1655     memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1656                           "watch", UINT64_MAX);
1657 }
1658
1659 static void mem_begin(MemoryListener *listener)
1660 {
1661     AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1662
1663     destroy_all_mappings(d);
1664     d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1665 }
1666
1667 static void core_begin(MemoryListener *listener)
1668 {
1669     phys_sections_clear();
1670     phys_section_unassigned = dummy_section(&io_mem_unassigned);
1671     phys_section_notdirty = dummy_section(&io_mem_notdirty);
1672     phys_section_rom = dummy_section(&io_mem_rom);
1673     phys_section_watch = dummy_section(&io_mem_watch);
1674 }
1675
1676 static void tcg_commit(MemoryListener *listener)
1677 {
1678     CPUArchState *env;
1679
1680     /* since each CPU stores ram addresses in its TLB cache, we must
1681        reset the modified entries */
1682     /* XXX: slow ! */
1683     for(env = first_cpu; env != NULL; env = env->next_cpu) {
1684         tlb_flush(env, 1);
1685     }
1686 }
1687
1688 static void core_log_global_start(MemoryListener *listener)
1689 {
1690     cpu_physical_memory_set_dirty_tracking(1);
1691 }
1692
1693 static void core_log_global_stop(MemoryListener *listener)
1694 {
1695     cpu_physical_memory_set_dirty_tracking(0);
1696 }
1697
1698 static void io_region_add(MemoryListener *listener,
1699                           MemoryRegionSection *section)
1700 {
1701     MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1702
1703     mrio->mr = section->mr;
1704     mrio->offset = section->offset_within_region;
1705     iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1706                  section->offset_within_address_space, section->size);
1707     ioport_register(&mrio->iorange);
1708 }
1709
1710 static void io_region_del(MemoryListener *listener,
1711                           MemoryRegionSection *section)
1712 {
1713     isa_unassign_ioport(section->offset_within_address_space, section->size);
1714 }
1715
1716 static MemoryListener core_memory_listener = {
1717     .begin = core_begin,
1718     .log_global_start = core_log_global_start,
1719     .log_global_stop = core_log_global_stop,
1720     .priority = 1,
1721 };
1722
1723 static MemoryListener io_memory_listener = {
1724     .region_add = io_region_add,
1725     .region_del = io_region_del,
1726     .priority = 0,
1727 };
1728
1729 static MemoryListener tcg_memory_listener = {
1730     .commit = tcg_commit,
1731 };
1732
1733 void address_space_init_dispatch(AddressSpace *as)
1734 {
1735     AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1736
1737     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1738     d->listener = (MemoryListener) {
1739         .begin = mem_begin,
1740         .region_add = mem_add,
1741         .region_nop = mem_add,
1742         .priority = 0,
1743     };
1744     as->dispatch = d;
1745     memory_listener_register(&d->listener, as);
1746 }
1747
1748 void address_space_destroy_dispatch(AddressSpace *as)
1749 {
1750     AddressSpaceDispatch *d = as->dispatch;
1751
1752     memory_listener_unregister(&d->listener);
1753     destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1754     g_free(d);
1755     as->dispatch = NULL;
1756 }
1757
1758 static void memory_map_init(void)
1759 {
1760     system_memory = g_malloc(sizeof(*system_memory));
1761     memory_region_init(system_memory, "system", INT64_MAX);
1762     address_space_init(&address_space_memory, system_memory);
1763     address_space_memory.name = "memory";
1764
1765     system_io = g_malloc(sizeof(*system_io));
1766     memory_region_init(system_io, "io", 65536);
1767     address_space_init(&address_space_io, system_io);
1768     address_space_io.name = "I/O";
1769
1770     memory_listener_register(&core_memory_listener, &address_space_memory);
1771     memory_listener_register(&io_memory_listener, &address_space_io);
1772     memory_listener_register(&tcg_memory_listener, &address_space_memory);
1773
1774     dma_context_init(&dma_context_memory, &address_space_memory,
1775                      NULL, NULL, NULL);
1776 }
1777
1778 MemoryRegion *get_system_memory(void)
1779 {
1780     return system_memory;
1781 }
1782
1783 MemoryRegion *get_system_io(void)
1784 {
1785     return system_io;
1786 }
1787
1788 #endif /* !defined(CONFIG_USER_ONLY) */
1789
1790 /* physical memory access (slow version, mainly for debug) */
1791 #if defined(CONFIG_USER_ONLY)
1792 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1793                         uint8_t *buf, int len, int is_write)
1794 {
1795     int l, flags;
1796     target_ulong page;
1797     void * p;
1798
1799     while (len > 0) {
1800         page = addr & TARGET_PAGE_MASK;
1801         l = (page + TARGET_PAGE_SIZE) - addr;
1802         if (l > len)
1803             l = len;
1804         flags = page_get_flags(page);
1805         if (!(flags & PAGE_VALID))
1806             return -1;
1807         if (is_write) {
1808             if (!(flags & PAGE_WRITE))
1809                 return -1;
1810             /* XXX: this code should not depend on lock_user */
1811             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1812                 return -1;
1813             memcpy(p, buf, l);
1814             unlock_user(p, addr, l);
1815         } else {
1816             if (!(flags & PAGE_READ))
1817                 return -1;
1818             /* XXX: this code should not depend on lock_user */
1819             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1820                 return -1;
1821             memcpy(buf, p, l);
1822             unlock_user(p, addr, 0);
1823         }
1824         len -= l;
1825         buf += l;
1826         addr += l;
1827     }
1828     return 0;
1829 }
1830
1831 #else
1832
1833 static void invalidate_and_set_dirty(hwaddr addr,
1834                                      hwaddr length)
1835 {
1836     if (!cpu_physical_memory_is_dirty(addr)) {
1837         /* invalidate code */
1838         tb_invalidate_phys_page_range(addr, addr + length, 0);
1839         /* set dirty bit */
1840         cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1841     }
1842     xen_modified_memory(addr, length);
1843 }
1844
1845 void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1846                       int len, bool is_write)
1847 {
1848     AddressSpaceDispatch *d = as->dispatch;
1849     int l;
1850     uint8_t *ptr;
1851     uint32_t val;
1852     hwaddr page;
1853     MemoryRegionSection *section;
1854
1855     while (len > 0) {
1856         page = addr & TARGET_PAGE_MASK;
1857         l = (page + TARGET_PAGE_SIZE) - addr;
1858         if (l > len)
1859             l = len;
1860         section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1861
1862         if (is_write) {
1863             if (!memory_region_is_ram(section->mr)) {
1864                 hwaddr addr1;
1865                 addr1 = memory_region_section_addr(section, addr);
1866                 /* XXX: could force cpu_single_env to NULL to avoid
1867                    potential bugs */
1868                 if (l >= 4 && ((addr1 & 3) == 0)) {
1869                     /* 32 bit write access */
1870                     val = ldl_p(buf);
1871                     io_mem_write(section->mr, addr1, val, 4);
1872                     l = 4;
1873                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1874                     /* 16 bit write access */
1875                     val = lduw_p(buf);
1876                     io_mem_write(section->mr, addr1, val, 2);
1877                     l = 2;
1878                 } else {
1879                     /* 8 bit write access */
1880                     val = ldub_p(buf);
1881                     io_mem_write(section->mr, addr1, val, 1);
1882                     l = 1;
1883                 }
1884             } else if (!section->readonly) {
1885                 ram_addr_t addr1;
1886                 addr1 = memory_region_get_ram_addr(section->mr)
1887                     + memory_region_section_addr(section, addr);
1888                 /* RAM case */
1889                 ptr = qemu_get_ram_ptr(addr1);
1890                 memcpy(ptr, buf, l);
1891                 invalidate_and_set_dirty(addr1, l);
1892                 qemu_put_ram_ptr(ptr);
1893             }
1894         } else {
1895             if (!(memory_region_is_ram(section->mr) ||
1896                   memory_region_is_romd(section->mr))) {
1897                 hwaddr addr1;
1898                 /* I/O case */
1899                 addr1 = memory_region_section_addr(section, addr);
1900                 if (l >= 4 && ((addr1 & 3) == 0)) {
1901                     /* 32 bit read access */
1902                     val = io_mem_read(section->mr, addr1, 4);
1903                     stl_p(buf, val);
1904                     l = 4;
1905                 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1906                     /* 16 bit read access */
1907                     val = io_mem_read(section->mr, addr1, 2);
1908                     stw_p(buf, val);
1909                     l = 2;
1910                 } else {
1911                     /* 8 bit read access */
1912                     val = io_mem_read(section->mr, addr1, 1);
1913                     stb_p(buf, val);
1914                     l = 1;
1915                 }
1916             } else {
1917                 /* RAM case */
1918                 ptr = qemu_get_ram_ptr(section->mr->ram_addr
1919                                        + memory_region_section_addr(section,
1920                                                                     addr));
1921                 memcpy(buf, ptr, l);
1922                 qemu_put_ram_ptr(ptr);
1923             }
1924         }
1925         len -= l;
1926         buf += l;
1927         addr += l;
1928     }
1929 }
1930
1931 void address_space_write(AddressSpace *as, hwaddr addr,
1932                          const uint8_t *buf, int len)
1933 {
1934     address_space_rw(as, addr, (uint8_t *)buf, len, true);
1935 }
1936
1937 /**
1938  * address_space_read: read from an address space.
1939  *
1940  * @as: #AddressSpace to be accessed
1941  * @addr: address within that address space
1942  * @buf: buffer with the data transferred
1943  */
1944 void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1945 {
1946     address_space_rw(as, addr, buf, len, false);
1947 }
1948
1949
1950 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1951                             int len, int is_write)
1952 {
1953     return address_space_rw(&address_space_memory, addr, buf, len, is_write);
1954 }
1955
1956 /* used for ROM loading : can write in RAM and ROM */
1957 void cpu_physical_memory_write_rom(hwaddr addr,
1958                                    const uint8_t *buf, int len)
1959 {
1960     AddressSpaceDispatch *d = address_space_memory.dispatch;
1961     int l;
1962     uint8_t *ptr;
1963     hwaddr page;
1964     MemoryRegionSection *section;
1965
1966     while (len > 0) {
1967         page = addr & TARGET_PAGE_MASK;
1968         l = (page + TARGET_PAGE_SIZE) - addr;
1969         if (l > len)
1970             l = len;
1971         section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1972
1973         if (!(memory_region_is_ram(section->mr) ||
1974               memory_region_is_romd(section->mr))) {
1975             /* do nothing */
1976         } else {
1977             unsigned long addr1;
1978             addr1 = memory_region_get_ram_addr(section->mr)
1979                 + memory_region_section_addr(section, addr);
1980             /* ROM/RAM case */
1981             ptr = qemu_get_ram_ptr(addr1);
1982             memcpy(ptr, buf, l);
1983             invalidate_and_set_dirty(addr1, l);
1984             qemu_put_ram_ptr(ptr);
1985         }
1986         len -= l;
1987         buf += l;
1988         addr += l;
1989     }
1990 }
1991
1992 typedef struct {
1993     void *buffer;
1994     hwaddr addr;
1995     hwaddr len;
1996 } BounceBuffer;
1997
1998 static BounceBuffer bounce;
1999
2000 typedef struct MapClient {
2001     void *opaque;
2002     void (*callback)(void *opaque);
2003     QLIST_ENTRY(MapClient) link;
2004 } MapClient;
2005
2006 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2007     = QLIST_HEAD_INITIALIZER(map_client_list);
2008
2009 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2010 {
2011     MapClient *client = g_malloc(sizeof(*client));
2012
2013     client->opaque = opaque;
2014     client->callback = callback;
2015     QLIST_INSERT_HEAD(&map_client_list, client, link);
2016     return client;
2017 }
2018
2019 static void cpu_unregister_map_client(void *_client)
2020 {
2021     MapClient *client = (MapClient *)_client;
2022
2023     QLIST_REMOVE(client, link);
2024     g_free(client);
2025 }
2026
2027 static void cpu_notify_map_clients(void)
2028 {
2029     MapClient *client;
2030
2031     while (!QLIST_EMPTY(&map_client_list)) {
2032         client = QLIST_FIRST(&map_client_list);
2033         client->callback(client->opaque);
2034         cpu_unregister_map_client(client);
2035     }
2036 }
2037
2038 /* Map a physical memory region into a host virtual address.
2039  * May map a subset of the requested range, given by and returned in *plen.
2040  * May return NULL if resources needed to perform the mapping are exhausted.
2041  * Use only for reads OR writes - not for read-modify-write operations.
2042  * Use cpu_register_map_client() to know when retrying the map operation is
2043  * likely to succeed.
2044  */
2045 void *address_space_map(AddressSpace *as,
2046                         hwaddr addr,
2047                         hwaddr *plen,
2048                         bool is_write)
2049 {
2050     AddressSpaceDispatch *d = as->dispatch;
2051     hwaddr len = *plen;
2052     hwaddr todo = 0;
2053     int l;
2054     hwaddr page;
2055     MemoryRegionSection *section;
2056     ram_addr_t raddr = RAM_ADDR_MAX;
2057     ram_addr_t rlen;
2058     void *ret;
2059
2060     while (len > 0) {
2061         page = addr & TARGET_PAGE_MASK;
2062         l = (page + TARGET_PAGE_SIZE) - addr;
2063         if (l > len)
2064             l = len;
2065         section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2066
2067         if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
2068             if (todo || bounce.buffer) {
2069                 break;
2070             }
2071             bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2072             bounce.addr = addr;
2073             bounce.len = l;
2074             if (!is_write) {
2075                 address_space_read(as, addr, bounce.buffer, l);
2076             }
2077
2078             *plen = l;
2079             return bounce.buffer;
2080         }
2081         if (!todo) {
2082             raddr = memory_region_get_ram_addr(section->mr)
2083                 + memory_region_section_addr(section, addr);
2084         }
2085
2086         len -= l;
2087         addr += l;
2088         todo += l;
2089     }
2090     rlen = todo;
2091     ret = qemu_ram_ptr_length(raddr, &rlen);
2092     *plen = rlen;
2093     return ret;
2094 }
2095
2096 /* Unmaps a memory region previously mapped by address_space_map().
2097  * Will also mark the memory as dirty if is_write == 1.  access_len gives
2098  * the amount of memory that was actually read or written by the caller.
2099  */
2100 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2101                          int is_write, hwaddr access_len)
2102 {
2103     if (buffer != bounce.buffer) {
2104         if (is_write) {
2105             ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2106             while (access_len) {
2107                 unsigned l;
2108                 l = TARGET_PAGE_SIZE;
2109                 if (l > access_len)
2110                     l = access_len;
2111                 invalidate_and_set_dirty(addr1, l);
2112                 addr1 += l;
2113                 access_len -= l;
2114             }
2115         }
2116         if (xen_enabled()) {
2117             xen_invalidate_map_cache_entry(buffer);
2118         }
2119         return;
2120     }
2121     if (is_write) {
2122         address_space_write(as, bounce.addr, bounce.buffer, access_len);
2123     }
2124     qemu_vfree(bounce.buffer);
2125     bounce.buffer = NULL;
2126     cpu_notify_map_clients();
2127 }
2128
2129 void *cpu_physical_memory_map(hwaddr addr,
2130                               hwaddr *plen,
2131                               int is_write)
2132 {
2133     return address_space_map(&address_space_memory, addr, plen, is_write);
2134 }
2135
2136 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2137                                int is_write, hwaddr access_len)
2138 {
2139     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2140 }
2141
2142 /* warning: addr must be aligned */
2143 static inline uint32_t ldl_phys_internal(hwaddr addr,
2144                                          enum device_endian endian)
2145 {
2146     uint8_t *ptr;
2147     uint32_t val;
2148     MemoryRegionSection *section;
2149
2150     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2151
2152     if (!(memory_region_is_ram(section->mr) ||
2153           memory_region_is_romd(section->mr))) {
2154         /* I/O case */
2155         addr = memory_region_section_addr(section, addr);
2156         val = io_mem_read(section->mr, addr, 4);
2157 #if defined(TARGET_WORDS_BIGENDIAN)
2158         if (endian == DEVICE_LITTLE_ENDIAN) {
2159             val = bswap32(val);
2160         }
2161 #else
2162         if (endian == DEVICE_BIG_ENDIAN) {
2163             val = bswap32(val);
2164         }
2165 #endif
2166     } else {
2167         /* RAM case */
2168         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2169                                 & TARGET_PAGE_MASK)
2170                                + memory_region_section_addr(section, addr));
2171         switch (endian) {
2172         case DEVICE_LITTLE_ENDIAN:
2173             val = ldl_le_p(ptr);
2174             break;
2175         case DEVICE_BIG_ENDIAN:
2176             val = ldl_be_p(ptr);
2177             break;
2178         default:
2179             val = ldl_p(ptr);
2180             break;
2181         }
2182     }
2183     return val;
2184 }
2185
2186 uint32_t ldl_phys(hwaddr addr)
2187 {
2188     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2189 }
2190
2191 uint32_t ldl_le_phys(hwaddr addr)
2192 {
2193     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2194 }
2195
2196 uint32_t ldl_be_phys(hwaddr addr)
2197 {
2198     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2199 }
2200
2201 /* warning: addr must be aligned */
2202 static inline uint64_t ldq_phys_internal(hwaddr addr,
2203                                          enum device_endian endian)
2204 {
2205     uint8_t *ptr;
2206     uint64_t val;
2207     MemoryRegionSection *section;
2208
2209     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2210
2211     if (!(memory_region_is_ram(section->mr) ||
2212           memory_region_is_romd(section->mr))) {
2213         /* I/O case */
2214         addr = memory_region_section_addr(section, addr);
2215
2216         /* XXX This is broken when device endian != cpu endian.
2217                Fix and add "endian" variable check */
2218 #ifdef TARGET_WORDS_BIGENDIAN
2219         val = io_mem_read(section->mr, addr, 4) << 32;
2220         val |= io_mem_read(section->mr, addr + 4, 4);
2221 #else
2222         val = io_mem_read(section->mr, addr, 4);
2223         val |= io_mem_read(section->mr, addr + 4, 4) << 32;
2224 #endif
2225     } else {
2226         /* RAM case */
2227         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2228                                 & TARGET_PAGE_MASK)
2229                                + memory_region_section_addr(section, addr));
2230         switch (endian) {
2231         case DEVICE_LITTLE_ENDIAN:
2232             val = ldq_le_p(ptr);
2233             break;
2234         case DEVICE_BIG_ENDIAN:
2235             val = ldq_be_p(ptr);
2236             break;
2237         default:
2238             val = ldq_p(ptr);
2239             break;
2240         }
2241     }
2242     return val;
2243 }
2244
2245 uint64_t ldq_phys(hwaddr addr)
2246 {
2247     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2248 }
2249
2250 uint64_t ldq_le_phys(hwaddr addr)
2251 {
2252     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2253 }
2254
2255 uint64_t ldq_be_phys(hwaddr addr)
2256 {
2257     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2258 }
2259
2260 /* XXX: optimize */
2261 uint32_t ldub_phys(hwaddr addr)
2262 {
2263     uint8_t val;
2264     cpu_physical_memory_read(addr, &val, 1);
2265     return val;
2266 }
2267
2268 /* warning: addr must be aligned */
2269 static inline uint32_t lduw_phys_internal(hwaddr addr,
2270                                           enum device_endian endian)
2271 {
2272     uint8_t *ptr;
2273     uint64_t val;
2274     MemoryRegionSection *section;
2275
2276     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2277
2278     if (!(memory_region_is_ram(section->mr) ||
2279           memory_region_is_romd(section->mr))) {
2280         /* I/O case */
2281         addr = memory_region_section_addr(section, addr);
2282         val = io_mem_read(section->mr, addr, 2);
2283 #if defined(TARGET_WORDS_BIGENDIAN)
2284         if (endian == DEVICE_LITTLE_ENDIAN) {
2285             val = bswap16(val);
2286         }
2287 #else
2288         if (endian == DEVICE_BIG_ENDIAN) {
2289             val = bswap16(val);
2290         }
2291 #endif
2292     } else {
2293         /* RAM case */
2294         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2295                                 & TARGET_PAGE_MASK)
2296                                + memory_region_section_addr(section, addr));
2297         switch (endian) {
2298         case DEVICE_LITTLE_ENDIAN:
2299             val = lduw_le_p(ptr);
2300             break;
2301         case DEVICE_BIG_ENDIAN:
2302             val = lduw_be_p(ptr);
2303             break;
2304         default:
2305             val = lduw_p(ptr);
2306             break;
2307         }
2308     }
2309     return val;
2310 }
2311
2312 uint32_t lduw_phys(hwaddr addr)
2313 {
2314     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2315 }
2316
2317 uint32_t lduw_le_phys(hwaddr addr)
2318 {
2319     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2320 }
2321
2322 uint32_t lduw_be_phys(hwaddr addr)
2323 {
2324     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2325 }
2326
2327 /* warning: addr must be aligned. The ram page is not masked as dirty
2328    and the code inside is not invalidated. It is useful if the dirty
2329    bits are used to track modified PTEs */
2330 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2331 {
2332     uint8_t *ptr;
2333     MemoryRegionSection *section;
2334
2335     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2336
2337     if (!memory_region_is_ram(section->mr) || section->readonly) {
2338         addr = memory_region_section_addr(section, addr);
2339         if (memory_region_is_ram(section->mr)) {
2340             section = &phys_sections[phys_section_rom];
2341         }
2342         io_mem_write(section->mr, addr, val, 4);
2343     } else {
2344         unsigned long addr1 = (memory_region_get_ram_addr(section->mr)
2345                                & TARGET_PAGE_MASK)
2346             + memory_region_section_addr(section, addr);
2347         ptr = qemu_get_ram_ptr(addr1);
2348         stl_p(ptr, val);
2349
2350         if (unlikely(in_migration)) {
2351             if (!cpu_physical_memory_is_dirty(addr1)) {
2352                 /* invalidate code */
2353                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2354                 /* set dirty bit */
2355                 cpu_physical_memory_set_dirty_flags(
2356                     addr1, (0xff & ~CODE_DIRTY_FLAG));
2357             }
2358         }
2359     }
2360 }
2361
2362 void stq_phys_notdirty(hwaddr addr, uint64_t val)
2363 {
2364     uint8_t *ptr;
2365     MemoryRegionSection *section;
2366
2367     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2368
2369     if (!memory_region_is_ram(section->mr) || section->readonly) {
2370         addr = memory_region_section_addr(section, addr);
2371         if (memory_region_is_ram(section->mr)) {
2372             section = &phys_sections[phys_section_rom];
2373         }
2374 #ifdef TARGET_WORDS_BIGENDIAN
2375         io_mem_write(section->mr, addr, val >> 32, 4);
2376         io_mem_write(section->mr, addr + 4, (uint32_t)val, 4);
2377 #else
2378         io_mem_write(section->mr, addr, (uint32_t)val, 4);
2379         io_mem_write(section->mr, addr + 4, val >> 32, 4);
2380 #endif
2381     } else {
2382         ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2383                                 & TARGET_PAGE_MASK)
2384                                + memory_region_section_addr(section, addr));
2385         stq_p(ptr, val);
2386     }
2387 }
2388
2389 /* warning: addr must be aligned */
2390 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2391                                      enum device_endian endian)
2392 {
2393     uint8_t *ptr;
2394     MemoryRegionSection *section;
2395
2396     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2397
2398     if (!memory_region_is_ram(section->mr) || section->readonly) {
2399         addr = memory_region_section_addr(section, addr);
2400         if (memory_region_is_ram(section->mr)) {
2401             section = &phys_sections[phys_section_rom];
2402         }
2403 #if defined(TARGET_WORDS_BIGENDIAN)
2404         if (endian == DEVICE_LITTLE_ENDIAN) {
2405             val = bswap32(val);
2406         }
2407 #else
2408         if (endian == DEVICE_BIG_ENDIAN) {
2409             val = bswap32(val);
2410         }
2411 #endif
2412         io_mem_write(section->mr, addr, val, 4);
2413     } else {
2414         unsigned long addr1;
2415         addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2416             + memory_region_section_addr(section, addr);
2417         /* RAM case */
2418         ptr = qemu_get_ram_ptr(addr1);
2419         switch (endian) {
2420         case DEVICE_LITTLE_ENDIAN:
2421             stl_le_p(ptr, val);
2422             break;
2423         case DEVICE_BIG_ENDIAN:
2424             stl_be_p(ptr, val);
2425             break;
2426         default:
2427             stl_p(ptr, val);
2428             break;
2429         }
2430         invalidate_and_set_dirty(addr1, 4);
2431     }
2432 }
2433
2434 void stl_phys(hwaddr addr, uint32_t val)
2435 {
2436     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2437 }
2438
2439 void stl_le_phys(hwaddr addr, uint32_t val)
2440 {
2441     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2442 }
2443
2444 void stl_be_phys(hwaddr addr, uint32_t val)
2445 {
2446     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2447 }
2448
2449 /* XXX: optimize */
2450 void stb_phys(hwaddr addr, uint32_t val)
2451 {
2452     uint8_t v = val;
2453     cpu_physical_memory_write(addr, &v, 1);
2454 }
2455
2456 /* warning: addr must be aligned */
2457 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2458                                      enum device_endian endian)
2459 {
2460     uint8_t *ptr;
2461     MemoryRegionSection *section;
2462
2463     section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2464
2465     if (!memory_region_is_ram(section->mr) || section->readonly) {
2466         addr = memory_region_section_addr(section, addr);
2467         if (memory_region_is_ram(section->mr)) {
2468             section = &phys_sections[phys_section_rom];
2469         }
2470 #if defined(TARGET_WORDS_BIGENDIAN)
2471         if (endian == DEVICE_LITTLE_ENDIAN) {
2472             val = bswap16(val);
2473         }
2474 #else
2475         if (endian == DEVICE_BIG_ENDIAN) {
2476             val = bswap16(val);
2477         }
2478 #endif
2479         io_mem_write(section->mr, addr, val, 2);
2480     } else {
2481         unsigned long addr1;
2482         addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2483             + memory_region_section_addr(section, addr);
2484         /* RAM case */
2485         ptr = qemu_get_ram_ptr(addr1);
2486         switch (endian) {
2487         case DEVICE_LITTLE_ENDIAN:
2488             stw_le_p(ptr, val);
2489             break;
2490         case DEVICE_BIG_ENDIAN:
2491             stw_be_p(ptr, val);
2492             break;
2493         default:
2494             stw_p(ptr, val);
2495             break;
2496         }
2497         invalidate_and_set_dirty(addr1, 2);
2498     }
2499 }
2500
2501 void stw_phys(hwaddr addr, uint32_t val)
2502 {
2503     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2504 }
2505
2506 void stw_le_phys(hwaddr addr, uint32_t val)
2507 {
2508     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2509 }
2510
2511 void stw_be_phys(hwaddr addr, uint32_t val)
2512 {
2513     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2514 }
2515
2516 /* XXX: optimize */
2517 void stq_phys(hwaddr addr, uint64_t val)
2518 {
2519     val = tswap64(val);
2520     cpu_physical_memory_write(addr, &val, 8);
2521 }
2522
2523 void stq_le_phys(hwaddr addr, uint64_t val)
2524 {
2525     val = cpu_to_le64(val);
2526     cpu_physical_memory_write(addr, &val, 8);
2527 }
2528
2529 void stq_be_phys(hwaddr addr, uint64_t val)
2530 {
2531     val = cpu_to_be64(val);
2532     cpu_physical_memory_write(addr, &val, 8);
2533 }
2534
2535 /* virtual memory access for debug (includes writing to ROM) */
2536 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2537                         uint8_t *buf, int len, int is_write)
2538 {
2539     int l;
2540     hwaddr phys_addr;
2541     target_ulong page;
2542
2543     while (len > 0) {
2544         page = addr & TARGET_PAGE_MASK;
2545         phys_addr = cpu_get_phys_page_debug(env, page);
2546         /* if no physical page mapped, return an error */
2547         if (phys_addr == -1)
2548             return -1;
2549         l = (page + TARGET_PAGE_SIZE) - addr;
2550         if (l > len)
2551             l = len;
2552         phys_addr += (addr & ~TARGET_PAGE_MASK);
2553         if (is_write)
2554             cpu_physical_memory_write_rom(phys_addr, buf, l);
2555         else
2556             cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2557         len -= l;
2558         buf += l;
2559         addr += l;
2560     }
2561     return 0;
2562 }
2563 #endif
2564
2565 #if !defined(CONFIG_USER_ONLY)
2566
2567 /*
2568  * A helper function for the _utterly broken_ virtio device model to find out if
2569  * it's running on a big endian machine. Don't do this at home kids!
2570  */
2571 bool virtio_is_big_endian(void);
2572 bool virtio_is_big_endian(void)
2573 {
2574 #if defined(TARGET_WORDS_BIGENDIAN)
2575     return true;
2576 #else
2577     return false;
2578 #endif
2579 }
2580
2581 #endif
2582
2583 #ifndef CONFIG_USER_ONLY
2584 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2585 {
2586     MemoryRegionSection *section;
2587
2588     section = phys_page_find(address_space_memory.dispatch,
2589                              phys_addr >> TARGET_PAGE_BITS);
2590
2591     return !(memory_region_is_ram(section->mr) ||
2592              memory_region_is_romd(section->mr));
2593 }
2594 #endif