include/exec/ram_addr.h

   1 /*
   2  * Declarations for cpu physical memory functions
   3  *
   4  * Copyright 2011 Red Hat, Inc. and/or its affiliates
   5  *
   6  * Authors:
   7  *  Avi Kivity <[email protected]>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * later.  See the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 /*
  15  * This header is for use by exec.c and memory.c ONLY.  Do not include it.
  16  * The functions declared here will be removed soon.
  17  */
  18
  19 #ifndef RAM_ADDR_H
  20 #define RAM_ADDR_H
  21
  22 #ifndef CONFIG_USER_ONLY
  23 #include "hw/xen/xen.h"
  24
  25 typedef struct RAMBlock RAMBlock;
  26
  27 struct RAMBlock {
  28     struct rcu_head rcu;
  29     struct MemoryRegion *mr;
  30     uint8_t *host;
  31     ram_addr_t offset;
  32     ram_addr_t used_length;
  33     ram_addr_t max_length;
  34     void (*resized)(const char*, uint64_t length, void *host);
  35     uint32_t flags;
  36     /* Protected by iothread lock.  */
  37     char idstr[256];
  38     /* RCU-enabled, writes protected by the ramlist lock */
  39     QLIST_ENTRY(RAMBlock) next;
  40     int fd;
  41 };
  42
  43 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
  44 {
  45     assert(offset < block->used_length);
  46     assert(block->host);
  47     return (char *)block->host + offset;
  48 }
  49
  50 typedef struct RAMList {
  51     QemuMutex mutex;
  52     /* Protected by the iothread lock.  */
  53     unsigned long *dirty_memory[DIRTY_MEMORY_NUM];
  54     RAMBlock *mru_block;
  55     /* RCU-enabled, writes protected by the ramlist lock. */
  56     QLIST_HEAD(, RAMBlock) blocks;
  57     uint32_t version;
  58 } RAMList;
  59 extern RAMList ram_list;
  60
  61 ram_addr_t last_ram_offset(void);
  62 void qemu_mutex_lock_ramlist(void);
  63 void qemu_mutex_unlock_ramlist(void);
  64
  65 ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  66                                     bool share, const char *mem_path,
  67                                     Error **errp);
  68 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
  69                                    MemoryRegion *mr, Error **errp);
  70 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
  71 ram_addr_t qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
  72                                      void (*resized)(const char*,
  73                                                      uint64_t length,
  74                                                      void *host),
  75                                      MemoryRegion *mr, Error **errp);
  76 int qemu_get_ram_fd(ram_addr_t addr);
  77 void *qemu_get_ram_block_host_ptr(ram_addr_t addr);
  78 void *qemu_get_ram_ptr(ram_addr_t addr);
  79 void qemu_ram_free(ram_addr_t addr);
  80 void qemu_ram_free_from_ptr(ram_addr_t addr);
  81
  82 int qemu_ram_resize(ram_addr_t base, ram_addr_t newsize, Error **errp);
  83
  84 #define DIRTY_CLIENTS_ALL     ((1 << DIRTY_MEMORY_NUM) - 1)
  85 #define DIRTY_CLIENTS_NOCODE  (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
  86
  87 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
  88                                                  ram_addr_t length,
  89                                                  unsigned client)
  90 {
  91     unsigned long end, page, next;
  92
  93     assert(client < DIRTY_MEMORY_NUM);
  94
  95     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
  96     page = start >> TARGET_PAGE_BITS;
  97     next = find_next_bit(ram_list.dirty_memory[client], end, page);
  98
  99     return next < end;
 100 }
 101
 102 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
 103                                                  ram_addr_t length,
 104                                                  unsigned client)
 105 {
 106     unsigned long end, page, next;
 107
 108     assert(client < DIRTY_MEMORY_NUM);
 109
 110     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 111     page = start >> TARGET_PAGE_BITS;
 112     next = find_next_zero_bit(ram_list.dirty_memory[client], end, page);
 113
 114     return next >= end;
 115 }
 116
 117 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
 118                                                       unsigned client)
 119 {
 120     return cpu_physical_memory_get_dirty(addr, 1, client);
 121 }
 122
 123 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
 124 {
 125     bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
 126     bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
 127     bool migration =
 128         cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
 129     return !(vga && code && migration);
 130 }
 131
 132 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
 133                                                                ram_addr_t length,
 134                                                                uint8_t mask)
 135 {
 136     uint8_t ret = 0;
 137
 138     if (mask & (1 << DIRTY_MEMORY_VGA) &&
 139         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
 140         ret |= (1 << DIRTY_MEMORY_VGA);
 141     }
 142     if (mask & (1 << DIRTY_MEMORY_CODE) &&
 143         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
 144         ret |= (1 << DIRTY_MEMORY_CODE);
 145     }
 146     if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
 147         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
 148         ret |= (1 << DIRTY_MEMORY_MIGRATION);
 149     }
 150     return ret;
 151 }
 152
 153 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
 154                                                       unsigned client)
 155 {
 156     assert(client < DIRTY_MEMORY_NUM);
 157     set_bit_atomic(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]);
 158 }
 159
 160 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
 161                                                        ram_addr_t length,
 162                                                        uint8_t mask)
 163 {
 164     unsigned long end, page;
 165     unsigned long **d = ram_list.dirty_memory;
 166
 167     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 168     page = start >> TARGET_PAGE_BITS;
 169     if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
 170         bitmap_set_atomic(d[DIRTY_MEMORY_MIGRATION], page, end - page);
 171     }
 172     if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
 173         bitmap_set_atomic(d[DIRTY_MEMORY_VGA], page, end - page);
 174     }
 175     if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
 176         bitmap_set_atomic(d[DIRTY_MEMORY_CODE], page, end - page);
 177     }
 178     xen_modified_memory(start, length);
 179 }
 180
 181 #if !defined(_WIN32)
 182 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
 183                                                           ram_addr_t start,
 184                                                           ram_addr_t pages)
 185 {
 186     unsigned long i, j;
 187     unsigned long page_number, c;
 188     hwaddr addr;
 189     ram_addr_t ram_addr;
 190     unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
 191     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 192     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 193
 194     /* start address is aligned at the start of a word? */
 195     if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
 196         (hpratio == 1)) {
 197         long k;
 198         long nr = BITS_TO_LONGS(pages);
 199
 200         for (k = 0; k < nr; k++) {
 201             if (bitmap[k]) {
 202                 unsigned long temp = leul_to_cpu(bitmap[k]);
 203                 unsigned long **d = ram_list.dirty_memory;
 204
 205                 atomic_or(&d[DIRTY_MEMORY_MIGRATION][page + k], temp);
 206                 atomic_or(&d[DIRTY_MEMORY_VGA][page + k], temp);
 207                 if (tcg_enabled()) {
 208                     atomic_or(&d[DIRTY_MEMORY_CODE][page + k], temp);
 209                 }
 210             }
 211         }
 212         xen_modified_memory(start, pages << TARGET_PAGE_BITS);
 213     } else {
 214         uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
 215         /*
 216          * bitmap-traveling is faster than memory-traveling (for addr...)
 217          * especially when most of the memory is not dirty.
 218          */
 219         for (i = 0; i < len; i++) {
 220             if (bitmap[i] != 0) {
 221                 c = leul_to_cpu(bitmap[i]);
 222                 do {
 223                     j = ctzl(c);
 224                     c &= ~(1ul << j);
 225                     page_number = (i * HOST_LONG_BITS + j) * hpratio;
 226                     addr = page_number * TARGET_PAGE_SIZE;
 227                     ram_addr = start + addr;
 228                     cpu_physical_memory_set_dirty_range(ram_addr,
 229                                        TARGET_PAGE_SIZE * hpratio, clients);
 230                 } while (c != 0);
 231             }
 232         }
 233     }
 234 }
 235 #endif /* not _WIN32 */
 236
 237 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
 238                                               ram_addr_t length,
 239                                               unsigned client);
 240
 241 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
 242                                                          ram_addr_t length)
 243 {
 244     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
 245     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
 246     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
 247 }
 248
 249
 250 static inline
 251 uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
 252                                                ram_addr_t start,
 253                                                ram_addr_t length)
 254 {
 255     ram_addr_t addr;
 256     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 257     uint64_t num_dirty = 0;
 258
 259     /* start address is aligned at the start of a word? */
 260     if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
 261         int k;
 262         int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
 263         unsigned long *src = ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION];
 264
 265         for (k = page; k < page + nr; k++) {
 266             if (src[k]) {
 267                 unsigned long bits = atomic_xchg(&src[k], 0);
 268                 unsigned long new_dirty;
 269                 new_dirty = ~dest[k];
 270                 dest[k] |= bits;
 271                 new_dirty &= bits;
 272                 num_dirty += ctpopl(new_dirty);
 273             }
 274         }
 275     } else {
 276         for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
 277             if (cpu_physical_memory_test_and_clear_dirty(
 278                         start + addr,
 279                         TARGET_PAGE_SIZE,
 280                         DIRTY_MEMORY_MIGRATION)) {
 281                 long k = (start + addr) >> TARGET_PAGE_BITS;
 282                 if (!test_and_set_bit(k, dest)) {
 283                     num_dirty++;
 284                 }
 285             }
 286         }
 287     }
 288
 289     return num_dirty;
 290 }
 291
 292 #endif
 293 #endif