hw/virtio/vhost.c

   1 /*
   2  * vhost support
   3  *
   4  * Copyright Red Hat, Inc. 2010
   5  *
   6  * Authors:
   7  *  Michael S. Tsirkin <[email protected]>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qapi/error.h"
  18 #include "hw/virtio/vhost.h"
  19 #include "hw/hw.h"
  20 #include "qemu/atomic.h"
  21 #include "qemu/range.h"
  22 #include "qemu/error-report.h"
  23 #include "qemu/memfd.h"
  24 #include <linux/vhost.h>
  25 #include "exec/address-spaces.h"
  26 #include "hw/virtio/virtio-bus.h"
  27 #include "hw/virtio/virtio-access.h"
  28 #include "migration/blocker.h"
  29 #include "sysemu/dma.h"
  30
  31 /* enabled until disconnected backend stabilizes */
  32 #define _VHOST_DEBUG 1
  33
  34 #ifdef _VHOST_DEBUG
  35 #define VHOST_OPS_DEBUG(fmt, ...) \
  36     do { error_report(fmt ": %s (%d)", ## __VA_ARGS__, \
  37                       strerror(errno), errno); } while (0)
  38 #else
  39 #define VHOST_OPS_DEBUG(fmt, ...) \
  40     do { } while (0)
  41 #endif
  42
  43 static struct vhost_log *vhost_log;
  44 static struct vhost_log *vhost_log_shm;
  45
  46 static unsigned int used_memslots;
  47 static QLIST_HEAD(, vhost_dev) vhost_devices =
  48     QLIST_HEAD_INITIALIZER(vhost_devices);
  49
  50 bool vhost_has_free_slot(void)
  51 {
  52     unsigned int slots_limit = ~0U;
  53     struct vhost_dev *hdev;
  54
  55     QLIST_FOREACH(hdev, &vhost_devices, entry) {
  56         unsigned int r = hdev->vhost_ops->vhost_backend_memslots_limit(hdev);
  57         slots_limit = MIN(slots_limit, r);
  58     }
  59     return slots_limit > used_memslots;
  60 }
  61
  62 static void vhost_dev_sync_region(struct vhost_dev *dev,
  63                                   MemoryRegionSection *section,
  64                                   uint64_t mfirst, uint64_t mlast,
  65                                   uint64_t rfirst, uint64_t rlast)
  66 {
  67     vhost_log_chunk_t *log = dev->log->log;
  68
  69     uint64_t start = MAX(mfirst, rfirst);
  70     uint64_t end = MIN(mlast, rlast);
  71     vhost_log_chunk_t *from = log + start / VHOST_LOG_CHUNK;
  72     vhost_log_chunk_t *to = log + end / VHOST_LOG_CHUNK + 1;
  73     uint64_t addr = (start / VHOST_LOG_CHUNK) * VHOST_LOG_CHUNK;
  74
  75     if (end < start) {
  76         return;
  77     }
  78     assert(end / VHOST_LOG_CHUNK < dev->log_size);
  79     assert(start / VHOST_LOG_CHUNK < dev->log_size);
  80
  81     for (;from < to; ++from) {
  82         vhost_log_chunk_t log;
  83         /* We first check with non-atomic: much cheaper,
  84          * and we expect non-dirty to be the common case. */
  85         if (!*from) {
  86             addr += VHOST_LOG_CHUNK;
  87             continue;
  88         }
  89         /* Data must be read atomically. We don't really need barrier semantics
  90          * but it's easier to use atomic_* than roll our own. */
  91         log = atomic_xchg(from, 0);
  92         while (log) {
  93             int bit = ctzl(log);
  94             hwaddr page_addr;
  95             hwaddr section_offset;
  96             hwaddr mr_offset;
  97             page_addr = addr + bit * VHOST_LOG_PAGE;
  98             section_offset = page_addr - section->offset_within_address_space;
  99             mr_offset = section_offset + section->offset_within_region;
 100             memory_region_set_dirty(section->mr, mr_offset, VHOST_LOG_PAGE);
 101             log &= ~(0x1ull << bit);
 102         }
 103         addr += VHOST_LOG_CHUNK;
 104     }
 105 }
 106
 107 static int vhost_sync_dirty_bitmap(struct vhost_dev *dev,
 108                                    MemoryRegionSection *section,
 109                                    hwaddr first,
 110                                    hwaddr last)
 111 {
 112     int i;
 113     hwaddr start_addr;
 114     hwaddr end_addr;
 115
 116     if (!dev->log_enabled || !dev->started) {
 117         return 0;
 118     }
 119     start_addr = section->offset_within_address_space;
 120     end_addr = range_get_last(start_addr, int128_get64(section->size));
 121     start_addr = MAX(first, start_addr);
 122     end_addr = MIN(last, end_addr);
 123
 124     for (i = 0; i < dev->mem->nregions; ++i) {
 125         struct vhost_memory_region *reg = dev->mem->regions + i;
 126         vhost_dev_sync_region(dev, section, start_addr, end_addr,
 127                               reg->guest_phys_addr,
 128                               range_get_last(reg->guest_phys_addr,
 129                                              reg->memory_size));
 130     }
 131     for (i = 0; i < dev->nvqs; ++i) {
 132         struct vhost_virtqueue *vq = dev->vqs + i;
 133         vhost_dev_sync_region(dev, section, start_addr, end_addr, vq->used_phys,
 134                               range_get_last(vq->used_phys, vq->used_size));
 135     }
 136     return 0;
 137 }
 138
 139 static void vhost_log_sync(MemoryListener *listener,
 140                           MemoryRegionSection *section)
 141 {
 142     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 143                                          memory_listener);
 144     vhost_sync_dirty_bitmap(dev, section, 0x0, ~0x0ULL);
 145 }
 146
 147 static void vhost_log_sync_range(struct vhost_dev *dev,
 148                                  hwaddr first, hwaddr last)
 149 {
 150     int i;
 151     /* FIXME: this is N^2 in number of sections */
 152     for (i = 0; i < dev->n_mem_sections; ++i) {
 153         MemoryRegionSection *section = &dev->mem_sections[i];
 154         vhost_sync_dirty_bitmap(dev, section, first, last);
 155     }
 156 }
 157
 158 /* Assign/unassign. Keep an unsorted array of non-overlapping
 159  * memory regions in dev->mem. */
 160 static void vhost_dev_unassign_memory(struct vhost_dev *dev,
 161                                       uint64_t start_addr,
 162                                       uint64_t size)
 163 {
 164     int from, to, n = dev->mem->nregions;
 165     /* Track overlapping/split regions for sanity checking. */
 166     int overlap_start = 0, overlap_end = 0, overlap_middle = 0, split = 0;
 167
 168     for (from = 0, to = 0; from < n; ++from, ++to) {
 169         struct vhost_memory_region *reg = dev->mem->regions + to;
 170         uint64_t reglast;
 171         uint64_t memlast;
 172         uint64_t change;
 173
 174         /* clone old region */
 175         if (to != from) {
 176             memcpy(reg, dev->mem->regions + from, sizeof *reg);
 177         }
 178
 179         /* No overlap is simple */
 180         if (!ranges_overlap(reg->guest_phys_addr, reg->memory_size,
 181                             start_addr, size)) {
 182             continue;
 183         }
 184
 185         /* Split only happens if supplied region
 186          * is in the middle of an existing one. Thus it can not
 187          * overlap with any other existing region. */
 188         assert(!split);
 189
 190         reglast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 191         memlast = range_get_last(start_addr, size);
 192
 193         /* Remove whole region */
 194         if (start_addr <= reg->guest_phys_addr && memlast >= reglast) {
 195             --dev->mem->nregions;
 196             --to;
 197             ++overlap_middle;
 198             continue;
 199         }
 200
 201         /* Shrink region */
 202         if (memlast >= reglast) {
 203             reg->memory_size = start_addr - reg->guest_phys_addr;
 204             assert(reg->memory_size);
 205             assert(!overlap_end);
 206             ++overlap_end;
 207             continue;
 208         }
 209
 210         /* Shift region */
 211         if (start_addr <= reg->guest_phys_addr) {
 212             change = memlast + 1 - reg->guest_phys_addr;
 213             reg->memory_size -= change;
 214             reg->guest_phys_addr += change;
 215             reg->userspace_addr += change;
 216             assert(reg->memory_size);
 217             assert(!overlap_start);
 218             ++overlap_start;
 219             continue;
 220         }
 221
 222         /* This only happens if supplied region
 223          * is in the middle of an existing one. Thus it can not
 224          * overlap with any other existing region. */
 225         assert(!overlap_start);
 226         assert(!overlap_end);
 227         assert(!overlap_middle);
 228         /* Split region: shrink first part, shift second part. */
 229         memcpy(dev->mem->regions + n, reg, sizeof *reg);
 230         reg->memory_size = start_addr - reg->guest_phys_addr;
 231         assert(reg->memory_size);
 232         change = memlast + 1 - reg->guest_phys_addr;
 233         reg = dev->mem->regions + n;
 234         reg->memory_size -= change;
 235         assert(reg->memory_size);
 236         reg->guest_phys_addr += change;
 237         reg->userspace_addr += change;
 238         /* Never add more than 1 region */
 239         assert(dev->mem->nregions == n);
 240         ++dev->mem->nregions;
 241         ++split;
 242     }
 243 }
 244
 245 /* Called after unassign, so no regions overlap the given range. */
 246 static void vhost_dev_assign_memory(struct vhost_dev *dev,
 247                                     uint64_t start_addr,
 248                                     uint64_t size,
 249                                     uint64_t uaddr)
 250 {
 251     int from, to;
 252     struct vhost_memory_region *merged = NULL;
 253     for (from = 0, to = 0; from < dev->mem->nregions; ++from, ++to) {
 254         struct vhost_memory_region *reg = dev->mem->regions + to;
 255         uint64_t prlast, urlast;
 256         uint64_t pmlast, umlast;
 257         uint64_t s, e, u;
 258
 259         /* clone old region */
 260         if (to != from) {
 261             memcpy(reg, dev->mem->regions + from, sizeof *reg);
 262         }
 263         prlast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 264         pmlast = range_get_last(start_addr, size);
 265         urlast = range_get_last(reg->userspace_addr, reg->memory_size);
 266         umlast = range_get_last(uaddr, size);
 267
 268         /* check for overlapping regions: should never happen. */
 269         assert(prlast < start_addr || pmlast < reg->guest_phys_addr);
 270         /* Not an adjacent or overlapping region - do not merge. */
 271         if ((prlast + 1 != start_addr || urlast + 1 != uaddr) &&
 272             (pmlast + 1 != reg->guest_phys_addr ||
 273              umlast + 1 != reg->userspace_addr)) {
 274             continue;
 275         }
 276
 277         if (dev->vhost_ops->vhost_backend_can_merge &&
 278             !dev->vhost_ops->vhost_backend_can_merge(dev, uaddr, size,
 279                                                      reg->userspace_addr,
 280                                                      reg->memory_size)) {
 281             continue;
 282         }
 283
 284         if (merged) {
 285             --to;
 286             assert(to >= 0);
 287         } else {
 288             merged = reg;
 289         }
 290         u = MIN(uaddr, reg->userspace_addr);
 291         s = MIN(start_addr, reg->guest_phys_addr);
 292         e = MAX(pmlast, prlast);
 293         uaddr = merged->userspace_addr = u;
 294         start_addr = merged->guest_phys_addr = s;
 295         size = merged->memory_size = e - s + 1;
 296         assert(merged->memory_size);
 297     }
 298
 299     if (!merged) {
 300         struct vhost_memory_region *reg = dev->mem->regions + to;
 301         memset(reg, 0, sizeof *reg);
 302         reg->memory_size = size;
 303         assert(reg->memory_size);
 304         reg->guest_phys_addr = start_addr;
 305         reg->userspace_addr = uaddr;
 306         ++to;
 307     }
 308     assert(to <= dev->mem->nregions + 1);
 309     dev->mem->nregions = to;
 310 }
 311
 312 static uint64_t vhost_get_log_size(struct vhost_dev *dev)
 313 {
 314     uint64_t log_size = 0;
 315     int i;
 316     for (i = 0; i < dev->mem->nregions; ++i) {
 317         struct vhost_memory_region *reg = dev->mem->regions + i;
 318         uint64_t last = range_get_last(reg->guest_phys_addr,
 319                                        reg->memory_size);
 320         log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
 321     }
 322     for (i = 0; i < dev->nvqs; ++i) {
 323         struct vhost_virtqueue *vq = dev->vqs + i;
 324         uint64_t last = vq->used_phys + vq->used_size - 1;
 325         log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
 326     }
 327     return log_size;
 328 }
 329
 330 static struct vhost_log *vhost_log_alloc(uint64_t size, bool share)
 331 {
 332     struct vhost_log *log;
 333     uint64_t logsize = size * sizeof(*(log->log));
 334     int fd = -1;
 335
 336     log = g_new0(struct vhost_log, 1);
 337     if (share) {
 338         log->log = qemu_memfd_alloc("vhost-log", logsize,
 339                                     F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
 340                                     &fd);
 341         memset(log->log, 0, logsize);
 342     } else {
 343         log->log = g_malloc0(logsize);
 344     }
 345
 346     log->size = size;
 347     log->refcnt = 1;
 348     log->fd = fd;
 349
 350     return log;
 351 }
 352
 353 static struct vhost_log *vhost_log_get(uint64_t size, bool share)
 354 {
 355     struct vhost_log *log = share ? vhost_log_shm : vhost_log;
 356
 357     if (!log || log->size != size) {
 358         log = vhost_log_alloc(size, share);
 359         if (share) {
 360             vhost_log_shm = log;
 361         } else {
 362             vhost_log = log;
 363         }
 364     } else {
 365         ++log->refcnt;
 366     }
 367
 368     return log;
 369 }
 370
 371 static void vhost_log_put(struct vhost_dev *dev, bool sync)
 372 {
 373     struct vhost_log *log = dev->log;
 374
 375     if (!log) {
 376         return;
 377     }
 378     dev->log = NULL;
 379     dev->log_size = 0;
 380
 381     --log->refcnt;
 382     if (log->refcnt == 0) {
 383         /* Sync only the range covered by the old log */
 384         if (dev->log_size && sync) {
 385             vhost_log_sync_range(dev, 0, dev->log_size * VHOST_LOG_CHUNK - 1);
 386         }
 387
 388         if (vhost_log == log) {
 389             g_free(log->log);
 390             vhost_log = NULL;
 391         } else if (vhost_log_shm == log) {
 392             qemu_memfd_free(log->log, log->size * sizeof(*(log->log)),
 393                             log->fd);
 394             vhost_log_shm = NULL;
 395         }
 396
 397         g_free(log);
 398     }
 399 }
 400
 401 static bool vhost_dev_log_is_shared(struct vhost_dev *dev)
 402 {
 403     return dev->vhost_ops->vhost_requires_shm_log &&
 404            dev->vhost_ops->vhost_requires_shm_log(dev);
 405 }
 406
 407 static inline void vhost_dev_log_resize(struct vhost_dev *dev, uint64_t size)
 408 {
 409     struct vhost_log *log = vhost_log_get(size, vhost_dev_log_is_shared(dev));
 410     uint64_t log_base = (uintptr_t)log->log;
 411     int r;
 412
 413     /* inform backend of log switching, this must be done before
 414        releasing the current log, to ensure no logging is lost */
 415     r = dev->vhost_ops->vhost_set_log_base(dev, log_base, log);
 416     if (r < 0) {
 417         VHOST_OPS_DEBUG("vhost_set_log_base failed");
 418     }
 419
 420     vhost_log_put(dev, true);
 421     dev->log = log;
 422     dev->log_size = size;
 423 }
 424
 425 static int vhost_dev_has_iommu(struct vhost_dev *dev)
 426 {
 427     VirtIODevice *vdev = dev->vdev;
 428
 429     return virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM);
 430 }
 431
 432 static void *vhost_memory_map(struct vhost_dev *dev, hwaddr addr,
 433                               hwaddr *plen, int is_write)
 434 {
 435     if (!vhost_dev_has_iommu(dev)) {
 436         return cpu_physical_memory_map(addr, plen, is_write);
 437     } else {
 438         return (void *)(uintptr_t)addr;
 439     }
 440 }
 441
 442 static void vhost_memory_unmap(struct vhost_dev *dev, void *buffer,
 443                                hwaddr len, int is_write,
 444                                hwaddr access_len)
 445 {
 446     if (!vhost_dev_has_iommu(dev)) {
 447         cpu_physical_memory_unmap(buffer, len, is_write, access_len);
 448     }
 449 }
 450
 451 static int vhost_verify_ring_part_mapping(struct vhost_dev *dev,
 452                                           void *part,
 453                                           uint64_t part_addr,
 454                                           uint64_t part_size,
 455                                           uint64_t start_addr,
 456                                           uint64_t size)
 457 {
 458     hwaddr l;
 459     void *p;
 460     int r = 0;
 461
 462     if (!ranges_overlap(start_addr, size, part_addr, part_size)) {
 463         return 0;
 464     }
 465     l = part_size;
 466     p = vhost_memory_map(dev, part_addr, &l, 1);
 467     if (!p || l != part_size) {
 468         r = -ENOMEM;
 469     }
 470     if (p != part) {
 471         r = -EBUSY;
 472     }
 473     vhost_memory_unmap(dev, p, l, 0, 0);
 474     return r;
 475 }
 476
 477 static int vhost_verify_ring_mappings(struct vhost_dev *dev,
 478                                       uint64_t start_addr,
 479                                       uint64_t size)
 480 {
 481     int i, j;
 482     int r = 0;
 483     const char *part_name[] = {
 484         "descriptor table",
 485         "available ring",
 486         "used ring"
 487     };
 488
 489     for (i = 0; i < dev->nvqs; ++i) {
 490         struct vhost_virtqueue *vq = dev->vqs + i;
 491
 492         j = 0;
 493         r = vhost_verify_ring_part_mapping(dev, vq->desc, vq->desc_phys,
 494                                            vq->desc_size, start_addr, size);
 495         if (!r) {
 496             break;
 497         }
 498
 499         j++;
 500         r = vhost_verify_ring_part_mapping(dev, vq->avail, vq->avail_phys,
 501                                            vq->avail_size, start_addr, size);
 502         if (!r) {
 503             break;
 504         }
 505
 506         j++;
 507         r = vhost_verify_ring_part_mapping(dev, vq->used, vq->used_phys,
 508                                            vq->used_size, start_addr, size);
 509         if (!r) {
 510             break;
 511         }
 512     }
 513
 514     if (r == -ENOMEM) {
 515         error_report("Unable to map %s for ring %d", part_name[j], i);
 516     } else if (r == -EBUSY) {
 517         error_report("%s relocated for ring %d", part_name[j], i);
 518     }
 519     return r;
 520 }
 521
 522 static struct vhost_memory_region *vhost_dev_find_reg(struct vhost_dev *dev,
 523                                                       uint64_t start_addr,
 524                                                       uint64_t size)
 525 {
 526     int i, n = dev->mem->nregions;
 527     for (i = 0; i < n; ++i) {
 528         struct vhost_memory_region *reg = dev->mem->regions + i;
 529         if (ranges_overlap(reg->guest_phys_addr, reg->memory_size,
 530                            start_addr, size)) {
 531             return reg;
 532         }
 533     }
 534     return NULL;
 535 }
 536
 537 static bool vhost_dev_cmp_memory(struct vhost_dev *dev,
 538                                  uint64_t start_addr,
 539                                  uint64_t size,
 540                                  uint64_t uaddr)
 541 {
 542     struct vhost_memory_region *reg = vhost_dev_find_reg(dev, start_addr, size);
 543     uint64_t reglast;
 544     uint64_t memlast;
 545
 546     if (!reg) {
 547         return true;
 548     }
 549
 550     reglast = range_get_last(reg->guest_phys_addr, reg->memory_size);
 551     memlast = range_get_last(start_addr, size);
 552
 553     /* Need to extend region? */
 554     if (start_addr < reg->guest_phys_addr || memlast > reglast) {
 555         return true;
 556     }
 557     /* userspace_addr changed? */
 558     return uaddr != reg->userspace_addr + start_addr - reg->guest_phys_addr;
 559 }
 560
 561 static void vhost_set_memory(MemoryListener *listener,
 562                              MemoryRegionSection *section,
 563                              bool add)
 564 {
 565     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 566                                          memory_listener);
 567     hwaddr start_addr = section->offset_within_address_space;
 568     ram_addr_t size = int128_get64(section->size);
 569     bool log_dirty =
 570         memory_region_get_dirty_log_mask(section->mr) & ~(1 << DIRTY_MEMORY_MIGRATION);
 571     int s = offsetof(struct vhost_memory, regions) +
 572         (dev->mem->nregions + 1) * sizeof dev->mem->regions[0];
 573     void *ram;
 574
 575     dev->mem = g_realloc(dev->mem, s);
 576
 577     if (log_dirty) {
 578         add = false;
 579     }
 580
 581     assert(size);
 582
 583     /* Optimize no-change case. At least cirrus_vga does this a lot at this time. */
 584     ram = memory_region_get_ram_ptr(section->mr) + section->offset_within_region;
 585     if (add) {
 586         if (!vhost_dev_cmp_memory(dev, start_addr, size, (uintptr_t)ram)) {
 587             /* Region exists with same address. Nothing to do. */
 588             return;
 589         }
 590     } else {
 591         if (!vhost_dev_find_reg(dev, start_addr, size)) {
 592             /* Removing region that we don't access. Nothing to do. */
 593             return;
 594         }
 595     }
 596
 597     vhost_dev_unassign_memory(dev, start_addr, size);
 598     if (add) {
 599         /* Add given mapping, merging adjacent regions if any */
 600         vhost_dev_assign_memory(dev, start_addr, size, (uintptr_t)ram);
 601     } else {
 602         /* Remove old mapping for this memory, if any. */
 603         vhost_dev_unassign_memory(dev, start_addr, size);
 604     }
 605     dev->mem_changed_start_addr = MIN(dev->mem_changed_start_addr, start_addr);
 606     dev->mem_changed_end_addr = MAX(dev->mem_changed_end_addr, start_addr + size - 1);
 607     dev->memory_changed = true;
 608     used_memslots = dev->mem->nregions;
 609 }
 610
 611 static bool vhost_section(MemoryRegionSection *section)
 612 {
 613     return memory_region_is_ram(section->mr) &&
 614         !memory_region_is_rom(section->mr);
 615 }
 616
 617 static void vhost_begin(MemoryListener *listener)
 618 {
 619     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 620                                          memory_listener);
 621     dev->mem_changed_end_addr = 0;
 622     dev->mem_changed_start_addr = -1;
 623 }
 624
 625 static void vhost_commit(MemoryListener *listener)
 626 {
 627     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 628                                          memory_listener);
 629     hwaddr start_addr = 0;
 630     ram_addr_t size = 0;
 631     uint64_t log_size;
 632     int r;
 633
 634     if (!dev->memory_changed) {
 635         return;
 636     }
 637     if (!dev->started) {
 638         return;
 639     }
 640     if (dev->mem_changed_start_addr > dev->mem_changed_end_addr) {
 641         return;
 642     }
 643
 644     if (dev->started) {
 645         start_addr = dev->mem_changed_start_addr;
 646         size = dev->mem_changed_end_addr - dev->mem_changed_start_addr + 1;
 647
 648         r = vhost_verify_ring_mappings(dev, start_addr, size);
 649         assert(r >= 0);
 650     }
 651
 652     if (!dev->log_enabled) {
 653         r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
 654         if (r < 0) {
 655             VHOST_OPS_DEBUG("vhost_set_mem_table failed");
 656         }
 657         dev->memory_changed = false;
 658         return;
 659     }
 660     log_size = vhost_get_log_size(dev);
 661     /* We allocate an extra 4K bytes to log,
 662      * to reduce the * number of reallocations. */
 663 #define VHOST_LOG_BUFFER (0x1000 / sizeof *dev->log)
 664     /* To log more, must increase log size before table update. */
 665     if (dev->log_size < log_size) {
 666         vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
 667     }
 668     r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
 669     if (r < 0) {
 670         VHOST_OPS_DEBUG("vhost_set_mem_table failed");
 671     }
 672     /* To log less, can only decrease log size after table update. */
 673     if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
 674         vhost_dev_log_resize(dev, log_size);
 675     }
 676     dev->memory_changed = false;
 677 }
 678
 679 static void vhost_region_add(MemoryListener *listener,
 680                              MemoryRegionSection *section)
 681 {
 682     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 683                                          memory_listener);
 684
 685     if (!vhost_section(section)) {
 686         return;
 687     }
 688
 689     ++dev->n_mem_sections;
 690     dev->mem_sections = g_renew(MemoryRegionSection, dev->mem_sections,
 691                                 dev->n_mem_sections);
 692     dev->mem_sections[dev->n_mem_sections - 1] = *section;
 693     memory_region_ref(section->mr);
 694     vhost_set_memory(listener, section, true);
 695 }
 696
 697 static void vhost_region_del(MemoryListener *listener,
 698                              MemoryRegionSection *section)
 699 {
 700     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 701                                          memory_listener);
 702     int i;
 703
 704     if (!vhost_section(section)) {
 705         return;
 706     }
 707
 708     vhost_set_memory(listener, section, false);
 709     memory_region_unref(section->mr);
 710     for (i = 0; i < dev->n_mem_sections; ++i) {
 711         if (dev->mem_sections[i].offset_within_address_space
 712             == section->offset_within_address_space) {
 713             --dev->n_mem_sections;
 714             memmove(&dev->mem_sections[i], &dev->mem_sections[i+1],
 715                     (dev->n_mem_sections - i) * sizeof(*dev->mem_sections));
 716             break;
 717         }
 718     }
 719 }
 720
 721 static void vhost_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
 722 {
 723     struct vhost_iommu *iommu = container_of(n, struct vhost_iommu, n);
 724     struct vhost_dev *hdev = iommu->hdev;
 725     hwaddr iova = iotlb->iova + iommu->iommu_offset;
 726
 727     if (hdev->vhost_ops->vhost_invalidate_device_iotlb(hdev, iova,
 728                                                        iotlb->addr_mask + 1)) {
 729         error_report("Fail to invalidate device iotlb");
 730     }
 731 }
 732
 733 static void vhost_iommu_region_add(MemoryListener *listener,
 734                                    MemoryRegionSection *section)
 735 {
 736     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 737                                          iommu_listener);
 738     struct vhost_iommu *iommu;
 739     Int128 end;
 740
 741     if (!memory_region_is_iommu(section->mr)) {
 742         return;
 743     }
 744
 745     iommu = g_malloc0(sizeof(*iommu));
 746     end = int128_add(int128_make64(section->offset_within_region),
 747                      section->size);
 748     end = int128_sub(end, int128_one());
 749     iommu_notifier_init(&iommu->n, vhost_iommu_unmap_notify,
 750                         IOMMU_NOTIFIER_UNMAP,
 751                         section->offset_within_region,
 752                         int128_get64(end));
 753     iommu->mr = section->mr;
 754     iommu->iommu_offset = section->offset_within_address_space -
 755                           section->offset_within_region;
 756     iommu->hdev = dev;
 757     memory_region_register_iommu_notifier(section->mr, &iommu->n);
 758     QLIST_INSERT_HEAD(&dev->iommu_list, iommu, iommu_next);
 759     /* TODO: can replay help performance here? */
 760 }
 761
 762 static void vhost_iommu_region_del(MemoryListener *listener,
 763                                    MemoryRegionSection *section)
 764 {
 765     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 766                                          iommu_listener);
 767     struct vhost_iommu *iommu;
 768
 769     if (!memory_region_is_iommu(section->mr)) {
 770         return;
 771     }
 772
 773     QLIST_FOREACH(iommu, &dev->iommu_list, iommu_next) {
 774         if (iommu->mr == section->mr &&
 775             iommu->n.start == section->offset_within_region) {
 776             memory_region_unregister_iommu_notifier(iommu->mr,
 777                                                     &iommu->n);
 778             QLIST_REMOVE(iommu, iommu_next);
 779             g_free(iommu);
 780             break;
 781         }
 782     }
 783 }
 784
 785 static void vhost_region_nop(MemoryListener *listener,
 786                              MemoryRegionSection *section)
 787 {
 788 }
 789
 790 static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
 791                                     struct vhost_virtqueue *vq,
 792                                     unsigned idx, bool enable_log)
 793 {
 794     struct vhost_vring_addr addr = {
 795         .index = idx,
 796         .desc_user_addr = (uint64_t)(unsigned long)vq->desc,
 797         .avail_user_addr = (uint64_t)(unsigned long)vq->avail,
 798         .used_user_addr = (uint64_t)(unsigned long)vq->used,
 799         .log_guest_addr = vq->used_phys,
 800         .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
 801     };
 802     int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
 803     if (r < 0) {
 804         VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
 805         return -errno;
 806     }
 807     return 0;
 808 }
 809
 810 static int vhost_dev_set_features(struct vhost_dev *dev,
 811                                   bool enable_log)
 812 {
 813     uint64_t features = dev->acked_features;
 814     int r;
 815     if (enable_log) {
 816         features |= 0x1ULL << VHOST_F_LOG_ALL;
 817     }
 818     r = dev->vhost_ops->vhost_set_features(dev, features);
 819     if (r < 0) {
 820         VHOST_OPS_DEBUG("vhost_set_features failed");
 821     }
 822     return r < 0 ? -errno : 0;
 823 }
 824
 825 static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
 826 {
 827     int r, i, idx;
 828     r = vhost_dev_set_features(dev, enable_log);
 829     if (r < 0) {
 830         goto err_features;
 831     }
 832     for (i = 0; i < dev->nvqs; ++i) {
 833         idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
 834         r = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
 835                                      enable_log);
 836         if (r < 0) {
 837             goto err_vq;
 838         }
 839     }
 840     return 0;
 841 err_vq:
 842     for (; i >= 0; --i) {
 843         idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
 844         vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
 845                                  dev->log_enabled);
 846     }
 847     vhost_dev_set_features(dev, dev->log_enabled);
 848 err_features:
 849     return r;
 850 }
 851
 852 static int vhost_migration_log(MemoryListener *listener, int enable)
 853 {
 854     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 855                                          memory_listener);
 856     int r;
 857     if (!!enable == dev->log_enabled) {
 858         return 0;
 859     }
 860     if (!dev->started) {
 861         dev->log_enabled = enable;
 862         return 0;
 863     }
 864     if (!enable) {
 865         r = vhost_dev_set_log(dev, false);
 866         if (r < 0) {
 867             return r;
 868         }
 869         vhost_log_put(dev, false);
 870     } else {
 871         vhost_dev_log_resize(dev, vhost_get_log_size(dev));
 872         r = vhost_dev_set_log(dev, true);
 873         if (r < 0) {
 874             return r;
 875         }
 876     }
 877     dev->log_enabled = enable;
 878     return 0;
 879 }
 880
 881 static void vhost_log_global_start(MemoryListener *listener)
 882 {
 883     int r;
 884
 885     r = vhost_migration_log(listener, true);
 886     if (r < 0) {
 887         abort();
 888     }
 889 }
 890
 891 static void vhost_log_global_stop(MemoryListener *listener)
 892 {
 893     int r;
 894
 895     r = vhost_migration_log(listener, false);
 896     if (r < 0) {
 897         abort();
 898     }
 899 }
 900
 901 static void vhost_log_start(MemoryListener *listener,
 902                             MemoryRegionSection *section,
 903                             int old, int new)
 904 {
 905     /* FIXME: implement */
 906 }
 907
 908 static void vhost_log_stop(MemoryListener *listener,
 909                            MemoryRegionSection *section,
 910                            int old, int new)
 911 {
 912     /* FIXME: implement */
 913 }
 914
 915 /* The vhost driver natively knows how to handle the vrings of non
 916  * cross-endian legacy devices and modern devices. Only legacy devices
 917  * exposed to a bi-endian guest may require the vhost driver to use a
 918  * specific endianness.
 919  */
 920 static inline bool vhost_needs_vring_endian(VirtIODevice *vdev)
 921 {
 922     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
 923         return false;
 924     }
 925 #ifdef HOST_WORDS_BIGENDIAN
 926     return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_LITTLE;
 927 #else
 928     return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_BIG;
 929 #endif
 930 }
 931
 932 static int vhost_virtqueue_set_vring_endian_legacy(struct vhost_dev *dev,
 933                                                    bool is_big_endian,
 934                                                    int vhost_vq_index)
 935 {
 936     struct vhost_vring_state s = {
 937         .index = vhost_vq_index,
 938         .num = is_big_endian
 939     };
 940
 941     if (!dev->vhost_ops->vhost_set_vring_endian(dev, &s)) {
 942         return 0;
 943     }
 944
 945     VHOST_OPS_DEBUG("vhost_set_vring_endian failed");
 946     if (errno == ENOTTY) {
 947         error_report("vhost does not support cross-endian");
 948         return -ENOSYS;
 949     }
 950
 951     return -errno;
 952 }
 953
 954 static int vhost_memory_region_lookup(struct vhost_dev *hdev,
 955                                       uint64_t gpa, uint64_t *uaddr,
 956                                       uint64_t *len)
 957 {
 958     int i;
 959
 960     for (i = 0; i < hdev->mem->nregions; i++) {
 961         struct vhost_memory_region *reg = hdev->mem->regions + i;
 962
 963         if (gpa >= reg->guest_phys_addr &&
 964             reg->guest_phys_addr + reg->memory_size > gpa) {
 965             *uaddr = reg->userspace_addr + gpa - reg->guest_phys_addr;
 966             *len = reg->guest_phys_addr + reg->memory_size - gpa;
 967             return 0;
 968         }
 969     }
 970
 971     return -EFAULT;
 972 }
 973
 974 int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int write)
 975 {
 976     IOMMUTLBEntry iotlb;
 977     uint64_t uaddr, len;
 978     int ret = -EFAULT;
 979
 980     rcu_read_lock();
 981
 982     iotlb = address_space_get_iotlb_entry(dev->vdev->dma_as,
 983                                           iova, write);
 984     if (iotlb.target_as != NULL) {
 985         ret = vhost_memory_region_lookup(dev, iotlb.translated_addr,
 986                                          &uaddr, &len);
 987         if (ret) {
 988             error_report("Fail to lookup the translated address "
 989                          "%"PRIx64, iotlb.translated_addr);
 990             goto out;
 991         }
 992
 993         len = MIN(iotlb.addr_mask + 1, len);
 994         iova = iova & ~iotlb.addr_mask;
 995
 996         ret = dev->vhost_ops->vhost_update_device_iotlb(dev, iova, uaddr,
 997                                                       len, iotlb.perm);
 998         if (ret) {
 999             error_report("Fail to update device iotlb");
1000             goto out;
1001         }
1002     }
1003 out:
1004     rcu_read_unlock();
1005
1006     return ret;
1007 }
1008
1009 static int vhost_virtqueue_start(struct vhost_dev *dev,
1010                                 struct VirtIODevice *vdev,
1011                                 struct vhost_virtqueue *vq,
1012                                 unsigned idx)
1013 {
1014     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
1015     VirtioBusState *vbus = VIRTIO_BUS(qbus);
1016     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
1017     hwaddr s, l, a;
1018     int r;
1019     int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
1020     struct vhost_vring_file file = {
1021         .index = vhost_vq_index
1022     };
1023     struct vhost_vring_state state = {
1024         .index = vhost_vq_index
1025     };
1026     struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
1027
1028
1029     vq->num = state.num = virtio_queue_get_num(vdev, idx);
1030     r = dev->vhost_ops->vhost_set_vring_num(dev, &state);
1031     if (r) {
1032         VHOST_OPS_DEBUG("vhost_set_vring_num failed");
1033         return -errno;
1034     }
1035
1036     state.num = virtio_queue_get_last_avail_idx(vdev, idx);
1037     r = dev->vhost_ops->vhost_set_vring_base(dev, &state);
1038     if (r) {
1039         VHOST_OPS_DEBUG("vhost_set_vring_base failed");
1040         return -errno;
1041     }
1042
1043     if (vhost_needs_vring_endian(vdev)) {
1044         r = vhost_virtqueue_set_vring_endian_legacy(dev,
1045                                                     virtio_is_big_endian(vdev),
1046                                                     vhost_vq_index);
1047         if (r) {
1048             return -errno;
1049         }
1050     }
1051
1052     vq->desc_size = s = l = virtio_queue_get_desc_size(vdev, idx);
1053     vq->desc_phys = a = virtio_queue_get_desc_addr(vdev, idx);
1054     vq->desc = vhost_memory_map(dev, a, &l, 0);
1055     if (!vq->desc || l != s) {
1056         r = -ENOMEM;
1057         goto fail_alloc_desc;
1058     }
1059     vq->avail_size = s = l = virtio_queue_get_avail_size(vdev, idx);
1060     vq->avail_phys = a = virtio_queue_get_avail_addr(vdev, idx);
1061     vq->avail = vhost_memory_map(dev, a, &l, 0);
1062     if (!vq->avail || l != s) {
1063         r = -ENOMEM;
1064         goto fail_alloc_avail;
1065     }
1066     vq->used_size = s = l = virtio_queue_get_used_size(vdev, idx);
1067     vq->used_phys = a = virtio_queue_get_used_addr(vdev, idx);
1068     vq->used = vhost_memory_map(dev, a, &l, 1);
1069     if (!vq->used || l != s) {
1070         r = -ENOMEM;
1071         goto fail_alloc_used;
1072     }
1073
1074     r = vhost_virtqueue_set_addr(dev, vq, vhost_vq_index, dev->log_enabled);
1075     if (r < 0) {
1076         r = -errno;
1077         goto fail_alloc;
1078     }
1079
1080     file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
1081     r = dev->vhost_ops->vhost_set_vring_kick(dev, &file);
1082     if (r) {
1083         VHOST_OPS_DEBUG("vhost_set_vring_kick failed");
1084         r = -errno;
1085         goto fail_kick;
1086     }
1087
1088     /* Clear and discard previous events if any. */
1089     event_notifier_test_and_clear(&vq->masked_notifier);
1090
1091     /* Init vring in unmasked state, unless guest_notifier_mask
1092      * will do it later.
1093      */
1094     if (!vdev->use_guest_notifier_mask) {
1095         /* TODO: check and handle errors. */
1096         vhost_virtqueue_mask(dev, vdev, idx, false);
1097     }
1098
1099     if (k->query_guest_notifiers &&
1100         k->query_guest_notifiers(qbus->parent) &&
1101         virtio_queue_vector(vdev, idx) == VIRTIO_NO_VECTOR) {
1102         file.fd = -1;
1103         r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
1104         if (r) {
1105             goto fail_vector;
1106         }
1107     }
1108
1109     return 0;
1110
1111 fail_vector:
1112 fail_kick:
1113 fail_alloc:
1114     vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx),
1115                        0, 0);
1116 fail_alloc_used:
1117     vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx),
1118                        0, 0);
1119 fail_alloc_avail:
1120     vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx),
1121                        0, 0);
1122 fail_alloc_desc:
1123     return r;
1124 }
1125
1126 static void vhost_virtqueue_stop(struct vhost_dev *dev,
1127                                     struct VirtIODevice *vdev,
1128                                     struct vhost_virtqueue *vq,
1129                                     unsigned idx)
1130 {
1131     int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
1132     struct vhost_vring_state state = {
1133         .index = vhost_vq_index,
1134     };
1135     int r;
1136
1137     r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
1138     if (r < 0) {
1139         VHOST_OPS_DEBUG("vhost VQ %d ring restore failed: %d", idx, r);
1140     } else {
1141         virtio_queue_set_last_avail_idx(vdev, idx, state.num);
1142     }
1143     virtio_queue_invalidate_signalled_used(vdev, idx);
1144     virtio_queue_update_used_idx(vdev, idx);
1145
1146     /* In the cross-endian case, we need to reset the vring endianness to
1147      * native as legacy devices expect so by default.
1148      */
1149     if (vhost_needs_vring_endian(vdev)) {
1150         vhost_virtqueue_set_vring_endian_legacy(dev,
1151                                                 !virtio_is_big_endian(vdev),
1152                                                 vhost_vq_index);
1153     }
1154
1155     vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx),
1156                        1, virtio_queue_get_used_size(vdev, idx));
1157     vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx),
1158                        0, virtio_queue_get_avail_size(vdev, idx));
1159     vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx),
1160                        0, virtio_queue_get_desc_size(vdev, idx));
1161 }
1162
1163 static void vhost_eventfd_add(MemoryListener *listener,
1164                               MemoryRegionSection *section,
1165                               bool match_data, uint64_t data, EventNotifier *e)
1166 {
1167 }
1168
1169 static void vhost_eventfd_del(MemoryListener *listener,
1170                               MemoryRegionSection *section,
1171                               bool match_data, uint64_t data, EventNotifier *e)
1172 {
1173 }
1174
1175 static int vhost_virtqueue_set_busyloop_timeout(struct vhost_dev *dev,
1176                                                 int n, uint32_t timeout)
1177 {
1178     int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
1179     struct vhost_vring_state state = {
1180         .index = vhost_vq_index,
1181         .num = timeout,
1182     };
1183     int r;
1184
1185     if (!dev->vhost_ops->vhost_set_vring_busyloop_timeout) {
1186         return -EINVAL;
1187     }
1188
1189     r = dev->vhost_ops->vhost_set_vring_busyloop_timeout(dev, &state);
1190     if (r) {
1191         VHOST_OPS_DEBUG("vhost_set_vring_busyloop_timeout failed");
1192         return r;
1193     }
1194
1195     return 0;
1196 }
1197
1198 static int vhost_virtqueue_init(struct vhost_dev *dev,
1199                                 struct vhost_virtqueue *vq, int n)
1200 {
1201     int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
1202     struct vhost_vring_file file = {
1203         .index = vhost_vq_index,
1204     };
1205     int r = event_notifier_init(&vq->masked_notifier, 0);
1206     if (r < 0) {
1207         return r;
1208     }
1209
1210     file.fd = event_notifier_get_fd(&vq->masked_notifier);
1211     r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
1212     if (r) {
1213         VHOST_OPS_DEBUG("vhost_set_vring_call failed");
1214         r = -errno;
1215         goto fail_call;
1216     }
1217
1218     vq->dev = dev;
1219
1220     return 0;
1221 fail_call:
1222     event_notifier_cleanup(&vq->masked_notifier);
1223     return r;
1224 }
1225
1226 static void vhost_virtqueue_cleanup(struct vhost_virtqueue *vq)
1227 {
1228     event_notifier_cleanup(&vq->masked_notifier);
1229 }
1230
1231 int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
1232                    VhostBackendType backend_type, uint32_t busyloop_timeout)
1233 {
1234     uint64_t features;
1235     int i, r, n_initialized_vqs = 0;
1236     Error *local_err = NULL;
1237
1238     hdev->vdev = NULL;
1239     hdev->migration_blocker = NULL;
1240
1241     r = vhost_set_backend_type(hdev, backend_type);
1242     assert(r >= 0);
1243
1244     r = hdev->vhost_ops->vhost_backend_init(hdev, opaque);
1245     if (r < 0) {
1246         goto fail;
1247     }
1248
1249     if (used_memslots > hdev->vhost_ops->vhost_backend_memslots_limit(hdev)) {
1250         error_report("vhost backend memory slots limit is less"
1251                 " than current number of present memory slots");
1252         r = -1;
1253         goto fail;
1254     }
1255
1256     r = hdev->vhost_ops->vhost_set_owner(hdev);
1257     if (r < 0) {
1258         VHOST_OPS_DEBUG("vhost_set_owner failed");
1259         goto fail;
1260     }
1261
1262     r = hdev->vhost_ops->vhost_get_features(hdev, &features);
1263     if (r < 0) {
1264         VHOST_OPS_DEBUG("vhost_get_features failed");
1265         goto fail;
1266     }
1267
1268     for (i = 0; i < hdev->nvqs; ++i, ++n_initialized_vqs) {
1269         r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i);
1270         if (r < 0) {
1271             goto fail;
1272         }
1273     }
1274
1275     if (busyloop_timeout) {
1276         for (i = 0; i < hdev->nvqs; ++i) {
1277             r = vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i,
1278                                                      busyloop_timeout);
1279             if (r < 0) {
1280                 goto fail_busyloop;
1281             }
1282         }
1283     }
1284
1285     hdev->features = features;
1286
1287     hdev->memory_listener = (MemoryListener) {
1288         .begin = vhost_begin,
1289         .commit = vhost_commit,
1290         .region_add = vhost_region_add,
1291         .region_del = vhost_region_del,
1292         .region_nop = vhost_region_nop,
1293         .log_start = vhost_log_start,
1294         .log_stop = vhost_log_stop,
1295         .log_sync = vhost_log_sync,
1296         .log_global_start = vhost_log_global_start,
1297         .log_global_stop = vhost_log_global_stop,
1298         .eventfd_add = vhost_eventfd_add,
1299         .eventfd_del = vhost_eventfd_del,
1300         .priority = 10
1301     };
1302
1303     hdev->iommu_listener = (MemoryListener) {
1304         .region_add = vhost_iommu_region_add,
1305         .region_del = vhost_iommu_region_del,
1306     };
1307
1308     if (hdev->migration_blocker == NULL) {
1309         if (!(hdev->features & (0x1ULL << VHOST_F_LOG_ALL))) {
1310             error_setg(&hdev->migration_blocker,
1311                        "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature.");
1312         } else if (vhost_dev_log_is_shared(hdev) && !qemu_memfd_check()) {
1313             error_setg(&hdev->migration_blocker,
1314                        "Migration disabled: failed to allocate shared memory");
1315         }
1316     }
1317
1318     if (hdev->migration_blocker != NULL) {
1319         r = migrate_add_blocker(hdev->migration_blocker, &local_err);
1320         if (local_err) {
1321             error_report_err(local_err);
1322             error_free(hdev->migration_blocker);
1323             goto fail_busyloop;
1324         }
1325     }
1326
1327     hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
1328     hdev->n_mem_sections = 0;
1329     hdev->mem_sections = NULL;
1330     hdev->log = NULL;
1331     hdev->log_size = 0;
1332     hdev->log_enabled = false;
1333     hdev->started = false;
1334     hdev->memory_changed = false;
1335     memory_listener_register(&hdev->memory_listener, &address_space_memory);
1336     QLIST_INSERT_HEAD(&vhost_devices, hdev, entry);
1337     return 0;
1338
1339 fail_busyloop:
1340     while (--i >= 0) {
1341         vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i, 0);
1342     }
1343 fail:
1344     hdev->nvqs = n_initialized_vqs;
1345     vhost_dev_cleanup(hdev);
1346     return r;
1347 }
1348
1349 void vhost_dev_cleanup(struct vhost_dev *hdev)
1350 {
1351     int i;
1352
1353     for (i = 0; i < hdev->nvqs; ++i) {
1354         vhost_virtqueue_cleanup(hdev->vqs + i);
1355     }
1356     if (hdev->mem) {
1357         /* those are only safe after successful init */
1358         memory_listener_unregister(&hdev->memory_listener);
1359         QLIST_REMOVE(hdev, entry);
1360     }
1361     if (hdev->migration_blocker) {
1362         migrate_del_blocker(hdev->migration_blocker);
1363         error_free(hdev->migration_blocker);
1364     }
1365     g_free(hdev->mem);
1366     g_free(hdev->mem_sections);
1367     if (hdev->vhost_ops) {
1368         hdev->vhost_ops->vhost_backend_cleanup(hdev);
1369     }
1370     assert(!hdev->log);
1371
1372     memset(hdev, 0, sizeof(struct vhost_dev));
1373 }
1374
1375 /* Stop processing guest IO notifications in qemu.
1376  * Start processing them in vhost in kernel.
1377  */
1378 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
1379 {
1380     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
1381     int i, r, e;
1382
1383     /* We will pass the notifiers to the kernel, make sure that QEMU
1384      * doesn't interfere.
1385      */
1386     r = virtio_device_grab_ioeventfd(vdev);
1387     if (r < 0) {
1388         error_report("binding does not support host notifiers");
1389         goto fail;
1390     }
1391
1392     for (i = 0; i < hdev->nvqs; ++i) {
1393         r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
1394                                          true);
1395         if (r < 0) {
1396             error_report("vhost VQ %d notifier binding failed: %d", i, -r);
1397             goto fail_vq;
1398         }
1399     }
1400
1401     return 0;
1402 fail_vq:
1403     while (--i >= 0) {
1404         e = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
1405                                          false);
1406         if (e < 0) {
1407             error_report("vhost VQ %d notifier cleanup error: %d", i, -r);
1408         }
1409         assert (e >= 0);
1410     }
1411     virtio_device_release_ioeventfd(vdev);
1412 fail:
1413     return r;
1414 }
1415
1416 /* Stop processing guest IO notifications in vhost.
1417  * Start processing them in qemu.
1418  * This might actually run the qemu handlers right away,
1419  * so virtio in qemu must be completely setup when this is called.
1420  */
1421 void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
1422 {
1423     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
1424     int i, r;
1425
1426     for (i = 0; i < hdev->nvqs; ++i) {
1427         r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
1428                                          false);
1429         if (r < 0) {
1430             error_report("vhost VQ %d notifier cleanup failed: %d", i, -r);
1431         }
1432         assert (r >= 0);
1433     }
1434     virtio_device_release_ioeventfd(vdev);
1435 }
1436
1437 /* Test and clear event pending status.
1438  * Should be called after unmask to avoid losing events.
1439  */
1440 bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n)
1441 {
1442     struct vhost_virtqueue *vq = hdev->vqs + n - hdev->vq_index;
1443     assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
1444     return event_notifier_test_and_clear(&vq->masked_notifier);
1445 }
1446
1447 /* Mask/unmask events from this vq. */
1448 void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
1449                          bool mask)
1450 {
1451     struct VirtQueue *vvq = virtio_get_queue(vdev, n);
1452     int r, index = n - hdev->vq_index;
1453     struct vhost_vring_file file;
1454
1455     /* should only be called after backend is connected */
1456     assert(hdev->vhost_ops);
1457
1458     if (mask) {
1459         assert(vdev->use_guest_notifier_mask);
1460         file.fd = event_notifier_get_fd(&hdev->vqs[index].masked_notifier);
1461     } else {
1462         file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
1463     }
1464
1465     file.index = hdev->vhost_ops->vhost_get_vq_index(hdev, n);
1466     r = hdev->vhost_ops->vhost_set_vring_call(hdev, &file);
1467     if (r < 0) {
1468         VHOST_OPS_DEBUG("vhost_set_vring_call failed");
1469     }
1470 }
1471
1472 uint64_t vhost_get_features(struct vhost_dev *hdev, const int *feature_bits,
1473                             uint64_t features)
1474 {
1475     const int *bit = feature_bits;
1476     while (*bit != VHOST_INVALID_FEATURE_BIT) {
1477         uint64_t bit_mask = (1ULL << *bit);
1478         if (!(hdev->features & bit_mask)) {
1479             features &= ~bit_mask;
1480         }
1481         bit++;
1482     }
1483     return features;
1484 }
1485
1486 void vhost_ack_features(struct vhost_dev *hdev, const int *feature_bits,
1487                         uint64_t features)
1488 {
1489     const int *bit = feature_bits;
1490     while (*bit != VHOST_INVALID_FEATURE_BIT) {
1491         uint64_t bit_mask = (1ULL << *bit);
1492         if (features & bit_mask) {
1493             hdev->acked_features |= bit_mask;
1494         }
1495         bit++;
1496     }
1497 }
1498
1499 /* Host notifiers must be enabled at this point. */
1500 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
1501 {
1502     int i, r;
1503
1504     /* should only be called after backend is connected */
1505     assert(hdev->vhost_ops);
1506
1507     hdev->started = true;
1508     hdev->vdev = vdev;
1509
1510     r = vhost_dev_set_features(hdev, hdev->log_enabled);
1511     if (r < 0) {
1512         goto fail_features;
1513     }
1514
1515     if (vhost_dev_has_iommu(hdev)) {
1516         memory_listener_register(&hdev->iommu_listener, vdev->dma_as);
1517     }
1518
1519     r = hdev->vhost_ops->vhost_set_mem_table(hdev, hdev->mem);
1520     if (r < 0) {
1521         VHOST_OPS_DEBUG("vhost_set_mem_table failed");
1522         r = -errno;
1523         goto fail_mem;
1524     }
1525     for (i = 0; i < hdev->nvqs; ++i) {
1526         r = vhost_virtqueue_start(hdev,
1527                                   vdev,
1528                                   hdev->vqs + i,
1529                                   hdev->vq_index + i);
1530         if (r < 0) {
1531             goto fail_vq;
1532         }
1533     }
1534
1535     if (hdev->log_enabled) {
1536         uint64_t log_base;
1537
1538         hdev->log_size = vhost_get_log_size(hdev);
1539         hdev->log = vhost_log_get(hdev->log_size,
1540                                   vhost_dev_log_is_shared(hdev));
1541         log_base = (uintptr_t)hdev->log->log;
1542         r = hdev->vhost_ops->vhost_set_log_base(hdev,
1543                                                 hdev->log_size ? log_base : 0,
1544                                                 hdev->log);
1545         if (r < 0) {
1546             VHOST_OPS_DEBUG("vhost_set_log_base failed");
1547             r = -errno;
1548             goto fail_log;
1549         }
1550     }
1551
1552     if (vhost_dev_has_iommu(hdev)) {
1553         hdev->vhost_ops->vhost_set_iotlb_callback(hdev, true);
1554
1555         /* Update used ring information for IOTLB to work correctly,
1556          * vhost-kernel code requires for this.*/
1557         for (i = 0; i < hdev->nvqs; ++i) {
1558             struct vhost_virtqueue *vq = hdev->vqs + i;
1559             vhost_device_iotlb_miss(hdev, vq->used_phys, true);
1560         }
1561     }
1562     return 0;
1563 fail_log:
1564     vhost_log_put(hdev, false);
1565 fail_vq:
1566     while (--i >= 0) {
1567         vhost_virtqueue_stop(hdev,
1568                              vdev,
1569                              hdev->vqs + i,
1570                              hdev->vq_index + i);
1571     }
1572     i = hdev->nvqs;
1573
1574 fail_mem:
1575 fail_features:
1576
1577     hdev->started = false;
1578     return r;
1579 }
1580
1581 /* Host notifiers must be enabled at this point. */
1582 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
1583 {
1584     int i;
1585
1586     /* should only be called after backend is connected */
1587     assert(hdev->vhost_ops);
1588
1589     for (i = 0; i < hdev->nvqs; ++i) {
1590         vhost_virtqueue_stop(hdev,
1591                              vdev,
1592                              hdev->vqs + i,
1593                              hdev->vq_index + i);
1594     }
1595
1596     if (vhost_dev_has_iommu(hdev)) {
1597         hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
1598         memory_listener_unregister(&hdev->iommu_listener);
1599     }
1600     vhost_log_put(hdev, true);
1601     hdev->started = false;
1602     hdev->vdev = NULL;
1603 }
1604
1605 int vhost_net_set_backend(struct vhost_dev *hdev,
1606                           struct vhost_vring_file *file)
1607 {
1608     if (hdev->vhost_ops->vhost_net_set_backend) {
1609         return hdev->vhost_ops->vhost_net_set_backend(hdev, file);
1610     }
1611
1612     return -1;
1613 }