hw/virtio/vhost.c

   1 /*
   2  * vhost support
   3  *
   4  * Copyright Red Hat, Inc. 2010
   5  *
   6  * Authors:
   7  *  Michael S. Tsirkin <[email protected]>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qapi/error.h"
  18 #include "hw/virtio/vhost.h"
  19 #include "qemu/atomic.h"
  20 #include "qemu/range.h"
  21 #include "qemu/error-report.h"
  22 #include "qemu/memfd.h"
  23 #include "standard-headers/linux/vhost_types.h"
  24 #include "hw/virtio/virtio-bus.h"
  25 #include "hw/virtio/virtio-access.h"
  26 #include "migration/blocker.h"
  27 #include "migration/qemu-file-types.h"
  28 #include "sysemu/dma.h"
  29 #include "sysemu/tcg.h"
  30 #include "trace.h"
  31
  32 /* enabled until disconnected backend stabilizes */
  33 #define _VHOST_DEBUG 1
  34
  35 #ifdef _VHOST_DEBUG
  36 #define VHOST_OPS_DEBUG(retval, fmt, ...) \
  37     do { \
  38         error_report(fmt ": %s (%d)", ## __VA_ARGS__, \
  39                      strerror(-retval), -retval); \
  40     } while (0)
  41 #else
  42 #define VHOST_OPS_DEBUG(retval, fmt, ...) \
  43     do { } while (0)
  44 #endif
  45
  46 static struct vhost_log *vhost_log;
  47 static struct vhost_log *vhost_log_shm;
  48
  49 static unsigned int used_memslots;
  50 static QLIST_HEAD(, vhost_dev) vhost_devices =
  51     QLIST_HEAD_INITIALIZER(vhost_devices);
  52
  53 bool vhost_has_free_slot(void)
  54 {
  55     unsigned int slots_limit = ~0U;
  56     struct vhost_dev *hdev;
  57
  58     QLIST_FOREACH(hdev, &vhost_devices, entry) {
  59         unsigned int r = hdev->vhost_ops->vhost_backend_memslots_limit(hdev);
  60         slots_limit = MIN(slots_limit, r);
  61     }
  62     return slots_limit > used_memslots;
  63 }
  64
  65 static void vhost_dev_sync_region(struct vhost_dev *dev,
  66                                   MemoryRegionSection *section,
  67                                   uint64_t mfirst, uint64_t mlast,
  68                                   uint64_t rfirst, uint64_t rlast)
  69 {
  70     vhost_log_chunk_t *log = dev->log->log;
  71
  72     uint64_t start = MAX(mfirst, rfirst);
  73     uint64_t end = MIN(mlast, rlast);
  74     vhost_log_chunk_t *from = log + start / VHOST_LOG_CHUNK;
  75     vhost_log_chunk_t *to = log + end / VHOST_LOG_CHUNK + 1;
  76     uint64_t addr = QEMU_ALIGN_DOWN(start, VHOST_LOG_CHUNK);
  77
  78     if (end < start) {
  79         return;
  80     }
  81     assert(end / VHOST_LOG_CHUNK < dev->log_size);
  82     assert(start / VHOST_LOG_CHUNK < dev->log_size);
  83
  84     for (;from < to; ++from) {
  85         vhost_log_chunk_t log;
  86         /* We first check with non-atomic: much cheaper,
  87          * and we expect non-dirty to be the common case. */
  88         if (!*from) {
  89             addr += VHOST_LOG_CHUNK;
  90             continue;
  91         }
  92         /* Data must be read atomically. We don't really need barrier semantics
  93          * but it's easier to use atomic_* than roll our own. */
  94         log = qatomic_xchg(from, 0);
  95         while (log) {
  96             int bit = ctzl(log);
  97             hwaddr page_addr;
  98             hwaddr section_offset;
  99             hwaddr mr_offset;
 100             page_addr = addr + bit * VHOST_LOG_PAGE;
 101             section_offset = page_addr - section->offset_within_address_space;
 102             mr_offset = section_offset + section->offset_within_region;
 103             memory_region_set_dirty(section->mr, mr_offset, VHOST_LOG_PAGE);
 104             log &= ~(0x1ull << bit);
 105         }
 106         addr += VHOST_LOG_CHUNK;
 107     }
 108 }
 109
 110 static int vhost_sync_dirty_bitmap(struct vhost_dev *dev,
 111                                    MemoryRegionSection *section,
 112                                    hwaddr first,
 113                                    hwaddr last)
 114 {
 115     int i;
 116     hwaddr start_addr;
 117     hwaddr end_addr;
 118
 119     if (!dev->log_enabled || !dev->started) {
 120         return 0;
 121     }
 122     start_addr = section->offset_within_address_space;
 123     end_addr = range_get_last(start_addr, int128_get64(section->size));
 124     start_addr = MAX(first, start_addr);
 125     end_addr = MIN(last, end_addr);
 126
 127     for (i = 0; i < dev->mem->nregions; ++i) {
 128         struct vhost_memory_region *reg = dev->mem->regions + i;
 129         vhost_dev_sync_region(dev, section, start_addr, end_addr,
 130                               reg->guest_phys_addr,
 131                               range_get_last(reg->guest_phys_addr,
 132                                              reg->memory_size));
 133     }
 134     for (i = 0; i < dev->nvqs; ++i) {
 135         struct vhost_virtqueue *vq = dev->vqs + i;
 136
 137         if (!vq->used_phys && !vq->used_size) {
 138             continue;
 139         }
 140
 141         vhost_dev_sync_region(dev, section, start_addr, end_addr, vq->used_phys,
 142                               range_get_last(vq->used_phys, vq->used_size));
 143     }
 144     return 0;
 145 }
 146
 147 static void vhost_log_sync(MemoryListener *listener,
 148                           MemoryRegionSection *section)
 149 {
 150     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 151                                          memory_listener);
 152     vhost_sync_dirty_bitmap(dev, section, 0x0, ~0x0ULL);
 153 }
 154
 155 static void vhost_log_sync_range(struct vhost_dev *dev,
 156                                  hwaddr first, hwaddr last)
 157 {
 158     int i;
 159     /* FIXME: this is N^2 in number of sections */
 160     for (i = 0; i < dev->n_mem_sections; ++i) {
 161         MemoryRegionSection *section = &dev->mem_sections[i];
 162         vhost_sync_dirty_bitmap(dev, section, first, last);
 163     }
 164 }
 165
 166 static uint64_t vhost_get_log_size(struct vhost_dev *dev)
 167 {
 168     uint64_t log_size = 0;
 169     int i;
 170     for (i = 0; i < dev->mem->nregions; ++i) {
 171         struct vhost_memory_region *reg = dev->mem->regions + i;
 172         uint64_t last = range_get_last(reg->guest_phys_addr,
 173                                        reg->memory_size);
 174         log_size = MAX(log_size, last / VHOST_LOG_CHUNK + 1);
 175     }
 176     return log_size;
 177 }
 178
 179 static int vhost_set_backend_type(struct vhost_dev *dev,
 180                                   VhostBackendType backend_type)
 181 {
 182     int r = 0;
 183
 184     switch (backend_type) {
 185 #ifdef CONFIG_VHOST_KERNEL
 186     case VHOST_BACKEND_TYPE_KERNEL:
 187         dev->vhost_ops = &kernel_ops;
 188         break;
 189 #endif
 190 #ifdef CONFIG_VHOST_USER
 191     case VHOST_BACKEND_TYPE_USER:
 192         dev->vhost_ops = &user_ops;
 193         break;
 194 #endif
 195 #ifdef CONFIG_VHOST_VDPA
 196     case VHOST_BACKEND_TYPE_VDPA:
 197         dev->vhost_ops = &vdpa_ops;
 198         break;
 199 #endif
 200     default:
 201         error_report("Unknown vhost backend type");
 202         r = -1;
 203     }
 204
 205     return r;
 206 }
 207
 208 static struct vhost_log *vhost_log_alloc(uint64_t size, bool share)
 209 {
 210     Error *err = NULL;
 211     struct vhost_log *log;
 212     uint64_t logsize = size * sizeof(*(log->log));
 213     int fd = -1;
 214
 215     log = g_new0(struct vhost_log, 1);
 216     if (share) {
 217         log->log = qemu_memfd_alloc("vhost-log", logsize,
 218                                     F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
 219                                     &fd, &err);
 220         if (err) {
 221             error_report_err(err);
 222             g_free(log);
 223             return NULL;
 224         }
 225         memset(log->log, 0, logsize);
 226     } else {
 227         log->log = g_malloc0(logsize);
 228     }
 229
 230     log->size = size;
 231     log->refcnt = 1;
 232     log->fd = fd;
 233
 234     return log;
 235 }
 236
 237 static struct vhost_log *vhost_log_get(uint64_t size, bool share)
 238 {
 239     struct vhost_log *log = share ? vhost_log_shm : vhost_log;
 240
 241     if (!log || log->size != size) {
 242         log = vhost_log_alloc(size, share);
 243         if (share) {
 244             vhost_log_shm = log;
 245         } else {
 246             vhost_log = log;
 247         }
 248     } else {
 249         ++log->refcnt;
 250     }
 251
 252     return log;
 253 }
 254
 255 static void vhost_log_put(struct vhost_dev *dev, bool sync)
 256 {
 257     struct vhost_log *log = dev->log;
 258
 259     if (!log) {
 260         return;
 261     }
 262
 263     --log->refcnt;
 264     if (log->refcnt == 0) {
 265         /* Sync only the range covered by the old log */
 266         if (dev->log_size && sync) {
 267             vhost_log_sync_range(dev, 0, dev->log_size * VHOST_LOG_CHUNK - 1);
 268         }
 269
 270         if (vhost_log == log) {
 271             g_free(log->log);
 272             vhost_log = NULL;
 273         } else if (vhost_log_shm == log) {
 274             qemu_memfd_free(log->log, log->size * sizeof(*(log->log)),
 275                             log->fd);
 276             vhost_log_shm = NULL;
 277         }
 278
 279         g_free(log);
 280     }
 281
 282     dev->log = NULL;
 283     dev->log_size = 0;
 284 }
 285
 286 static bool vhost_dev_log_is_shared(struct vhost_dev *dev)
 287 {
 288     return dev->vhost_ops->vhost_requires_shm_log &&
 289            dev->vhost_ops->vhost_requires_shm_log(dev);
 290 }
 291
 292 static inline void vhost_dev_log_resize(struct vhost_dev *dev, uint64_t size)
 293 {
 294     struct vhost_log *log = vhost_log_get(size, vhost_dev_log_is_shared(dev));
 295     uint64_t log_base = (uintptr_t)log->log;
 296     int r;
 297
 298     /* inform backend of log switching, this must be done before
 299        releasing the current log, to ensure no logging is lost */
 300     r = dev->vhost_ops->vhost_set_log_base(dev, log_base, log);
 301     if (r < 0) {
 302         VHOST_OPS_DEBUG(r, "vhost_set_log_base failed");
 303     }
 304
 305     vhost_log_put(dev, true);
 306     dev->log = log;
 307     dev->log_size = size;
 308 }
 309
 310 static int vhost_dev_has_iommu(struct vhost_dev *dev)
 311 {
 312     VirtIODevice *vdev = dev->vdev;
 313
 314     /*
 315      * For vhost, VIRTIO_F_IOMMU_PLATFORM means the backend support
 316      * incremental memory mapping API via IOTLB API. For platform that
 317      * does not have IOMMU, there's no need to enable this feature
 318      * which may cause unnecessary IOTLB miss/update transactions.
 319      */
 320     return virtio_bus_device_iommu_enabled(vdev) &&
 321            virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM);
 322 }
 323
 324 static void *vhost_memory_map(struct vhost_dev *dev, hwaddr addr,
 325                               hwaddr *plen, bool is_write)
 326 {
 327     if (!vhost_dev_has_iommu(dev)) {
 328         return cpu_physical_memory_map(addr, plen, is_write);
 329     } else {
 330         return (void *)(uintptr_t)addr;
 331     }
 332 }
 333
 334 static void vhost_memory_unmap(struct vhost_dev *dev, void *buffer,
 335                                hwaddr len, int is_write,
 336                                hwaddr access_len)
 337 {
 338     if (!vhost_dev_has_iommu(dev)) {
 339         cpu_physical_memory_unmap(buffer, len, is_write, access_len);
 340     }
 341 }
 342
 343 static int vhost_verify_ring_part_mapping(void *ring_hva,
 344                                           uint64_t ring_gpa,
 345                                           uint64_t ring_size,
 346                                           void *reg_hva,
 347                                           uint64_t reg_gpa,
 348                                           uint64_t reg_size)
 349 {
 350     uint64_t hva_ring_offset;
 351     uint64_t ring_last = range_get_last(ring_gpa, ring_size);
 352     uint64_t reg_last = range_get_last(reg_gpa, reg_size);
 353
 354     if (ring_last < reg_gpa || ring_gpa > reg_last) {
 355         return 0;
 356     }
 357     /* check that whole ring's is mapped */
 358     if (ring_last > reg_last) {
 359         return -ENOMEM;
 360     }
 361     /* check that ring's MemoryRegion wasn't replaced */
 362     hva_ring_offset = ring_gpa - reg_gpa;
 363     if (ring_hva != reg_hva + hva_ring_offset) {
 364         return -EBUSY;
 365     }
 366
 367     return 0;
 368 }
 369
 370 static int vhost_verify_ring_mappings(struct vhost_dev *dev,
 371                                       void *reg_hva,
 372                                       uint64_t reg_gpa,
 373                                       uint64_t reg_size)
 374 {
 375     int i, j;
 376     int r = 0;
 377     const char *part_name[] = {
 378         "descriptor table",
 379         "available ring",
 380         "used ring"
 381     };
 382
 383     if (vhost_dev_has_iommu(dev)) {
 384         return 0;
 385     }
 386
 387     for (i = 0; i < dev->nvqs; ++i) {
 388         struct vhost_virtqueue *vq = dev->vqs + i;
 389
 390         if (vq->desc_phys == 0) {
 391             continue;
 392         }
 393
 394         j = 0;
 395         r = vhost_verify_ring_part_mapping(
 396                 vq->desc, vq->desc_phys, vq->desc_size,
 397                 reg_hva, reg_gpa, reg_size);
 398         if (r) {
 399             break;
 400         }
 401
 402         j++;
 403         r = vhost_verify_ring_part_mapping(
 404                 vq->avail, vq->avail_phys, vq->avail_size,
 405                 reg_hva, reg_gpa, reg_size);
 406         if (r) {
 407             break;
 408         }
 409
 410         j++;
 411         r = vhost_verify_ring_part_mapping(
 412                 vq->used, vq->used_phys, vq->used_size,
 413                 reg_hva, reg_gpa, reg_size);
 414         if (r) {
 415             break;
 416         }
 417     }
 418
 419     if (r == -ENOMEM) {
 420         error_report("Unable to map %s for ring %d", part_name[j], i);
 421     } else if (r == -EBUSY) {
 422         error_report("%s relocated for ring %d", part_name[j], i);
 423     }
 424     return r;
 425 }
 426
 427 /*
 428  * vhost_section: identify sections needed for vhost access
 429  *
 430  * We only care about RAM sections here (where virtqueue and guest
 431  * internals accessed by virtio might live). If we find one we still
 432  * allow the backend to potentially filter it out of our list.
 433  */
 434 static bool vhost_section(struct vhost_dev *dev, MemoryRegionSection *section)
 435 {
 436     MemoryRegion *mr = section->mr;
 437
 438     if (memory_region_is_ram(mr) && !memory_region_is_rom(mr)) {
 439         uint8_t dirty_mask = memory_region_get_dirty_log_mask(mr);
 440         uint8_t handled_dirty;
 441
 442         /*
 443          * Kernel based vhost doesn't handle any block which is doing
 444          * dirty-tracking other than migration for which it has
 445          * specific logging support. However for TCG the kernel never
 446          * gets involved anyway so we can also ignore it's
 447          * self-modiying code detection flags. However a vhost-user
 448          * client could still confuse a TCG guest if it re-writes
 449          * executable memory that has already been translated.
 450          */
 451         handled_dirty = (1 << DIRTY_MEMORY_MIGRATION) |
 452             (1 << DIRTY_MEMORY_CODE);
 453
 454         if (dirty_mask & ~handled_dirty) {
 455             trace_vhost_reject_section(mr->name, 1);
 456             return false;
 457         }
 458
 459         if (dev->vhost_ops->vhost_backend_mem_section_filter &&
 460             !dev->vhost_ops->vhost_backend_mem_section_filter(dev, section)) {
 461             trace_vhost_reject_section(mr->name, 2);
 462             return false;
 463         }
 464
 465         trace_vhost_section(mr->name);
 466         return true;
 467     } else {
 468         trace_vhost_reject_section(mr->name, 3);
 469         return false;
 470     }
 471 }
 472
 473 static void vhost_begin(MemoryListener *listener)
 474 {
 475     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 476                                          memory_listener);
 477     dev->tmp_sections = NULL;
 478     dev->n_tmp_sections = 0;
 479 }
 480
 481 static void vhost_commit(MemoryListener *listener)
 482 {
 483     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 484                                          memory_listener);
 485     MemoryRegionSection *old_sections;
 486     int n_old_sections;
 487     uint64_t log_size;
 488     size_t regions_size;
 489     int r;
 490     int i;
 491     bool changed = false;
 492
 493     /* Note we can be called before the device is started, but then
 494      * starting the device calls set_mem_table, so we need to have
 495      * built the data structures.
 496      */
 497     old_sections = dev->mem_sections;
 498     n_old_sections = dev->n_mem_sections;
 499     dev->mem_sections = dev->tmp_sections;
 500     dev->n_mem_sections = dev->n_tmp_sections;
 501
 502     if (dev->n_mem_sections != n_old_sections) {
 503         changed = true;
 504     } else {
 505         /* Same size, lets check the contents */
 506         for (int i = 0; i < n_old_sections; i++) {
 507             if (!MemoryRegionSection_eq(&old_sections[i],
 508                                         &dev->mem_sections[i])) {
 509                 changed = true;
 510                 break;
 511             }
 512         }
 513     }
 514
 515     trace_vhost_commit(dev->started, changed);
 516     if (!changed) {
 517         goto out;
 518     }
 519
 520     /* Rebuild the regions list from the new sections list */
 521     regions_size = offsetof(struct vhost_memory, regions) +
 522                        dev->n_mem_sections * sizeof dev->mem->regions[0];
 523     dev->mem = g_realloc(dev->mem, regions_size);
 524     dev->mem->nregions = dev->n_mem_sections;
 525     used_memslots = dev->mem->nregions;
 526     for (i = 0; i < dev->n_mem_sections; i++) {
 527         struct vhost_memory_region *cur_vmr = dev->mem->regions + i;
 528         struct MemoryRegionSection *mrs = dev->mem_sections + i;
 529
 530         cur_vmr->guest_phys_addr = mrs->offset_within_address_space;
 531         cur_vmr->memory_size     = int128_get64(mrs->size);
 532         cur_vmr->userspace_addr  =
 533             (uintptr_t)memory_region_get_ram_ptr(mrs->mr) +
 534             mrs->offset_within_region;
 535         cur_vmr->flags_padding   = 0;
 536     }
 537
 538     if (!dev->started) {
 539         goto out;
 540     }
 541
 542     for (i = 0; i < dev->mem->nregions; i++) {
 543         if (vhost_verify_ring_mappings(dev,
 544                        (void *)(uintptr_t)dev->mem->regions[i].userspace_addr,
 545                        dev->mem->regions[i].guest_phys_addr,
 546                        dev->mem->regions[i].memory_size)) {
 547             error_report("Verify ring failure on region %d", i);
 548             abort();
 549         }
 550     }
 551
 552     if (!dev->log_enabled) {
 553         r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
 554         if (r < 0) {
 555             VHOST_OPS_DEBUG(r, "vhost_set_mem_table failed");
 556         }
 557         goto out;
 558     }
 559     log_size = vhost_get_log_size(dev);
 560     /* We allocate an extra 4K bytes to log,
 561      * to reduce the * number of reallocations. */
 562 #define VHOST_LOG_BUFFER (0x1000 / sizeof *dev->log)
 563     /* To log more, must increase log size before table update. */
 564     if (dev->log_size < log_size) {
 565         vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
 566     }
 567     r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
 568     if (r < 0) {
 569         VHOST_OPS_DEBUG(r, "vhost_set_mem_table failed");
 570     }
 571     /* To log less, can only decrease log size after table update. */
 572     if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
 573         vhost_dev_log_resize(dev, log_size);
 574     }
 575
 576 out:
 577     /* Deref the old list of sections, this must happen _after_ the
 578      * vhost_set_mem_table to ensure the client isn't still using the
 579      * section we're about to unref.
 580      */
 581     while (n_old_sections--) {
 582         memory_region_unref(old_sections[n_old_sections].mr);
 583     }
 584     g_free(old_sections);
 585     return;
 586 }
 587
 588 /* Adds the section data to the tmp_section structure.
 589  * It relies on the listener calling us in memory address order
 590  * and for each region (via the _add and _nop methods) to
 591  * join neighbours.
 592  */
 593 static void vhost_region_add_section(struct vhost_dev *dev,
 594                                      MemoryRegionSection *section)
 595 {
 596     bool need_add = true;
 597     uint64_t mrs_size = int128_get64(section->size);
 598     uint64_t mrs_gpa = section->offset_within_address_space;
 599     uintptr_t mrs_host = (uintptr_t)memory_region_get_ram_ptr(section->mr) +
 600                          section->offset_within_region;
 601     RAMBlock *mrs_rb = section->mr->ram_block;
 602
 603     trace_vhost_region_add_section(section->mr->name, mrs_gpa, mrs_size,
 604                                    mrs_host);
 605
 606     if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER) {
 607         /* Round the section to it's page size */
 608         /* First align the start down to a page boundary */
 609         size_t mrs_page = qemu_ram_pagesize(mrs_rb);
 610         uint64_t alignage = mrs_host & (mrs_page - 1);
 611         if (alignage) {
 612             mrs_host -= alignage;
 613             mrs_size += alignage;
 614             mrs_gpa  -= alignage;
 615         }
 616         /* Now align the size up to a page boundary */
 617         alignage = mrs_size & (mrs_page - 1);
 618         if (alignage) {
 619             mrs_size += mrs_page - alignage;
 620         }
 621         trace_vhost_region_add_section_aligned(section->mr->name, mrs_gpa,
 622                                                mrs_size, mrs_host);
 623     }
 624
 625     if (dev->n_tmp_sections) {
 626         /* Since we already have at least one section, lets see if
 627          * this extends it; since we're scanning in order, we only
 628          * have to look at the last one, and the FlatView that calls
 629          * us shouldn't have overlaps.
 630          */
 631         MemoryRegionSection *prev_sec = dev->tmp_sections +
 632                                                (dev->n_tmp_sections - 1);
 633         uint64_t prev_gpa_start = prev_sec->offset_within_address_space;
 634         uint64_t prev_size = int128_get64(prev_sec->size);
 635         uint64_t prev_gpa_end   = range_get_last(prev_gpa_start, prev_size);
 636         uint64_t prev_host_start =
 637                         (uintptr_t)memory_region_get_ram_ptr(prev_sec->mr) +
 638                         prev_sec->offset_within_region;
 639         uint64_t prev_host_end   = range_get_last(prev_host_start, prev_size);
 640
 641         if (mrs_gpa <= (prev_gpa_end + 1)) {
 642             /* OK, looks like overlapping/intersecting - it's possible that
 643              * the rounding to page sizes has made them overlap, but they should
 644              * match up in the same RAMBlock if they do.
 645              */
 646             if (mrs_gpa < prev_gpa_start) {
 647                 error_report("%s:Section '%s' rounded to %"PRIx64
 648                              " prior to previous '%s' %"PRIx64,
 649                              __func__, section->mr->name, mrs_gpa,
 650                              prev_sec->mr->name, prev_gpa_start);
 651                 /* A way to cleanly fail here would be better */
 652                 return;
 653             }
 654             /* Offset from the start of the previous GPA to this GPA */
 655             size_t offset = mrs_gpa - prev_gpa_start;
 656
 657             if (prev_host_start + offset == mrs_host &&
 658                 section->mr == prev_sec->mr &&
 659                 (!dev->vhost_ops->vhost_backend_can_merge ||
 660                  dev->vhost_ops->vhost_backend_can_merge(dev,
 661                     mrs_host, mrs_size,
 662                     prev_host_start, prev_size))) {
 663                 uint64_t max_end = MAX(prev_host_end, mrs_host + mrs_size);
 664                 need_add = false;
 665                 prev_sec->offset_within_address_space =
 666                     MIN(prev_gpa_start, mrs_gpa);
 667                 prev_sec->offset_within_region =
 668                     MIN(prev_host_start, mrs_host) -
 669                     (uintptr_t)memory_region_get_ram_ptr(prev_sec->mr);
 670                 prev_sec->size = int128_make64(max_end - MIN(prev_host_start,
 671                                                mrs_host));
 672                 trace_vhost_region_add_section_merge(section->mr->name,
 673                                         int128_get64(prev_sec->size),
 674                                         prev_sec->offset_within_address_space,
 675                                         prev_sec->offset_within_region);
 676             } else {
 677                 /* adjoining regions are fine, but overlapping ones with
 678                  * different blocks/offsets shouldn't happen
 679                  */
 680                 if (mrs_gpa != prev_gpa_end + 1) {
 681                     error_report("%s: Overlapping but not coherent sections "
 682                                  "at %"PRIx64,
 683                                  __func__, mrs_gpa);
 684                     return;
 685                 }
 686             }
 687         }
 688     }
 689
 690     if (need_add) {
 691         ++dev->n_tmp_sections;
 692         dev->tmp_sections = g_renew(MemoryRegionSection, dev->tmp_sections,
 693                                     dev->n_tmp_sections);
 694         dev->tmp_sections[dev->n_tmp_sections - 1] = *section;
 695         /* The flatview isn't stable and we don't use it, making it NULL
 696          * means we can memcmp the list.
 697          */
 698         dev->tmp_sections[dev->n_tmp_sections - 1].fv = NULL;
 699         memory_region_ref(section->mr);
 700     }
 701 }
 702
 703 /* Used for both add and nop callbacks */
 704 static void vhost_region_addnop(MemoryListener *listener,
 705                                 MemoryRegionSection *section)
 706 {
 707     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 708                                          memory_listener);
 709
 710     if (!vhost_section(dev, section)) {
 711         return;
 712     }
 713     vhost_region_add_section(dev, section);
 714 }
 715
 716 static void vhost_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
 717 {
 718     struct vhost_iommu *iommu = container_of(n, struct vhost_iommu, n);
 719     struct vhost_dev *hdev = iommu->hdev;
 720     hwaddr iova = iotlb->iova + iommu->iommu_offset;
 721
 722     if (vhost_backend_invalidate_device_iotlb(hdev, iova,
 723                                               iotlb->addr_mask + 1)) {
 724         error_report("Fail to invalidate device iotlb");
 725     }
 726 }
 727
 728 static void vhost_iommu_region_add(MemoryListener *listener,
 729                                    MemoryRegionSection *section)
 730 {
 731     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 732                                          iommu_listener);
 733     struct vhost_iommu *iommu;
 734     Int128 end;
 735     int iommu_idx;
 736     IOMMUMemoryRegion *iommu_mr;
 737     int ret;
 738
 739     if (!memory_region_is_iommu(section->mr)) {
 740         return;
 741     }
 742
 743     iommu_mr = IOMMU_MEMORY_REGION(section->mr);
 744
 745     iommu = g_malloc0(sizeof(*iommu));
 746     end = int128_add(int128_make64(section->offset_within_region),
 747                      section->size);
 748     end = int128_sub(end, int128_one());
 749     iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
 750                                                    MEMTXATTRS_UNSPECIFIED);
 751     iommu_notifier_init(&iommu->n, vhost_iommu_unmap_notify,
 752                         IOMMU_NOTIFIER_DEVIOTLB_UNMAP,
 753                         section->offset_within_region,
 754                         int128_get64(end),
 755                         iommu_idx);
 756     iommu->mr = section->mr;
 757     iommu->iommu_offset = section->offset_within_address_space -
 758                           section->offset_within_region;
 759     iommu->hdev = dev;
 760     ret = memory_region_register_iommu_notifier(section->mr, &iommu->n, NULL);
 761     if (ret) {
 762         /*
 763          * Some vIOMMUs do not support dev-iotlb yet.  If so, try to use the
 764          * UNMAP legacy message
 765          */
 766         iommu->n.notifier_flags = IOMMU_NOTIFIER_UNMAP;
 767         memory_region_register_iommu_notifier(section->mr, &iommu->n,
 768                                               &error_fatal);
 769     }
 770     QLIST_INSERT_HEAD(&dev->iommu_list, iommu, iommu_next);
 771     /* TODO: can replay help performance here? */
 772 }
 773
 774 static void vhost_iommu_region_del(MemoryListener *listener,
 775                                    MemoryRegionSection *section)
 776 {
 777     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 778                                          iommu_listener);
 779     struct vhost_iommu *iommu;
 780
 781     if (!memory_region_is_iommu(section->mr)) {
 782         return;
 783     }
 784
 785     QLIST_FOREACH(iommu, &dev->iommu_list, iommu_next) {
 786         if (iommu->mr == section->mr &&
 787             iommu->n.start == section->offset_within_region) {
 788             memory_region_unregister_iommu_notifier(iommu->mr,
 789                                                     &iommu->n);
 790             QLIST_REMOVE(iommu, iommu_next);
 791             g_free(iommu);
 792             break;
 793         }
 794     }
 795 }
 796
 797 static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
 798                                     struct vhost_virtqueue *vq,
 799                                     unsigned idx, bool enable_log)
 800 {
 801     struct vhost_vring_addr addr;
 802     int r;
 803     memset(&addr, 0, sizeof(struct vhost_vring_addr));
 804
 805     if (dev->vhost_ops->vhost_vq_get_addr) {
 806         r = dev->vhost_ops->vhost_vq_get_addr(dev, &addr, vq);
 807         if (r < 0) {
 808             VHOST_OPS_DEBUG(r, "vhost_vq_get_addr failed");
 809             return r;
 810         }
 811     } else {
 812         addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc;
 813         addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail;
 814         addr.used_user_addr = (uint64_t)(unsigned long)vq->used;
 815     }
 816     addr.index = idx;
 817     addr.log_guest_addr = vq->used_phys;
 818     addr.flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0;
 819     r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
 820     if (r < 0) {
 821         VHOST_OPS_DEBUG(r, "vhost_set_vring_addr failed");
 822     }
 823     return r;
 824 }
 825
 826 static int vhost_dev_set_features(struct vhost_dev *dev,
 827                                   bool enable_log)
 828 {
 829     uint64_t features = dev->acked_features;
 830     int r;
 831     if (enable_log) {
 832         features |= 0x1ULL << VHOST_F_LOG_ALL;
 833     }
 834     if (!vhost_dev_has_iommu(dev)) {
 835         features &= ~(0x1ULL << VIRTIO_F_IOMMU_PLATFORM);
 836     }
 837     if (dev->vhost_ops->vhost_force_iommu) {
 838         if (dev->vhost_ops->vhost_force_iommu(dev) == true) {
 839             features |= 0x1ULL << VIRTIO_F_IOMMU_PLATFORM;
 840        }
 841     }
 842     r = dev->vhost_ops->vhost_set_features(dev, features);
 843     if (r < 0) {
 844         VHOST_OPS_DEBUG(r, "vhost_set_features failed");
 845         goto out;
 846     }
 847     if (dev->vhost_ops->vhost_set_backend_cap) {
 848         r = dev->vhost_ops->vhost_set_backend_cap(dev);
 849         if (r < 0) {
 850             VHOST_OPS_DEBUG(r, "vhost_set_backend_cap failed");
 851             goto out;
 852         }
 853     }
 854
 855 out:
 856     return r;
 857 }
 858
 859 static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
 860 {
 861     int r, i, idx;
 862     hwaddr addr;
 863
 864     r = vhost_dev_set_features(dev, enable_log);
 865     if (r < 0) {
 866         goto err_features;
 867     }
 868     for (i = 0; i < dev->nvqs; ++i) {
 869         idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
 870         addr = virtio_queue_get_desc_addr(dev->vdev, idx);
 871         if (!addr) {
 872             /*
 873              * The queue might not be ready for start. If this
 874              * is the case there is no reason to continue the process.
 875              * The similar logic is used by the vhost_virtqueue_start()
 876              * routine.
 877              */
 878             continue;
 879         }
 880         r = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
 881                                      enable_log);
 882         if (r < 0) {
 883             goto err_vq;
 884         }
 885     }
 886     return 0;
 887 err_vq:
 888     for (; i >= 0; --i) {
 889         idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
 890         vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
 891                                  dev->log_enabled);
 892     }
 893     vhost_dev_set_features(dev, dev->log_enabled);
 894 err_features:
 895     return r;
 896 }
 897
 898 static int vhost_migration_log(MemoryListener *listener, bool enable)
 899 {
 900     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
 901                                          memory_listener);
 902     int r;
 903     if (enable == dev->log_enabled) {
 904         return 0;
 905     }
 906     if (!dev->started) {
 907         dev->log_enabled = enable;
 908         return 0;
 909     }
 910
 911     r = 0;
 912     if (!enable) {
 913         r = vhost_dev_set_log(dev, false);
 914         if (r < 0) {
 915             goto check_dev_state;
 916         }
 917         vhost_log_put(dev, false);
 918     } else {
 919         vhost_dev_log_resize(dev, vhost_get_log_size(dev));
 920         r = vhost_dev_set_log(dev, true);
 921         if (r < 0) {
 922             goto check_dev_state;
 923         }
 924     }
 925
 926 check_dev_state:
 927     dev->log_enabled = enable;
 928     /*
 929      * vhost-user-* devices could change their state during log
 930      * initialization due to disconnect. So check dev state after
 931      * vhost communication.
 932      */
 933     if (!dev->started) {
 934         /*
 935          * Since device is in the stopped state, it is okay for
 936          * migration. Return success.
 937          */
 938         r = 0;
 939     }
 940     if (r) {
 941         /* An error occurred. */
 942         dev->log_enabled = false;
 943     }
 944
 945     return r;
 946 }
 947
 948 static void vhost_log_global_start(MemoryListener *listener)
 949 {
 950     int r;
 951
 952     r = vhost_migration_log(listener, true);
 953     if (r < 0) {
 954         abort();
 955     }
 956 }
 957
 958 static void vhost_log_global_stop(MemoryListener *listener)
 959 {
 960     int r;
 961
 962     r = vhost_migration_log(listener, false);
 963     if (r < 0) {
 964         abort();
 965     }
 966 }
 967
 968 static void vhost_log_start(MemoryListener *listener,
 969                             MemoryRegionSection *section,
 970                             int old, int new)
 971 {
 972     /* FIXME: implement */
 973 }
 974
 975 static void vhost_log_stop(MemoryListener *listener,
 976                            MemoryRegionSection *section,
 977                            int old, int new)
 978 {
 979     /* FIXME: implement */
 980 }
 981
 982 /* The vhost driver natively knows how to handle the vrings of non
 983  * cross-endian legacy devices and modern devices. Only legacy devices
 984  * exposed to a bi-endian guest may require the vhost driver to use a
 985  * specific endianness.
 986  */
 987 static inline bool vhost_needs_vring_endian(VirtIODevice *vdev)
 988 {
 989     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
 990         return false;
 991     }
 992 #ifdef HOST_WORDS_BIGENDIAN
 993     return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_LITTLE;
 994 #else
 995     return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_BIG;
 996 #endif
 997 }
 998
 999 static int vhost_virtqueue_set_vring_endian_legacy(struct vhost_dev *dev,
1000                                                    bool is_big_endian,
1001                                                    int vhost_vq_index)
1002 {
1003     int r;
1004     struct vhost_vring_state s = {
1005         .index = vhost_vq_index,
1006         .num = is_big_endian
1007     };
1008
1009     r = dev->vhost_ops->vhost_set_vring_endian(dev, &s);
1010     if (r < 0) {
1011         VHOST_OPS_DEBUG(r, "vhost_set_vring_endian failed");
1012     }
1013     return r;
1014 }
1015
1016 static int vhost_memory_region_lookup(struct vhost_dev *hdev,
1017                                       uint64_t gpa, uint64_t *uaddr,
1018                                       uint64_t *len)
1019 {
1020     int i;
1021
1022     for (i = 0; i < hdev->mem->nregions; i++) {
1023         struct vhost_memory_region *reg = hdev->mem->regions + i;
1024
1025         if (gpa >= reg->guest_phys_addr &&
1026             reg->guest_phys_addr + reg->memory_size > gpa) {
1027             *uaddr = reg->userspace_addr + gpa - reg->guest_phys_addr;
1028             *len = reg->guest_phys_addr + reg->memory_size - gpa;
1029             return 0;
1030         }
1031     }
1032
1033     return -EFAULT;
1034 }
1035
1036 int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int write)
1037 {
1038     IOMMUTLBEntry iotlb;
1039     uint64_t uaddr, len;
1040     int ret = -EFAULT;
1041
1042     RCU_READ_LOCK_GUARD();
1043
1044     trace_vhost_iotlb_miss(dev, 1);
1045
1046     iotlb = address_space_get_iotlb_entry(dev->vdev->dma_as,
1047                                           iova, write,
1048                                           MEMTXATTRS_UNSPECIFIED);
1049     if (iotlb.target_as != NULL) {
1050         ret = vhost_memory_region_lookup(dev, iotlb.translated_addr,
1051                                          &uaddr, &len);
1052         if (ret) {
1053             trace_vhost_iotlb_miss(dev, 3);
1054             error_report("Fail to lookup the translated address "
1055                          "%"PRIx64, iotlb.translated_addr);
1056             goto out;
1057         }
1058
1059         len = MIN(iotlb.addr_mask + 1, len);
1060         iova = iova & ~iotlb.addr_mask;
1061
1062         ret = vhost_backend_update_device_iotlb(dev, iova, uaddr,
1063                                                 len, iotlb.perm);
1064         if (ret) {
1065             trace_vhost_iotlb_miss(dev, 4);
1066             error_report("Fail to update device iotlb");
1067             goto out;
1068         }
1069     }
1070
1071     trace_vhost_iotlb_miss(dev, 2);
1072
1073 out:
1074     return ret;
1075 }
1076
1077 static int vhost_virtqueue_start(struct vhost_dev *dev,
1078                                 struct VirtIODevice *vdev,
1079                                 struct vhost_virtqueue *vq,
1080                                 unsigned idx)
1081 {
1082     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
1083     VirtioBusState *vbus = VIRTIO_BUS(qbus);
1084     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
1085     hwaddr s, l, a;
1086     int r;
1087     int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
1088     struct vhost_vring_file file = {
1089         .index = vhost_vq_index
1090     };
1091     struct vhost_vring_state state = {
1092         .index = vhost_vq_index
1093     };
1094     struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
1095
1096     a = virtio_queue_get_desc_addr(vdev, idx);
1097     if (a == 0) {
1098         /* Queue might not be ready for start */
1099         return 0;
1100     }
1101
1102     vq->num = state.num = virtio_queue_get_num(vdev, idx);
1103     r = dev->vhost_ops->vhost_set_vring_num(dev, &state);
1104     if (r) {
1105         VHOST_OPS_DEBUG(r, "vhost_set_vring_num failed");
1106         return r;
1107     }
1108
1109     state.num = virtio_queue_get_last_avail_idx(vdev, idx);
1110     r = dev->vhost_ops->vhost_set_vring_base(dev, &state);
1111     if (r) {
1112         VHOST_OPS_DEBUG(r, "vhost_set_vring_base failed");
1113         return r;
1114     }
1115
1116     if (vhost_needs_vring_endian(vdev)) {
1117         r = vhost_virtqueue_set_vring_endian_legacy(dev,
1118                                                     virtio_is_big_endian(vdev),
1119                                                     vhost_vq_index);
1120         if (r) {
1121             return r;
1122         }
1123     }
1124
1125     vq->desc_size = s = l = virtio_queue_get_desc_size(vdev, idx);
1126     vq->desc_phys = a;
1127     vq->desc = vhost_memory_map(dev, a, &l, false);
1128     if (!vq->desc || l != s) {
1129         r = -ENOMEM;
1130         goto fail_alloc_desc;
1131     }
1132     vq->avail_size = s = l = virtio_queue_get_avail_size(vdev, idx);
1133     vq->avail_phys = a = virtio_queue_get_avail_addr(vdev, idx);
1134     vq->avail = vhost_memory_map(dev, a, &l, false);
1135     if (!vq->avail || l != s) {
1136         r = -ENOMEM;
1137         goto fail_alloc_avail;
1138     }
1139     vq->used_size = s = l = virtio_queue_get_used_size(vdev, idx);
1140     vq->used_phys = a = virtio_queue_get_used_addr(vdev, idx);
1141     vq->used = vhost_memory_map(dev, a, &l, true);
1142     if (!vq->used || l != s) {
1143         r = -ENOMEM;
1144         goto fail_alloc_used;
1145     }
1146
1147     r = vhost_virtqueue_set_addr(dev, vq, vhost_vq_index, dev->log_enabled);
1148     if (r < 0) {
1149         goto fail_alloc;
1150     }
1151
1152     file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
1153     r = dev->vhost_ops->vhost_set_vring_kick(dev, &file);
1154     if (r) {
1155         VHOST_OPS_DEBUG(r, "vhost_set_vring_kick failed");
1156         goto fail_kick;
1157     }
1158
1159     /* Clear and discard previous events if any. */
1160     event_notifier_test_and_clear(&vq->masked_notifier);
1161
1162     /* Init vring in unmasked state, unless guest_notifier_mask
1163      * will do it later.
1164      */
1165     if (!vdev->use_guest_notifier_mask) {
1166         /* TODO: check and handle errors. */
1167         vhost_virtqueue_mask(dev, vdev, idx, false);
1168     }
1169
1170     if (k->query_guest_notifiers &&
1171         k->query_guest_notifiers(qbus->parent) &&
1172         virtio_queue_vector(vdev, idx) == VIRTIO_NO_VECTOR) {
1173         file.fd = -1;
1174         r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
1175         if (r) {
1176             goto fail_vector;
1177         }
1178     }
1179
1180     return 0;
1181
1182 fail_vector:
1183 fail_kick:
1184 fail_alloc:
1185     vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx),
1186                        0, 0);
1187 fail_alloc_used:
1188     vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx),
1189                        0, 0);
1190 fail_alloc_avail:
1191     vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx),
1192                        0, 0);
1193 fail_alloc_desc:
1194     return r;
1195 }
1196
1197 static void vhost_virtqueue_stop(struct vhost_dev *dev,
1198                                     struct VirtIODevice *vdev,
1199                                     struct vhost_virtqueue *vq,
1200                                     unsigned idx)
1201 {
1202     int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
1203     struct vhost_vring_state state = {
1204         .index = vhost_vq_index,
1205     };
1206     int r;
1207
1208     if (virtio_queue_get_desc_addr(vdev, idx) == 0) {
1209         /* Don't stop the virtqueue which might have not been started */
1210         return;
1211     }
1212
1213     r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
1214     if (r < 0) {
1215         VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d", idx, r);
1216         /* Connection to the backend is broken, so let's sync internal
1217          * last avail idx to the device used idx.
1218          */
1219         virtio_queue_restore_last_avail_idx(vdev, idx);
1220     } else {
1221         virtio_queue_set_last_avail_idx(vdev, idx, state.num);
1222     }
1223     virtio_queue_invalidate_signalled_used(vdev, idx);
1224     virtio_queue_update_used_idx(vdev, idx);
1225
1226     /* In the cross-endian case, we need to reset the vring endianness to
1227      * native as legacy devices expect so by default.
1228      */
1229     if (vhost_needs_vring_endian(vdev)) {
1230         vhost_virtqueue_set_vring_endian_legacy(dev,
1231                                                 !virtio_is_big_endian(vdev),
1232                                                 vhost_vq_index);
1233     }
1234
1235     vhost_memory_unmap(dev, vq->used, virtio_queue_get_used_size(vdev, idx),
1236                        1, virtio_queue_get_used_size(vdev, idx));
1237     vhost_memory_unmap(dev, vq->avail, virtio_queue_get_avail_size(vdev, idx),
1238                        0, virtio_queue_get_avail_size(vdev, idx));
1239     vhost_memory_unmap(dev, vq->desc, virtio_queue_get_desc_size(vdev, idx),
1240                        0, virtio_queue_get_desc_size(vdev, idx));
1241 }
1242
1243 static void vhost_eventfd_add(MemoryListener *listener,
1244                               MemoryRegionSection *section,
1245                               bool match_data, uint64_t data, EventNotifier *e)
1246 {
1247 }
1248
1249 static void vhost_eventfd_del(MemoryListener *listener,
1250                               MemoryRegionSection *section,
1251                               bool match_data, uint64_t data, EventNotifier *e)
1252 {
1253 }
1254
1255 static int vhost_virtqueue_set_busyloop_timeout(struct vhost_dev *dev,
1256                                                 int n, uint32_t timeout)
1257 {
1258     int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
1259     struct vhost_vring_state state = {
1260         .index = vhost_vq_index,
1261         .num = timeout,
1262     };
1263     int r;
1264
1265     if (!dev->vhost_ops->vhost_set_vring_busyloop_timeout) {
1266         return -EINVAL;
1267     }
1268
1269     r = dev->vhost_ops->vhost_set_vring_busyloop_timeout(dev, &state);
1270     if (r) {
1271         VHOST_OPS_DEBUG(r, "vhost_set_vring_busyloop_timeout failed");
1272         return r;
1273     }
1274
1275     return 0;
1276 }
1277
1278 static int vhost_virtqueue_init(struct vhost_dev *dev,
1279                                 struct vhost_virtqueue *vq, int n)
1280 {
1281     int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
1282     struct vhost_vring_file file = {
1283         .index = vhost_vq_index,
1284     };
1285     int r = event_notifier_init(&vq->masked_notifier, 0);
1286     if (r < 0) {
1287         return r;
1288     }
1289
1290     file.fd = event_notifier_get_fd(&vq->masked_notifier);
1291     r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
1292     if (r) {
1293         VHOST_OPS_DEBUG(r, "vhost_set_vring_call failed");
1294         goto fail_call;
1295     }
1296
1297     vq->dev = dev;
1298
1299     return 0;
1300 fail_call:
1301     event_notifier_cleanup(&vq->masked_notifier);
1302     return r;
1303 }
1304
1305 static void vhost_virtqueue_cleanup(struct vhost_virtqueue *vq)
1306 {
1307     event_notifier_cleanup(&vq->masked_notifier);
1308 }
1309
1310 int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
1311                    VhostBackendType backend_type, uint32_t busyloop_timeout,
1312                    Error **errp)
1313 {
1314     uint64_t features;
1315     int i, r, n_initialized_vqs = 0;
1316
1317     hdev->vdev = NULL;
1318     hdev->migration_blocker = NULL;
1319
1320     r = vhost_set_backend_type(hdev, backend_type);
1321     assert(r >= 0);
1322
1323     r = hdev->vhost_ops->vhost_backend_init(hdev, opaque, errp);
1324     if (r < 0) {
1325         goto fail;
1326     }
1327
1328     r = hdev->vhost_ops->vhost_set_owner(hdev);
1329     if (r < 0) {
1330         error_setg_errno(errp, -r, "vhost_set_owner failed");
1331         goto fail;
1332     }
1333
1334     r = hdev->vhost_ops->vhost_get_features(hdev, &features);
1335     if (r < 0) {
1336         error_setg_errno(errp, -r, "vhost_get_features failed");
1337         goto fail;
1338     }
1339
1340     for (i = 0; i < hdev->nvqs; ++i, ++n_initialized_vqs) {
1341         r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i);
1342         if (r < 0) {
1343             error_setg_errno(errp, -r, "Failed to initialize virtqueue %d", i);
1344             goto fail;
1345         }
1346     }
1347
1348     if (busyloop_timeout) {
1349         for (i = 0; i < hdev->nvqs; ++i) {
1350             r = vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i,
1351                                                      busyloop_timeout);
1352             if (r < 0) {
1353                 error_setg_errno(errp, -r, "Failed to set busyloop timeout");
1354                 goto fail_busyloop;
1355             }
1356         }
1357     }
1358
1359     hdev->features = features;
1360
1361     hdev->memory_listener = (MemoryListener) {
1362         .name = "vhost",
1363         .begin = vhost_begin,
1364         .commit = vhost_commit,
1365         .region_add = vhost_region_addnop,
1366         .region_nop = vhost_region_addnop,
1367         .log_start = vhost_log_start,
1368         .log_stop = vhost_log_stop,
1369         .log_sync = vhost_log_sync,
1370         .log_global_start = vhost_log_global_start,
1371         .log_global_stop = vhost_log_global_stop,
1372         .eventfd_add = vhost_eventfd_add,
1373         .eventfd_del = vhost_eventfd_del,
1374         .priority = 10
1375     };
1376
1377     hdev->iommu_listener = (MemoryListener) {
1378         .name = "vhost-iommu",
1379         .region_add = vhost_iommu_region_add,
1380         .region_del = vhost_iommu_region_del,
1381     };
1382
1383     if (hdev->migration_blocker == NULL) {
1384         if (!(hdev->features & (0x1ULL << VHOST_F_LOG_ALL))) {
1385             error_setg(&hdev->migration_blocker,
1386                        "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature.");
1387         } else if (vhost_dev_log_is_shared(hdev) && !qemu_memfd_alloc_check()) {
1388             error_setg(&hdev->migration_blocker,
1389                        "Migration disabled: failed to allocate shared memory");
1390         }
1391     }
1392
1393     if (hdev->migration_blocker != NULL) {
1394         r = migrate_add_blocker(hdev->migration_blocker, errp);
1395         if (r < 0) {
1396             error_free(hdev->migration_blocker);
1397             goto fail_busyloop;
1398         }
1399     }
1400
1401     hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
1402     hdev->n_mem_sections = 0;
1403     hdev->mem_sections = NULL;
1404     hdev->log = NULL;
1405     hdev->log_size = 0;
1406     hdev->log_enabled = false;
1407     hdev->started = false;
1408     memory_listener_register(&hdev->memory_listener, &address_space_memory);
1409     QLIST_INSERT_HEAD(&vhost_devices, hdev, entry);
1410
1411     if (used_memslots > hdev->vhost_ops->vhost_backend_memslots_limit(hdev)) {
1412         error_setg(errp, "vhost backend memory slots limit is less"
1413                    " than current number of present memory slots");
1414         r = -EINVAL;
1415         goto fail_busyloop;
1416     }
1417
1418     return 0;
1419
1420 fail_busyloop:
1421     if (busyloop_timeout) {
1422         while (--i >= 0) {
1423             vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i, 0);
1424         }
1425     }
1426 fail:
1427     hdev->nvqs = n_initialized_vqs;
1428     vhost_dev_cleanup(hdev);
1429     return r;
1430 }
1431
1432 void vhost_dev_cleanup(struct vhost_dev *hdev)
1433 {
1434     int i;
1435
1436     for (i = 0; i < hdev->nvqs; ++i) {
1437         vhost_virtqueue_cleanup(hdev->vqs + i);
1438     }
1439     if (hdev->mem) {
1440         /* those are only safe after successful init */
1441         memory_listener_unregister(&hdev->memory_listener);
1442         QLIST_REMOVE(hdev, entry);
1443     }
1444     if (hdev->migration_blocker) {
1445         migrate_del_blocker(hdev->migration_blocker);
1446         error_free(hdev->migration_blocker);
1447     }
1448     g_free(hdev->mem);
1449     g_free(hdev->mem_sections);
1450     if (hdev->vhost_ops) {
1451         hdev->vhost_ops->vhost_backend_cleanup(hdev);
1452     }
1453     assert(!hdev->log);
1454
1455     memset(hdev, 0, sizeof(struct vhost_dev));
1456 }
1457
1458 /* Stop processing guest IO notifications in qemu.
1459  * Start processing them in vhost in kernel.
1460  */
1461 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
1462 {
1463     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
1464     int i, r, e;
1465
1466     /* We will pass the notifiers to the kernel, make sure that QEMU
1467      * doesn't interfere.
1468      */
1469     r = virtio_device_grab_ioeventfd(vdev);
1470     if (r < 0) {
1471         error_report("binding does not support host notifiers");
1472         goto fail;
1473     }
1474
1475     for (i = 0; i < hdev->nvqs; ++i) {
1476         r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
1477                                          true);
1478         if (r < 0) {
1479             error_report("vhost VQ %d notifier binding failed: %d", i, -r);
1480             goto fail_vq;
1481         }
1482     }
1483
1484     return 0;
1485 fail_vq:
1486     while (--i >= 0) {
1487         e = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
1488                                          false);
1489         if (e < 0) {
1490             error_report("vhost VQ %d notifier cleanup error: %d", i, -r);
1491         }
1492         assert (e >= 0);
1493         virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i);
1494     }
1495     virtio_device_release_ioeventfd(vdev);
1496 fail:
1497     return r;
1498 }
1499
1500 /* Stop processing guest IO notifications in vhost.
1501  * Start processing them in qemu.
1502  * This might actually run the qemu handlers right away,
1503  * so virtio in qemu must be completely setup when this is called.
1504  */
1505 void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
1506 {
1507     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
1508     int i, r;
1509
1510     for (i = 0; i < hdev->nvqs; ++i) {
1511         r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
1512                                          false);
1513         if (r < 0) {
1514             error_report("vhost VQ %d notifier cleanup failed: %d", i, -r);
1515         }
1516         assert (r >= 0);
1517         virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i);
1518     }
1519     virtio_device_release_ioeventfd(vdev);
1520 }
1521
1522 /* Test and clear event pending status.
1523  * Should be called after unmask to avoid losing events.
1524  */
1525 bool vhost_virtqueue_pending(struct vhost_dev *hdev, int n)
1526 {
1527     struct vhost_virtqueue *vq = hdev->vqs + n - hdev->vq_index;
1528     assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
1529     return event_notifier_test_and_clear(&vq->masked_notifier);
1530 }
1531
1532 /* Mask/unmask events from this vq. */
1533 void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
1534                          bool mask)
1535 {
1536     struct VirtQueue *vvq = virtio_get_queue(vdev, n);
1537     int r, index = n - hdev->vq_index;
1538     struct vhost_vring_file file;
1539
1540     /* should only be called after backend is connected */
1541     assert(hdev->vhost_ops);
1542
1543     if (mask) {
1544         assert(vdev->use_guest_notifier_mask);
1545         file.fd = event_notifier_get_fd(&hdev->vqs[index].masked_notifier);
1546     } else {
1547         file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
1548     }
1549
1550     file.index = hdev->vhost_ops->vhost_get_vq_index(hdev, n);
1551     r = hdev->vhost_ops->vhost_set_vring_call(hdev, &file);
1552     if (r < 0) {
1553         VHOST_OPS_DEBUG(r, "vhost_set_vring_call failed");
1554     }
1555 }
1556
1557 uint64_t vhost_get_features(struct vhost_dev *hdev, const int *feature_bits,
1558                             uint64_t features)
1559 {
1560     const int *bit = feature_bits;
1561     while (*bit != VHOST_INVALID_FEATURE_BIT) {
1562         uint64_t bit_mask = (1ULL << *bit);
1563         if (!(hdev->features & bit_mask)) {
1564             features &= ~bit_mask;
1565         }
1566         bit++;
1567     }
1568     return features;
1569 }
1570
1571 void vhost_ack_features(struct vhost_dev *hdev, const int *feature_bits,
1572                         uint64_t features)
1573 {
1574     const int *bit = feature_bits;
1575     while (*bit != VHOST_INVALID_FEATURE_BIT) {
1576         uint64_t bit_mask = (1ULL << *bit);
1577         if (features & bit_mask) {
1578             hdev->acked_features |= bit_mask;
1579         }
1580         bit++;
1581     }
1582 }
1583
1584 int vhost_dev_get_config(struct vhost_dev *hdev, uint8_t *config,
1585                          uint32_t config_len, Error **errp)
1586 {
1587     assert(hdev->vhost_ops);
1588
1589     if (hdev->vhost_ops->vhost_get_config) {
1590         return hdev->vhost_ops->vhost_get_config(hdev, config, config_len,
1591                                                  errp);
1592     }
1593
1594     error_setg(errp, "vhost_get_config not implemented");
1595     return -ENOSYS;
1596 }
1597
1598 int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *data,
1599                          uint32_t offset, uint32_t size, uint32_t flags)
1600 {
1601     assert(hdev->vhost_ops);
1602
1603     if (hdev->vhost_ops->vhost_set_config) {
1604         return hdev->vhost_ops->vhost_set_config(hdev, data, offset,
1605                                                  size, flags);
1606     }
1607
1608     return -ENOSYS;
1609 }
1610
1611 void vhost_dev_set_config_notifier(struct vhost_dev *hdev,
1612                                    const VhostDevConfigOps *ops)
1613 {
1614     hdev->config_ops = ops;
1615 }
1616
1617 void vhost_dev_free_inflight(struct vhost_inflight *inflight)
1618 {
1619     if (inflight && inflight->addr) {
1620         qemu_memfd_free(inflight->addr, inflight->size, inflight->fd);
1621         inflight->addr = NULL;
1622         inflight->fd = -1;
1623     }
1624 }
1625
1626 static int vhost_dev_resize_inflight(struct vhost_inflight *inflight,
1627                                      uint64_t new_size)
1628 {
1629     Error *err = NULL;
1630     int fd = -1;
1631     void *addr = qemu_memfd_alloc("vhost-inflight", new_size,
1632                                   F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
1633                                   &fd, &err);
1634
1635     if (err) {
1636         error_report_err(err);
1637         return -ENOMEM;
1638     }
1639
1640     vhost_dev_free_inflight(inflight);
1641     inflight->offset = 0;
1642     inflight->addr = addr;
1643     inflight->fd = fd;
1644     inflight->size = new_size;
1645
1646     return 0;
1647 }
1648
1649 void vhost_dev_save_inflight(struct vhost_inflight *inflight, QEMUFile *f)
1650 {
1651     if (inflight->addr) {
1652         qemu_put_be64(f, inflight->size);
1653         qemu_put_be16(f, inflight->queue_size);
1654         qemu_put_buffer(f, inflight->addr, inflight->size);
1655     } else {
1656         qemu_put_be64(f, 0);
1657     }
1658 }
1659
1660 int vhost_dev_load_inflight(struct vhost_inflight *inflight, QEMUFile *f)
1661 {
1662     uint64_t size;
1663
1664     size = qemu_get_be64(f);
1665     if (!size) {
1666         return 0;
1667     }
1668
1669     if (inflight->size != size) {
1670         int ret = vhost_dev_resize_inflight(inflight, size);
1671         if (ret < 0) {
1672             return ret;
1673         }
1674     }
1675     inflight->queue_size = qemu_get_be16(f);
1676
1677     qemu_get_buffer(f, inflight->addr, size);
1678
1679     return 0;
1680 }
1681
1682 int vhost_dev_prepare_inflight(struct vhost_dev *hdev, VirtIODevice *vdev)
1683 {
1684     int r;
1685
1686     if (hdev->vhost_ops->vhost_get_inflight_fd == NULL ||
1687         hdev->vhost_ops->vhost_set_inflight_fd == NULL) {
1688         return 0;
1689     }
1690
1691     hdev->vdev = vdev;
1692
1693     r = vhost_dev_set_features(hdev, hdev->log_enabled);
1694     if (r < 0) {
1695         VHOST_OPS_DEBUG(r, "vhost_dev_prepare_inflight failed");
1696         return r;
1697     }
1698
1699     return 0;
1700 }
1701
1702 int vhost_dev_set_inflight(struct vhost_dev *dev,
1703                            struct vhost_inflight *inflight)
1704 {
1705     int r;
1706
1707     if (dev->vhost_ops->vhost_set_inflight_fd && inflight->addr) {
1708         r = dev->vhost_ops->vhost_set_inflight_fd(dev, inflight);
1709         if (r) {
1710             VHOST_OPS_DEBUG(r, "vhost_set_inflight_fd failed");
1711             return r;
1712         }
1713     }
1714
1715     return 0;
1716 }
1717
1718 int vhost_dev_get_inflight(struct vhost_dev *dev, uint16_t queue_size,
1719                            struct vhost_inflight *inflight)
1720 {
1721     int r;
1722
1723     if (dev->vhost_ops->vhost_get_inflight_fd) {
1724         r = dev->vhost_ops->vhost_get_inflight_fd(dev, queue_size, inflight);
1725         if (r) {
1726             VHOST_OPS_DEBUG(r, "vhost_get_inflight_fd failed");
1727             return r;
1728         }
1729     }
1730
1731     return 0;
1732 }
1733
1734 /* Host notifiers must be enabled at this point. */
1735 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
1736 {
1737     int i, r;
1738
1739     /* should only be called after backend is connected */
1740     assert(hdev->vhost_ops);
1741
1742     hdev->started = true;
1743     hdev->vdev = vdev;
1744
1745     r = vhost_dev_set_features(hdev, hdev->log_enabled);
1746     if (r < 0) {
1747         goto fail_features;
1748     }
1749
1750     if (vhost_dev_has_iommu(hdev)) {
1751         memory_listener_register(&hdev->iommu_listener, vdev->dma_as);
1752     }
1753
1754     r = hdev->vhost_ops->vhost_set_mem_table(hdev, hdev->mem);
1755     if (r < 0) {
1756         VHOST_OPS_DEBUG(r, "vhost_set_mem_table failed");
1757         goto fail_mem;
1758     }
1759     for (i = 0; i < hdev->nvqs; ++i) {
1760         r = vhost_virtqueue_start(hdev,
1761                                   vdev,
1762                                   hdev->vqs + i,
1763                                   hdev->vq_index + i);
1764         if (r < 0) {
1765             goto fail_vq;
1766         }
1767     }
1768
1769     if (hdev->log_enabled) {
1770         uint64_t log_base;
1771
1772         hdev->log_size = vhost_get_log_size(hdev);
1773         hdev->log = vhost_log_get(hdev->log_size,
1774                                   vhost_dev_log_is_shared(hdev));
1775         log_base = (uintptr_t)hdev->log->log;
1776         r = hdev->vhost_ops->vhost_set_log_base(hdev,
1777                                                 hdev->log_size ? log_base : 0,
1778                                                 hdev->log);
1779         if (r < 0) {
1780             VHOST_OPS_DEBUG(r, "vhost_set_log_base failed");
1781             goto fail_log;
1782         }
1783     }
1784     if (hdev->vhost_ops->vhost_dev_start) {
1785         r = hdev->vhost_ops->vhost_dev_start(hdev, true);
1786         if (r) {
1787             goto fail_log;
1788         }
1789     }
1790     if (vhost_dev_has_iommu(hdev) &&
1791         hdev->vhost_ops->vhost_set_iotlb_callback) {
1792             hdev->vhost_ops->vhost_set_iotlb_callback(hdev, true);
1793
1794         /* Update used ring information for IOTLB to work correctly,
1795          * vhost-kernel code requires for this.*/
1796         for (i = 0; i < hdev->nvqs; ++i) {
1797             struct vhost_virtqueue *vq = hdev->vqs + i;
1798             vhost_device_iotlb_miss(hdev, vq->used_phys, true);
1799         }
1800     }
1801     return 0;
1802 fail_log:
1803     vhost_log_put(hdev, false);
1804 fail_vq:
1805     while (--i >= 0) {
1806         vhost_virtqueue_stop(hdev,
1807                              vdev,
1808                              hdev->vqs + i,
1809                              hdev->vq_index + i);
1810     }
1811
1812 fail_mem:
1813 fail_features:
1814
1815     hdev->started = false;
1816     return r;
1817 }
1818
1819 /* Host notifiers must be enabled at this point. */
1820 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
1821 {
1822     int i;
1823
1824     /* should only be called after backend is connected */
1825     assert(hdev->vhost_ops);
1826
1827     if (hdev->vhost_ops->vhost_dev_start) {
1828         hdev->vhost_ops->vhost_dev_start(hdev, false);
1829     }
1830     for (i = 0; i < hdev->nvqs; ++i) {
1831         vhost_virtqueue_stop(hdev,
1832                              vdev,
1833                              hdev->vqs + i,
1834                              hdev->vq_index + i);
1835     }
1836
1837     if (vhost_dev_has_iommu(hdev)) {
1838         if (hdev->vhost_ops->vhost_set_iotlb_callback) {
1839             hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
1840         }
1841         memory_listener_unregister(&hdev->iommu_listener);
1842     }
1843     vhost_log_put(hdev, true);
1844     hdev->started = false;
1845     hdev->vdev = NULL;
1846 }
1847
1848 int vhost_net_set_backend(struct vhost_dev *hdev,
1849                           struct vhost_vring_file *file)
1850 {
1851     if (hdev->vhost_ops->vhost_net_set_backend) {
1852         return hdev->vhost_ops->vhost_net_set_backend(hdev, file);
1853     }
1854
1855     return -ENOSYS;
1856 }