block/nvme.c

   1 /*
   2  * NVMe block driver based on vfio
   3  *
   4  * Copyright 2016 - 2018 Red Hat, Inc.
   5  *
   6  * Authors:
   7  *   Fam Zheng <[email protected]>
   8  *   Paolo Bonzini <[email protected]>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  */
  13
  14 #include "qemu/osdep.h"
  15 #include <linux/vfio.h>
  16 #include "qapi/error.h"
  17 #include "qapi/qmp/qdict.h"
  18 #include "qapi/qmp/qstring.h"
  19 #include "qemu/error-report.h"
  20 #include "qemu/main-loop.h"
  21 #include "qemu/module.h"
  22 #include "qemu/cutils.h"
  23 #include "qemu/option.h"
  24 #include "qemu/vfio-helpers.h"
  25 #include "block/block_int.h"
  26 #include "sysemu/replay.h"
  27 #include "trace.h"
  28
  29 #include "block/nvme.h"
  30
  31 #define NVME_SQ_ENTRY_BYTES 64
  32 #define NVME_CQ_ENTRY_BYTES 16
  33 #define NVME_QUEUE_SIZE 128
  34 #define NVME_BAR_SIZE 8192
  35
  36 typedef struct {
  37     int32_t  head, tail;
  38     uint8_t  *queue;
  39     uint64_t iova;
  40     /* Hardware MMIO register */
  41     volatile uint32_t *doorbell;
  42 } NVMeQueue;
  43
  44 typedef struct {
  45     BlockCompletionFunc *cb;
  46     void *opaque;
  47     int cid;
  48     void *prp_list_page;
  49     uint64_t prp_list_iova;
  50     bool busy;
  51 } NVMeRequest;
  52
  53 typedef struct {
  54     CoQueue     free_req_queue;
  55     QemuMutex   lock;
  56
  57     /* Fields protected by BQL */
  58     int         index;
  59     uint8_t     *prp_list_pages;
  60
  61     /* Fields protected by @lock */
  62     NVMeQueue   sq, cq;
  63     int         cq_phase;
  64     NVMeRequest reqs[NVME_QUEUE_SIZE];
  65     bool        busy;
  66     int         need_kick;
  67     int         inflight;
  68 } NVMeQueuePair;
  69
  70 /* Memory mapped registers */
  71 typedef volatile struct {
  72     uint64_t cap;
  73     uint32_t vs;
  74     uint32_t intms;
  75     uint32_t intmc;
  76     uint32_t cc;
  77     uint32_t reserved0;
  78     uint32_t csts;
  79     uint32_t nssr;
  80     uint32_t aqa;
  81     uint64_t asq;
  82     uint64_t acq;
  83     uint32_t cmbloc;
  84     uint32_t cmbsz;
  85     uint8_t  reserved1[0xec0];
  86     uint8_t  cmd_set_specfic[0x100];
  87     uint32_t doorbells[];
  88 } NVMeRegs;
  89
  90 QEMU_BUILD_BUG_ON(offsetof(NVMeRegs, doorbells) != 0x1000);
  91
  92 typedef struct {
  93     AioContext *aio_context;
  94     QEMUVFIOState *vfio;
  95     NVMeRegs *regs;
  96     /* The submission/completion queue pairs.
  97      * [0]: admin queue.
  98      * [1..]: io queues.
  99      */
 100     NVMeQueuePair **queues;
 101     int nr_queues;
 102     size_t page_size;
 103     /* How many uint32_t elements does each doorbell entry take. */
 104     size_t doorbell_scale;
 105     bool write_cache_supported;
 106     EventNotifier irq_notifier;
 107
 108     uint64_t nsze; /* Namespace size reported by identify command */
 109     int nsid;      /* The namespace id to read/write data. */
 110     int blkshift;
 111
 112     uint64_t max_transfer;
 113     bool plugged;
 114
 115     bool supports_write_zeroes;
 116     bool supports_discard;
 117
 118     CoMutex dma_map_lock;
 119     CoQueue dma_flush_queue;
 120
 121     /* Total size of mapped qiov, accessed under dma_map_lock */
 122     int dma_map_count;
 123
 124     /* PCI address (required for nvme_refresh_filename()) */
 125     char *device;
 126 } BDRVNVMeState;
 127
 128 #define NVME_BLOCK_OPT_DEVICE "device"
 129 #define NVME_BLOCK_OPT_NAMESPACE "namespace"
 130
 131 static QemuOptsList runtime_opts = {
 132     .name = "nvme",
 133     .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
 134     .desc = {
 135         {
 136             .name = NVME_BLOCK_OPT_DEVICE,
 137             .type = QEMU_OPT_STRING,
 138             .help = "NVMe PCI device address",
 139         },
 140         {
 141             .name = NVME_BLOCK_OPT_NAMESPACE,
 142             .type = QEMU_OPT_NUMBER,
 143             .help = "NVMe namespace",
 144         },
 145         { /* end of list */ }
 146     },
 147 };
 148
 149 static void nvme_init_queue(BlockDriverState *bs, NVMeQueue *q,
 150                             int nentries, int entry_bytes, Error **errp)
 151 {
 152     BDRVNVMeState *s = bs->opaque;
 153     size_t bytes;
 154     int r;
 155
 156     bytes = ROUND_UP(nentries * entry_bytes, s->page_size);
 157     q->head = q->tail = 0;
 158     q->queue = qemu_try_blockalign0(bs, bytes);
 159
 160     if (!q->queue) {
 161         error_setg(errp, "Cannot allocate queue");
 162         return;
 163     }
 164     r = qemu_vfio_dma_map(s->vfio, q->queue, bytes, false, &q->iova);
 165     if (r) {
 166         error_setg(errp, "Cannot map queue");
 167     }
 168 }
 169
 170 static void nvme_free_queue_pair(BlockDriverState *bs, NVMeQueuePair *q)
 171 {
 172     qemu_vfree(q->prp_list_pages);
 173     qemu_vfree(q->sq.queue);
 174     qemu_vfree(q->cq.queue);
 175     qemu_mutex_destroy(&q->lock);
 176     g_free(q);
 177 }
 178
 179 static void nvme_free_req_queue_cb(void *opaque)
 180 {
 181     NVMeQueuePair *q = opaque;
 182
 183     qemu_mutex_lock(&q->lock);
 184     while (qemu_co_enter_next(&q->free_req_queue, &q->lock)) {
 185         /* Retry all pending requests */
 186     }
 187     qemu_mutex_unlock(&q->lock);
 188 }
 189
 190 static NVMeQueuePair *nvme_create_queue_pair(BlockDriverState *bs,
 191                                              int idx, int size,
 192                                              Error **errp)
 193 {
 194     int i, r;
 195     BDRVNVMeState *s = bs->opaque;
 196     Error *local_err = NULL;
 197     NVMeQueuePair *q = g_new0(NVMeQueuePair, 1);
 198     uint64_t prp_list_iova;
 199
 200     qemu_mutex_init(&q->lock);
 201     q->index = idx;
 202     qemu_co_queue_init(&q->free_req_queue);
 203     q->prp_list_pages = qemu_blockalign0(bs, s->page_size * NVME_QUEUE_SIZE);
 204     r = qemu_vfio_dma_map(s->vfio, q->prp_list_pages,
 205                           s->page_size * NVME_QUEUE_SIZE,
 206                           false, &prp_list_iova);
 207     if (r) {
 208         goto fail;
 209     }
 210     for (i = 0; i < NVME_QUEUE_SIZE; i++) {
 211         NVMeRequest *req = &q->reqs[i];
 212         req->cid = i + 1;
 213         req->prp_list_page = q->prp_list_pages + i * s->page_size;
 214         req->prp_list_iova = prp_list_iova + i * s->page_size;
 215     }
 216     nvme_init_queue(bs, &q->sq, size, NVME_SQ_ENTRY_BYTES, &local_err);
 217     if (local_err) {
 218         error_propagate(errp, local_err);
 219         goto fail;
 220     }
 221     q->sq.doorbell = &s->regs->doorbells[idx * 2 * s->doorbell_scale];
 222
 223     nvme_init_queue(bs, &q->cq, size, NVME_CQ_ENTRY_BYTES, &local_err);
 224     if (local_err) {
 225         error_propagate(errp, local_err);
 226         goto fail;
 227     }
 228     q->cq.doorbell = &s->regs->doorbells[(idx * 2 + 1) * s->doorbell_scale];
 229
 230     return q;
 231 fail:
 232     nvme_free_queue_pair(bs, q);
 233     return NULL;
 234 }
 235
 236 /* With q->lock */
 237 static void nvme_kick(BDRVNVMeState *s, NVMeQueuePair *q)
 238 {
 239     if (s->plugged || !q->need_kick) {
 240         return;
 241     }
 242     trace_nvme_kick(s, q->index);
 243     assert(!(q->sq.tail & 0xFF00));
 244     /* Fence the write to submission queue entry before notifying the device. */
 245     smp_wmb();
 246     *q->sq.doorbell = cpu_to_le32(q->sq.tail);
 247     q->inflight += q->need_kick;
 248     q->need_kick = 0;
 249 }
 250
 251 /* Find a free request element if any, otherwise:
 252  * a) if in coroutine context, try to wait for one to become available;
 253  * b) if not in coroutine, return NULL;
 254  */
 255 static NVMeRequest *nvme_get_free_req(NVMeQueuePair *q)
 256 {
 257     int i;
 258     NVMeRequest *req = NULL;
 259
 260     qemu_mutex_lock(&q->lock);
 261     while (q->inflight + q->need_kick > NVME_QUEUE_SIZE - 2) {
 262         /* We have to leave one slot empty as that is the full queue case (head
 263          * == tail + 1). */
 264         if (qemu_in_coroutine()) {
 265             trace_nvme_free_req_queue_wait(q);
 266             qemu_co_queue_wait(&q->free_req_queue, &q->lock);
 267         } else {
 268             qemu_mutex_unlock(&q->lock);
 269             return NULL;
 270         }
 271     }
 272     for (i = 0; i < NVME_QUEUE_SIZE; i++) {
 273         if (!q->reqs[i].busy) {
 274             q->reqs[i].busy = true;
 275             req = &q->reqs[i];
 276             break;
 277         }
 278     }
 279     /* We have checked inflight and need_kick while holding q->lock, so one
 280      * free req must be available. */
 281     assert(req);
 282     qemu_mutex_unlock(&q->lock);
 283     return req;
 284 }
 285
 286 static inline int nvme_translate_error(const NvmeCqe *c)
 287 {
 288     uint16_t status = (le16_to_cpu(c->status) >> 1) & 0xFF;
 289     if (status) {
 290         trace_nvme_error(le32_to_cpu(c->result),
 291                          le16_to_cpu(c->sq_head),
 292                          le16_to_cpu(c->sq_id),
 293                          le16_to_cpu(c->cid),
 294                          le16_to_cpu(status));
 295     }
 296     switch (status) {
 297     case 0:
 298         return 0;
 299     case 1:
 300         return -ENOSYS;
 301     case 2:
 302         return -EINVAL;
 303     default:
 304         return -EIO;
 305     }
 306 }
 307
 308 /* With q->lock */
 309 static bool nvme_process_completion(BDRVNVMeState *s, NVMeQueuePair *q)
 310 {
 311     bool progress = false;
 312     NVMeRequest *preq;
 313     NVMeRequest req;
 314     NvmeCqe *c;
 315
 316     trace_nvme_process_completion(s, q->index, q->inflight);
 317     if (q->busy || s->plugged) {
 318         trace_nvme_process_completion_queue_busy(s, q->index);
 319         return false;
 320     }
 321     q->busy = true;
 322     assert(q->inflight >= 0);
 323     while (q->inflight) {
 324         int16_t cid;
 325         c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES];
 326         if ((le16_to_cpu(c->status) & 0x1) == q->cq_phase) {
 327             break;
 328         }
 329         q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE;
 330         if (!q->cq.head) {
 331             q->cq_phase = !q->cq_phase;
 332         }
 333         cid = le16_to_cpu(c->cid);
 334         if (cid == 0 || cid > NVME_QUEUE_SIZE) {
 335             fprintf(stderr, "Unexpected CID in completion queue: %" PRIu32 "\n",
 336                     cid);
 337             continue;
 338         }
 339         assert(cid <= NVME_QUEUE_SIZE);
 340         trace_nvme_complete_command(s, q->index, cid);
 341         preq = &q->reqs[cid - 1];
 342         req = *preq;
 343         assert(req.cid == cid);
 344         assert(req.cb);
 345         preq->busy = false;
 346         preq->cb = preq->opaque = NULL;
 347         qemu_mutex_unlock(&q->lock);
 348         req.cb(req.opaque, nvme_translate_error(c));
 349         qemu_mutex_lock(&q->lock);
 350         q->inflight--;
 351         progress = true;
 352     }
 353     if (progress) {
 354         /* Notify the device so it can post more completions. */
 355         smp_mb_release();
 356         *q->cq.doorbell = cpu_to_le32(q->cq.head);
 357         if (!qemu_co_queue_empty(&q->free_req_queue)) {
 358             replay_bh_schedule_oneshot_event(s->aio_context,
 359                                              nvme_free_req_queue_cb, q);
 360         }
 361     }
 362     q->busy = false;
 363     return progress;
 364 }
 365
 366 static void nvme_trace_command(const NvmeCmd *cmd)
 367 {
 368     int i;
 369
 370     for (i = 0; i < 8; ++i) {
 371         uint8_t *cmdp = (uint8_t *)cmd + i * 8;
 372         trace_nvme_submit_command_raw(cmdp[0], cmdp[1], cmdp[2], cmdp[3],
 373                                       cmdp[4], cmdp[5], cmdp[6], cmdp[7]);
 374     }
 375 }
 376
 377 static void nvme_submit_command(BDRVNVMeState *s, NVMeQueuePair *q,
 378                                 NVMeRequest *req,
 379                                 NvmeCmd *cmd, BlockCompletionFunc cb,
 380                                 void *opaque)
 381 {
 382     assert(!req->cb);
 383     req->cb = cb;
 384     req->opaque = opaque;
 385     cmd->cid = cpu_to_le32(req->cid);
 386
 387     trace_nvme_submit_command(s, q->index, req->cid);
 388     nvme_trace_command(cmd);
 389     qemu_mutex_lock(&q->lock);
 390     memcpy((uint8_t *)q->sq.queue +
 391            q->sq.tail * NVME_SQ_ENTRY_BYTES, cmd, sizeof(*cmd));
 392     q->sq.tail = (q->sq.tail + 1) % NVME_QUEUE_SIZE;
 393     q->need_kick++;
 394     nvme_kick(s, q);
 395     nvme_process_completion(s, q);
 396     qemu_mutex_unlock(&q->lock);
 397 }
 398
 399 static void nvme_cmd_sync_cb(void *opaque, int ret)
 400 {
 401     int *pret = opaque;
 402     *pret = ret;
 403     aio_wait_kick();
 404 }
 405
 406 static int nvme_cmd_sync(BlockDriverState *bs, NVMeQueuePair *q,
 407                          NvmeCmd *cmd)
 408 {
 409     NVMeRequest *req;
 410     BDRVNVMeState *s = bs->opaque;
 411     int ret = -EINPROGRESS;
 412     req = nvme_get_free_req(q);
 413     if (!req) {
 414         return -EBUSY;
 415     }
 416     nvme_submit_command(s, q, req, cmd, nvme_cmd_sync_cb, &ret);
 417
 418     BDRV_POLL_WHILE(bs, ret == -EINPROGRESS);
 419     return ret;
 420 }
 421
 422 static void nvme_identify(BlockDriverState *bs, int namespace, Error **errp)
 423 {
 424     BDRVNVMeState *s = bs->opaque;
 425     NvmeIdCtrl *idctrl;
 426     NvmeIdNs *idns;
 427     NvmeLBAF *lbaf;
 428     uint8_t *resp;
 429     uint16_t oncs;
 430     int r;
 431     uint64_t iova;
 432     NvmeCmd cmd = {
 433         .opcode = NVME_ADM_CMD_IDENTIFY,
 434         .cdw10 = cpu_to_le32(0x1),
 435     };
 436
 437     resp = qemu_try_blockalign0(bs, sizeof(NvmeIdCtrl));
 438     if (!resp) {
 439         error_setg(errp, "Cannot allocate buffer for identify response");
 440         goto out;
 441     }
 442     idctrl = (NvmeIdCtrl *)resp;
 443     idns = (NvmeIdNs *)resp;
 444     r = qemu_vfio_dma_map(s->vfio, resp, sizeof(NvmeIdCtrl), true, &iova);
 445     if (r) {
 446         error_setg(errp, "Cannot map buffer for DMA");
 447         goto out;
 448     }
 449     cmd.prp1 = cpu_to_le64(iova);
 450
 451     if (nvme_cmd_sync(bs, s->queues[0], &cmd)) {
 452         error_setg(errp, "Failed to identify controller");
 453         goto out;
 454     }
 455
 456     if (le32_to_cpu(idctrl->nn) < namespace) {
 457         error_setg(errp, "Invalid namespace");
 458         goto out;
 459     }
 460     s->write_cache_supported = le32_to_cpu(idctrl->vwc) & 0x1;
 461     s->max_transfer = (idctrl->mdts ? 1 << idctrl->mdts : 0) * s->page_size;
 462     /* For now the page list buffer per command is one page, to hold at most
 463      * s->page_size / sizeof(uint64_t) entries. */
 464     s->max_transfer = MIN_NON_ZERO(s->max_transfer,
 465                           s->page_size / sizeof(uint64_t) * s->page_size);
 466
 467     oncs = le16_to_cpu(idctrl->oncs);
 468     s->supports_write_zeroes = !!(oncs & NVME_ONCS_WRITE_ZEROS);
 469     s->supports_discard = !!(oncs & NVME_ONCS_DSM);
 470
 471     memset(resp, 0, 4096);
 472
 473     cmd.cdw10 = 0;
 474     cmd.nsid = cpu_to_le32(namespace);
 475     if (nvme_cmd_sync(bs, s->queues[0], &cmd)) {
 476         error_setg(errp, "Failed to identify namespace");
 477         goto out;
 478     }
 479
 480     s->nsze = le64_to_cpu(idns->nsze);
 481     lbaf = &idns->lbaf[NVME_ID_NS_FLBAS_INDEX(idns->flbas)];
 482
 483     if (NVME_ID_NS_DLFEAT_WRITE_ZEROES(idns->dlfeat) &&
 484             NVME_ID_NS_DLFEAT_READ_BEHAVIOR(idns->dlfeat) ==
 485                     NVME_ID_NS_DLFEAT_READ_BEHAVIOR_ZEROES) {
 486         bs->supported_write_flags |= BDRV_REQ_MAY_UNMAP;
 487     }
 488
 489     if (lbaf->ms) {
 490         error_setg(errp, "Namespaces with metadata are not yet supported");
 491         goto out;
 492     }
 493
 494     if (lbaf->ds < BDRV_SECTOR_BITS || lbaf->ds > 12 ||
 495         (1 << lbaf->ds) > s->page_size)
 496     {
 497         error_setg(errp, "Namespace has unsupported block size (2^%d)",
 498                    lbaf->ds);
 499         goto out;
 500     }
 501
 502     s->blkshift = lbaf->ds;
 503 out:
 504     qemu_vfio_dma_unmap(s->vfio, resp);
 505     qemu_vfree(resp);
 506 }
 507
 508 static bool nvme_poll_queues(BDRVNVMeState *s)
 509 {
 510     bool progress = false;
 511     int i;
 512
 513     for (i = 0; i < s->nr_queues; i++) {
 514         NVMeQueuePair *q = s->queues[i];
 515         qemu_mutex_lock(&q->lock);
 516         while (nvme_process_completion(s, q)) {
 517             /* Keep polling */
 518             progress = true;
 519         }
 520         qemu_mutex_unlock(&q->lock);
 521     }
 522     return progress;
 523 }
 524
 525 static void nvme_handle_event(EventNotifier *n)
 526 {
 527     BDRVNVMeState *s = container_of(n, BDRVNVMeState, irq_notifier);
 528
 529     trace_nvme_handle_event(s);
 530     event_notifier_test_and_clear(n);
 531     nvme_poll_queues(s);
 532 }
 533
 534 static bool nvme_add_io_queue(BlockDriverState *bs, Error **errp)
 535 {
 536     BDRVNVMeState *s = bs->opaque;
 537     int n = s->nr_queues;
 538     NVMeQueuePair *q;
 539     NvmeCmd cmd;
 540     int queue_size = NVME_QUEUE_SIZE;
 541
 542     q = nvme_create_queue_pair(bs, n, queue_size, errp);
 543     if (!q) {
 544         return false;
 545     }
 546     cmd = (NvmeCmd) {
 547         .opcode = NVME_ADM_CMD_CREATE_CQ,
 548         .prp1 = cpu_to_le64(q->cq.iova),
 549         .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
 550         .cdw11 = cpu_to_le32(0x3),
 551     };
 552     if (nvme_cmd_sync(bs, s->queues[0], &cmd)) {
 553         error_setg(errp, "Failed to create io queue [%d]", n);
 554         nvme_free_queue_pair(bs, q);
 555         return false;
 556     }
 557     cmd = (NvmeCmd) {
 558         .opcode = NVME_ADM_CMD_CREATE_SQ,
 559         .prp1 = cpu_to_le64(q->sq.iova),
 560         .cdw10 = cpu_to_le32(((queue_size - 1) << 16) | (n & 0xFFFF)),
 561         .cdw11 = cpu_to_le32(0x1 | (n << 16)),
 562     };
 563     if (nvme_cmd_sync(bs, s->queues[0], &cmd)) {
 564         error_setg(errp, "Failed to create io queue [%d]", n);
 565         nvme_free_queue_pair(bs, q);
 566         return false;
 567     }
 568     s->queues = g_renew(NVMeQueuePair *, s->queues, n + 1);
 569     s->queues[n] = q;
 570     s->nr_queues++;
 571     return true;
 572 }
 573
 574 static bool nvme_poll_cb(void *opaque)
 575 {
 576     EventNotifier *e = opaque;
 577     BDRVNVMeState *s = container_of(e, BDRVNVMeState, irq_notifier);
 578
 579     trace_nvme_poll_cb(s);
 580     return nvme_poll_queues(s);
 581 }
 582
 583 static int nvme_init(BlockDriverState *bs, const char *device, int namespace,
 584                      Error **errp)
 585 {
 586     BDRVNVMeState *s = bs->opaque;
 587     int ret;
 588     uint64_t cap;
 589     uint64_t timeout_ms;
 590     uint64_t deadline, now;
 591     Error *local_err = NULL;
 592
 593     qemu_co_mutex_init(&s->dma_map_lock);
 594     qemu_co_queue_init(&s->dma_flush_queue);
 595     s->device = g_strdup(device);
 596     s->nsid = namespace;
 597     s->aio_context = bdrv_get_aio_context(bs);
 598     ret = event_notifier_init(&s->irq_notifier, 0);
 599     if (ret) {
 600         error_setg(errp, "Failed to init event notifier");
 601         return ret;
 602     }
 603
 604     s->vfio = qemu_vfio_open_pci(device, errp);
 605     if (!s->vfio) {
 606         ret = -EINVAL;
 607         goto out;
 608     }
 609
 610     s->regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, NVME_BAR_SIZE, errp);
 611     if (!s->regs) {
 612         ret = -EINVAL;
 613         goto out;
 614     }
 615
 616     /* Perform initialize sequence as described in NVMe spec "7.6.1
 617      * Initialization". */
 618
 619     cap = le64_to_cpu(s->regs->cap);
 620     if (!(cap & (1ULL << 37))) {
 621         error_setg(errp, "Device doesn't support NVMe command set");
 622         ret = -EINVAL;
 623         goto out;
 624     }
 625
 626     s->page_size = MAX(4096, 1 << (12 + ((cap >> 48) & 0xF)));
 627     s->doorbell_scale = (4 << (((cap >> 32) & 0xF))) / sizeof(uint32_t);
 628     bs->bl.opt_mem_alignment = s->page_size;
 629     timeout_ms = MIN(500 * ((cap >> 24) & 0xFF), 30000);
 630
 631     /* Reset device to get a clean state. */
 632     s->regs->cc = cpu_to_le32(le32_to_cpu(s->regs->cc) & 0xFE);
 633     /* Wait for CSTS.RDY = 0. */
 634     deadline = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + timeout_ms * 1000000ULL;
 635     while (le32_to_cpu(s->regs->csts) & 0x1) {
 636         if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
 637             error_setg(errp, "Timeout while waiting for device to reset (%"
 638                              PRId64 " ms)",
 639                        timeout_ms);
 640             ret = -ETIMEDOUT;
 641             goto out;
 642         }
 643     }
 644
 645     /* Set up admin queue. */
 646     s->queues = g_new(NVMeQueuePair *, 1);
 647     s->queues[0] = nvme_create_queue_pair(bs, 0, NVME_QUEUE_SIZE, errp);
 648     if (!s->queues[0]) {
 649         ret = -EINVAL;
 650         goto out;
 651     }
 652     s->nr_queues = 1;
 653     QEMU_BUILD_BUG_ON(NVME_QUEUE_SIZE & 0xF000);
 654     s->regs->aqa = cpu_to_le32((NVME_QUEUE_SIZE << 16) | NVME_QUEUE_SIZE);
 655     s->regs->asq = cpu_to_le64(s->queues[0]->sq.iova);
 656     s->regs->acq = cpu_to_le64(s->queues[0]->cq.iova);
 657
 658     /* After setting up all control registers we can enable device now. */
 659     s->regs->cc = cpu_to_le32((ctz32(NVME_CQ_ENTRY_BYTES) << 20) |
 660                               (ctz32(NVME_SQ_ENTRY_BYTES) << 16) |
 661                               0x1);
 662     /* Wait for CSTS.RDY = 1. */
 663     now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
 664     deadline = now + timeout_ms * 1000000;
 665     while (!(le32_to_cpu(s->regs->csts) & 0x1)) {
 666         if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) > deadline) {
 667             error_setg(errp, "Timeout while waiting for device to start (%"
 668                              PRId64 " ms)",
 669                        timeout_ms);
 670             ret = -ETIMEDOUT;
 671             goto out;
 672         }
 673     }
 674
 675     ret = qemu_vfio_pci_init_irq(s->vfio, &s->irq_notifier,
 676                                  VFIO_PCI_MSIX_IRQ_INDEX, errp);
 677     if (ret) {
 678         goto out;
 679     }
 680     aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
 681                            false, nvme_handle_event, nvme_poll_cb);
 682
 683     nvme_identify(bs, namespace, &local_err);
 684     if (local_err) {
 685         error_propagate(errp, local_err);
 686         ret = -EIO;
 687         goto out;
 688     }
 689
 690     /* Set up command queues. */
 691     if (!nvme_add_io_queue(bs, errp)) {
 692         ret = -EIO;
 693     }
 694 out:
 695     /* Cleaning up is done in nvme_file_open() upon error. */
 696     return ret;
 697 }
 698
 699 /* Parse a filename in the format of nvme://XXXX:XX:XX.X/X. Example:
 700  *
 701  *     nvme://0000:44:00.0/1
 702  *
 703  * where the "nvme://" is a fixed form of the protocol prefix, the middle part
 704  * is the PCI address, and the last part is the namespace number starting from
 705  * 1 according to the NVMe spec. */
 706 static void nvme_parse_filename(const char *filename, QDict *options,
 707                                 Error **errp)
 708 {
 709     int pref = strlen("nvme://");
 710
 711     if (strlen(filename) > pref && !strncmp(filename, "nvme://", pref)) {
 712         const char *tmp = filename + pref;
 713         char *device;
 714         const char *namespace;
 715         unsigned long ns;
 716         const char *slash = strchr(tmp, '/');
 717         if (!slash) {
 718             qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, tmp);
 719             return;
 720         }
 721         device = g_strndup(tmp, slash - tmp);
 722         qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, device);
 723         g_free(device);
 724         namespace = slash + 1;
 725         if (*namespace && qemu_strtoul(namespace, NULL, 10, &ns)) {
 726             error_setg(errp, "Invalid namespace '%s', positive number expected",
 727                        namespace);
 728             return;
 729         }
 730         qdict_put_str(options, NVME_BLOCK_OPT_NAMESPACE,
 731                       *namespace ? namespace : "1");
 732     }
 733 }
 734
 735 static int nvme_enable_disable_write_cache(BlockDriverState *bs, bool enable,
 736                                            Error **errp)
 737 {
 738     int ret;
 739     BDRVNVMeState *s = bs->opaque;
 740     NvmeCmd cmd = {
 741         .opcode = NVME_ADM_CMD_SET_FEATURES,
 742         .nsid = cpu_to_le32(s->nsid),
 743         .cdw10 = cpu_to_le32(0x06),
 744         .cdw11 = cpu_to_le32(enable ? 0x01 : 0x00),
 745     };
 746
 747     ret = nvme_cmd_sync(bs, s->queues[0], &cmd);
 748     if (ret) {
 749         error_setg(errp, "Failed to configure NVMe write cache");
 750     }
 751     return ret;
 752 }
 753
 754 static void nvme_close(BlockDriverState *bs)
 755 {
 756     int i;
 757     BDRVNVMeState *s = bs->opaque;
 758
 759     for (i = 0; i < s->nr_queues; ++i) {
 760         nvme_free_queue_pair(bs, s->queues[i]);
 761     }
 762     g_free(s->queues);
 763     aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
 764                            false, NULL, NULL);
 765     event_notifier_cleanup(&s->irq_notifier);
 766     qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->regs, 0, NVME_BAR_SIZE);
 767     qemu_vfio_close(s->vfio);
 768
 769     g_free(s->device);
 770 }
 771
 772 static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags,
 773                           Error **errp)
 774 {
 775     const char *device;
 776     QemuOpts *opts;
 777     int namespace;
 778     int ret;
 779     BDRVNVMeState *s = bs->opaque;
 780
 781     bs->supported_write_flags = BDRV_REQ_FUA;
 782
 783     opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
 784     qemu_opts_absorb_qdict(opts, options, &error_abort);
 785     device = qemu_opt_get(opts, NVME_BLOCK_OPT_DEVICE);
 786     if (!device) {
 787         error_setg(errp, "'" NVME_BLOCK_OPT_DEVICE "' option is required");
 788         qemu_opts_del(opts);
 789         return -EINVAL;
 790     }
 791
 792     namespace = qemu_opt_get_number(opts, NVME_BLOCK_OPT_NAMESPACE, 1);
 793     ret = nvme_init(bs, device, namespace, errp);
 794     qemu_opts_del(opts);
 795     if (ret) {
 796         goto fail;
 797     }
 798     if (flags & BDRV_O_NOCACHE) {
 799         if (!s->write_cache_supported) {
 800             error_setg(errp,
 801                        "NVMe controller doesn't support write cache configuration");
 802             ret = -EINVAL;
 803         } else {
 804             ret = nvme_enable_disable_write_cache(bs, !(flags & BDRV_O_NOCACHE),
 805                                                   errp);
 806         }
 807         if (ret) {
 808             goto fail;
 809         }
 810     }
 811     return 0;
 812 fail:
 813     nvme_close(bs);
 814     return ret;
 815 }
 816
 817 static int64_t nvme_getlength(BlockDriverState *bs)
 818 {
 819     BDRVNVMeState *s = bs->opaque;
 820     return s->nsze << s->blkshift;
 821 }
 822
 823 static uint32_t nvme_get_blocksize(BlockDriverState *bs)
 824 {
 825     BDRVNVMeState *s = bs->opaque;
 826     assert(s->blkshift >= BDRV_SECTOR_BITS && s->blkshift <= 12);
 827     return UINT32_C(1) << s->blkshift;
 828 }
 829
 830 static int nvme_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
 831 {
 832     uint32_t blocksize = nvme_get_blocksize(bs);
 833     bsz->phys = blocksize;
 834     bsz->log = blocksize;
 835     return 0;
 836 }
 837
 838 /* Called with s->dma_map_lock */
 839 static coroutine_fn int nvme_cmd_unmap_qiov(BlockDriverState *bs,
 840                                             QEMUIOVector *qiov)
 841 {
 842     int r = 0;
 843     BDRVNVMeState *s = bs->opaque;
 844
 845     s->dma_map_count -= qiov->size;
 846     if (!s->dma_map_count && !qemu_co_queue_empty(&s->dma_flush_queue)) {
 847         r = qemu_vfio_dma_reset_temporary(s->vfio);
 848         if (!r) {
 849             qemu_co_queue_restart_all(&s->dma_flush_queue);
 850         }
 851     }
 852     return r;
 853 }
 854
 855 /* Called with s->dma_map_lock */
 856 static coroutine_fn int nvme_cmd_map_qiov(BlockDriverState *bs, NvmeCmd *cmd,
 857                                           NVMeRequest *req, QEMUIOVector *qiov)
 858 {
 859     BDRVNVMeState *s = bs->opaque;
 860     uint64_t *pagelist = req->prp_list_page;
 861     int i, j, r;
 862     int entries = 0;
 863
 864     assert(qiov->size);
 865     assert(QEMU_IS_ALIGNED(qiov->size, s->page_size));
 866     assert(qiov->size / s->page_size <= s->page_size / sizeof(uint64_t));
 867     for (i = 0; i < qiov->niov; ++i) {
 868         bool retry = true;
 869         uint64_t iova;
 870 try_map:
 871         r = qemu_vfio_dma_map(s->vfio,
 872                               qiov->iov[i].iov_base,
 873                               qiov->iov[i].iov_len,
 874                               true, &iova);
 875         if (r == -ENOMEM && retry) {
 876             retry = false;
 877             trace_nvme_dma_flush_queue_wait(s);
 878             if (s->dma_map_count) {
 879                 trace_nvme_dma_map_flush(s);
 880                 qemu_co_queue_wait(&s->dma_flush_queue, &s->dma_map_lock);
 881             } else {
 882                 r = qemu_vfio_dma_reset_temporary(s->vfio);
 883                 if (r) {
 884                     goto fail;
 885                 }
 886             }
 887             goto try_map;
 888         }
 889         if (r) {
 890             goto fail;
 891         }
 892
 893         for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) {
 894             pagelist[entries++] = cpu_to_le64(iova + j * s->page_size);
 895         }
 896         trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base,
 897                                     qiov->iov[i].iov_len / s->page_size);
 898     }
 899
 900     s->dma_map_count += qiov->size;
 901
 902     assert(entries <= s->page_size / sizeof(uint64_t));
 903     switch (entries) {
 904     case 0:
 905         abort();
 906     case 1:
 907         cmd->prp1 = pagelist[0];
 908         cmd->prp2 = 0;
 909         break;
 910     case 2:
 911         cmd->prp1 = pagelist[0];
 912         cmd->prp2 = pagelist[1];
 913         break;
 914     default:
 915         cmd->prp1 = pagelist[0];
 916         cmd->prp2 = cpu_to_le64(req->prp_list_iova + sizeof(uint64_t));
 917         break;
 918     }
 919     trace_nvme_cmd_map_qiov(s, cmd, req, qiov, entries);
 920     for (i = 0; i < entries; ++i) {
 921         trace_nvme_cmd_map_qiov_pages(s, i, pagelist[i]);
 922     }
 923     return 0;
 924 fail:
 925     /* No need to unmap [0 - i) iovs even if we've failed, since we don't
 926      * increment s->dma_map_count. This is okay for fixed mapping memory areas
 927      * because they are already mapped before calling this function; for
 928      * temporary mappings, a later nvme_cmd_(un)map_qiov will reclaim by
 929      * calling qemu_vfio_dma_reset_temporary when necessary. */
 930     return r;
 931 }
 932
 933 typedef struct {
 934     Coroutine *co;
 935     int ret;
 936     AioContext *ctx;
 937 } NVMeCoData;
 938
 939 static void nvme_rw_cb_bh(void *opaque)
 940 {
 941     NVMeCoData *data = opaque;
 942     qemu_coroutine_enter(data->co);
 943 }
 944
 945 static void nvme_rw_cb(void *opaque, int ret)
 946 {
 947     NVMeCoData *data = opaque;
 948     data->ret = ret;
 949     if (!data->co) {
 950         /* The rw coroutine hasn't yielded, don't try to enter. */
 951         return;
 952     }
 953     replay_bh_schedule_oneshot_event(data->ctx, nvme_rw_cb_bh, data);
 954 }
 955
 956 static coroutine_fn int nvme_co_prw_aligned(BlockDriverState *bs,
 957                                             uint64_t offset, uint64_t bytes,
 958                                             QEMUIOVector *qiov,
 959                                             bool is_write,
 960                                             int flags)
 961 {
 962     int r;
 963     BDRVNVMeState *s = bs->opaque;
 964     NVMeQueuePair *ioq = s->queues[1];
 965     NVMeRequest *req;
 966
 967     uint32_t cdw12 = (((bytes >> s->blkshift) - 1) & 0xFFFF) |
 968                        (flags & BDRV_REQ_FUA ? 1 << 30 : 0);
 969     NvmeCmd cmd = {
 970         .opcode = is_write ? NVME_CMD_WRITE : NVME_CMD_READ,
 971         .nsid = cpu_to_le32(s->nsid),
 972         .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
 973         .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
 974         .cdw12 = cpu_to_le32(cdw12),
 975     };
 976     NVMeCoData data = {
 977         .ctx = bdrv_get_aio_context(bs),
 978         .ret = -EINPROGRESS,
 979     };
 980
 981     trace_nvme_prw_aligned(s, is_write, offset, bytes, flags, qiov->niov);
 982     assert(s->nr_queues > 1);
 983     req = nvme_get_free_req(ioq);
 984     assert(req);
 985
 986     qemu_co_mutex_lock(&s->dma_map_lock);
 987     r = nvme_cmd_map_qiov(bs, &cmd, req, qiov);
 988     qemu_co_mutex_unlock(&s->dma_map_lock);
 989     if (r) {
 990         req->busy = false;
 991         return r;
 992     }
 993     nvme_submit_command(s, ioq, req, &cmd, nvme_rw_cb, &data);
 994
 995     data.co = qemu_coroutine_self();
 996     while (data.ret == -EINPROGRESS) {
 997         qemu_coroutine_yield();
 998     }
 999
1000     qemu_co_mutex_lock(&s->dma_map_lock);
1001     r = nvme_cmd_unmap_qiov(bs, qiov);
1002     qemu_co_mutex_unlock(&s->dma_map_lock);
1003     if (r) {
1004         return r;
1005     }
1006
1007     trace_nvme_rw_done(s, is_write, offset, bytes, data.ret);
1008     return data.ret;
1009 }
1010
1011 static inline bool nvme_qiov_aligned(BlockDriverState *bs,
1012                                      const QEMUIOVector *qiov)
1013 {
1014     int i;
1015     BDRVNVMeState *s = bs->opaque;
1016
1017     for (i = 0; i < qiov->niov; ++i) {
1018         if (!QEMU_PTR_IS_ALIGNED(qiov->iov[i].iov_base, s->page_size) ||
1019             !QEMU_IS_ALIGNED(qiov->iov[i].iov_len, s->page_size)) {
1020             trace_nvme_qiov_unaligned(qiov, i, qiov->iov[i].iov_base,
1021                                       qiov->iov[i].iov_len, s->page_size);
1022             return false;
1023         }
1024     }
1025     return true;
1026 }
1027
1028 static int nvme_co_prw(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
1029                        QEMUIOVector *qiov, bool is_write, int flags)
1030 {
1031     BDRVNVMeState *s = bs->opaque;
1032     int r;
1033     uint8_t *buf = NULL;
1034     QEMUIOVector local_qiov;
1035
1036     assert(QEMU_IS_ALIGNED(offset, s->page_size));
1037     assert(QEMU_IS_ALIGNED(bytes, s->page_size));
1038     assert(bytes <= s->max_transfer);
1039     if (nvme_qiov_aligned(bs, qiov)) {
1040         return nvme_co_prw_aligned(bs, offset, bytes, qiov, is_write, flags);
1041     }
1042     trace_nvme_prw_buffered(s, offset, bytes, qiov->niov, is_write);
1043     buf = qemu_try_blockalign(bs, bytes);
1044
1045     if (!buf) {
1046         return -ENOMEM;
1047     }
1048     qemu_iovec_init(&local_qiov, 1);
1049     if (is_write) {
1050         qemu_iovec_to_buf(qiov, 0, buf, bytes);
1051     }
1052     qemu_iovec_add(&local_qiov, buf, bytes);
1053     r = nvme_co_prw_aligned(bs, offset, bytes, &local_qiov, is_write, flags);
1054     qemu_iovec_destroy(&local_qiov);
1055     if (!r && !is_write) {
1056         qemu_iovec_from_buf(qiov, 0, buf, bytes);
1057     }
1058     qemu_vfree(buf);
1059     return r;
1060 }
1061
1062 static coroutine_fn int nvme_co_preadv(BlockDriverState *bs,
1063                                        uint64_t offset, uint64_t bytes,
1064                                        QEMUIOVector *qiov, int flags)
1065 {
1066     return nvme_co_prw(bs, offset, bytes, qiov, false, flags);
1067 }
1068
1069 static coroutine_fn int nvme_co_pwritev(BlockDriverState *bs,
1070                                         uint64_t offset, uint64_t bytes,
1071                                         QEMUIOVector *qiov, int flags)
1072 {
1073     return nvme_co_prw(bs, offset, bytes, qiov, true, flags);
1074 }
1075
1076 static coroutine_fn int nvme_co_flush(BlockDriverState *bs)
1077 {
1078     BDRVNVMeState *s = bs->opaque;
1079     NVMeQueuePair *ioq = s->queues[1];
1080     NVMeRequest *req;
1081     NvmeCmd cmd = {
1082         .opcode = NVME_CMD_FLUSH,
1083         .nsid = cpu_to_le32(s->nsid),
1084     };
1085     NVMeCoData data = {
1086         .ctx = bdrv_get_aio_context(bs),
1087         .ret = -EINPROGRESS,
1088     };
1089
1090     assert(s->nr_queues > 1);
1091     req = nvme_get_free_req(ioq);
1092     assert(req);
1093     nvme_submit_command(s, ioq, req, &cmd, nvme_rw_cb, &data);
1094
1095     data.co = qemu_coroutine_self();
1096     if (data.ret == -EINPROGRESS) {
1097         qemu_coroutine_yield();
1098     }
1099
1100     return data.ret;
1101 }
1102
1103
1104 static coroutine_fn int nvme_co_pwrite_zeroes(BlockDriverState *bs,
1105                                               int64_t offset,
1106                                               int bytes,
1107                                               BdrvRequestFlags flags)
1108 {
1109     BDRVNVMeState *s = bs->opaque;
1110     NVMeQueuePair *ioq = s->queues[1];
1111     NVMeRequest *req;
1112
1113     uint32_t cdw12 = ((bytes >> s->blkshift) - 1) & 0xFFFF;
1114
1115     if (!s->supports_write_zeroes) {
1116         return -ENOTSUP;
1117     }
1118
1119     NvmeCmd cmd = {
1120         .opcode = NVME_CMD_WRITE_ZEROS,
1121         .nsid = cpu_to_le32(s->nsid),
1122         .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
1123         .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
1124     };
1125
1126     NVMeCoData data = {
1127         .ctx = bdrv_get_aio_context(bs),
1128         .ret = -EINPROGRESS,
1129     };
1130
1131     if (flags & BDRV_REQ_MAY_UNMAP) {
1132         cdw12 |= (1 << 25);
1133     }
1134
1135     if (flags & BDRV_REQ_FUA) {
1136         cdw12 |= (1 << 30);
1137     }
1138
1139     cmd.cdw12 = cpu_to_le32(cdw12);
1140
1141     trace_nvme_write_zeroes(s, offset, bytes, flags);
1142     assert(s->nr_queues > 1);
1143     req = nvme_get_free_req(ioq);
1144     assert(req);
1145
1146     nvme_submit_command(s, ioq, req, &cmd, nvme_rw_cb, &data);
1147
1148     data.co = qemu_coroutine_self();
1149     while (data.ret == -EINPROGRESS) {
1150         qemu_coroutine_yield();
1151     }
1152
1153     trace_nvme_rw_done(s, true, offset, bytes, data.ret);
1154     return data.ret;
1155 }
1156
1157
1158 static int coroutine_fn nvme_co_pdiscard(BlockDriverState *bs,
1159                                          int64_t offset,
1160                                          int bytes)
1161 {
1162     BDRVNVMeState *s = bs->opaque;
1163     NVMeQueuePair *ioq = s->queues[1];
1164     NVMeRequest *req;
1165     NvmeDsmRange *buf;
1166     QEMUIOVector local_qiov;
1167     int ret;
1168
1169     NvmeCmd cmd = {
1170         .opcode = NVME_CMD_DSM,
1171         .nsid = cpu_to_le32(s->nsid),
1172         .cdw10 = cpu_to_le32(0), /*number of ranges - 0 based*/
1173         .cdw11 = cpu_to_le32(1 << 2), /*deallocate bit*/
1174     };
1175
1176     NVMeCoData data = {
1177         .ctx = bdrv_get_aio_context(bs),
1178         .ret = -EINPROGRESS,
1179     };
1180
1181     if (!s->supports_discard) {
1182         return -ENOTSUP;
1183     }
1184
1185     assert(s->nr_queues > 1);
1186
1187     buf = qemu_try_blockalign0(bs, s->page_size);
1188     if (!buf) {
1189         return -ENOMEM;
1190     }
1191
1192     buf->nlb = cpu_to_le32(bytes >> s->blkshift);
1193     buf->slba = cpu_to_le64(offset >> s->blkshift);
1194     buf->cattr = 0;
1195
1196     qemu_iovec_init(&local_qiov, 1);
1197     qemu_iovec_add(&local_qiov, buf, 4096);
1198
1199     req = nvme_get_free_req(ioq);
1200     assert(req);
1201
1202     qemu_co_mutex_lock(&s->dma_map_lock);
1203     ret = nvme_cmd_map_qiov(bs, &cmd, req, &local_qiov);
1204     qemu_co_mutex_unlock(&s->dma_map_lock);
1205
1206     if (ret) {
1207         req->busy = false;
1208         goto out;
1209     }
1210
1211     trace_nvme_dsm(s, offset, bytes);
1212
1213     nvme_submit_command(s, ioq, req, &cmd, nvme_rw_cb, &data);
1214
1215     data.co = qemu_coroutine_self();
1216     while (data.ret == -EINPROGRESS) {
1217         qemu_coroutine_yield();
1218     }
1219
1220     qemu_co_mutex_lock(&s->dma_map_lock);
1221     ret = nvme_cmd_unmap_qiov(bs, &local_qiov);
1222     qemu_co_mutex_unlock(&s->dma_map_lock);
1223
1224     if (ret) {
1225         goto out;
1226     }
1227
1228     ret = data.ret;
1229     trace_nvme_dsm_done(s, offset, bytes, ret);
1230 out:
1231     qemu_iovec_destroy(&local_qiov);
1232     qemu_vfree(buf);
1233     return ret;
1234
1235 }
1236
1237
1238 static int nvme_reopen_prepare(BDRVReopenState *reopen_state,
1239                                BlockReopenQueue *queue, Error **errp)
1240 {
1241     return 0;
1242 }
1243
1244 static void nvme_refresh_filename(BlockDriverState *bs)
1245 {
1246     BDRVNVMeState *s = bs->opaque;
1247
1248     snprintf(bs->exact_filename, sizeof(bs->exact_filename), "nvme://%s/%i",
1249              s->device, s->nsid);
1250 }
1251
1252 static void nvme_refresh_limits(BlockDriverState *bs, Error **errp)
1253 {
1254     BDRVNVMeState *s = bs->opaque;
1255
1256     bs->bl.opt_mem_alignment = s->page_size;
1257     bs->bl.request_alignment = s->page_size;
1258     bs->bl.max_transfer = s->max_transfer;
1259 }
1260
1261 static void nvme_detach_aio_context(BlockDriverState *bs)
1262 {
1263     BDRVNVMeState *s = bs->opaque;
1264
1265     aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
1266                            false, NULL, NULL);
1267 }
1268
1269 static void nvme_attach_aio_context(BlockDriverState *bs,
1270                                     AioContext *new_context)
1271 {
1272     BDRVNVMeState *s = bs->opaque;
1273
1274     s->aio_context = new_context;
1275     aio_set_event_notifier(new_context, &s->irq_notifier,
1276                            false, nvme_handle_event, nvme_poll_cb);
1277 }
1278
1279 static void nvme_aio_plug(BlockDriverState *bs)
1280 {
1281     BDRVNVMeState *s = bs->opaque;
1282     assert(!s->plugged);
1283     s->plugged = true;
1284 }
1285
1286 static void nvme_aio_unplug(BlockDriverState *bs)
1287 {
1288     int i;
1289     BDRVNVMeState *s = bs->opaque;
1290     assert(s->plugged);
1291     s->plugged = false;
1292     for (i = 1; i < s->nr_queues; i++) {
1293         NVMeQueuePair *q = s->queues[i];
1294         qemu_mutex_lock(&q->lock);
1295         nvme_kick(s, q);
1296         nvme_process_completion(s, q);
1297         qemu_mutex_unlock(&q->lock);
1298     }
1299 }
1300
1301 static void nvme_register_buf(BlockDriverState *bs, void *host, size_t size)
1302 {
1303     int ret;
1304     BDRVNVMeState *s = bs->opaque;
1305
1306     ret = qemu_vfio_dma_map(s->vfio, host, size, false, NULL);
1307     if (ret) {
1308         /* FIXME: we may run out of IOVA addresses after repeated
1309          * bdrv_register_buf/bdrv_unregister_buf, because nvme_vfio_dma_unmap
1310          * doesn't reclaim addresses for fixed mappings. */
1311         error_report("nvme_register_buf failed: %s", strerror(-ret));
1312     }
1313 }
1314
1315 static void nvme_unregister_buf(BlockDriverState *bs, void *host)
1316 {
1317     BDRVNVMeState *s = bs->opaque;
1318
1319     qemu_vfio_dma_unmap(s->vfio, host);
1320 }
1321
1322 static const char *const nvme_strong_runtime_opts[] = {
1323     NVME_BLOCK_OPT_DEVICE,
1324     NVME_BLOCK_OPT_NAMESPACE,
1325
1326     NULL
1327 };
1328
1329 static BlockDriver bdrv_nvme = {
1330     .format_name              = "nvme",
1331     .protocol_name            = "nvme",
1332     .instance_size            = sizeof(BDRVNVMeState),
1333
1334     .bdrv_co_create_opts      = bdrv_co_create_opts_simple,
1335     .create_opts              = &bdrv_create_opts_simple,
1336
1337     .bdrv_parse_filename      = nvme_parse_filename,
1338     .bdrv_file_open           = nvme_file_open,
1339     .bdrv_close               = nvme_close,
1340     .bdrv_getlength           = nvme_getlength,
1341     .bdrv_probe_blocksizes    = nvme_probe_blocksizes,
1342
1343     .bdrv_co_preadv           = nvme_co_preadv,
1344     .bdrv_co_pwritev          = nvme_co_pwritev,
1345
1346     .bdrv_co_pwrite_zeroes    = nvme_co_pwrite_zeroes,
1347     .bdrv_co_pdiscard         = nvme_co_pdiscard,
1348
1349     .bdrv_co_flush_to_disk    = nvme_co_flush,
1350     .bdrv_reopen_prepare      = nvme_reopen_prepare,
1351
1352     .bdrv_refresh_filename    = nvme_refresh_filename,
1353     .bdrv_refresh_limits      = nvme_refresh_limits,
1354     .strong_runtime_opts      = nvme_strong_runtime_opts,
1355
1356     .bdrv_detach_aio_context  = nvme_detach_aio_context,
1357     .bdrv_attach_aio_context  = nvme_attach_aio_context,
1358
1359     .bdrv_io_plug             = nvme_aio_plug,
1360     .bdrv_io_unplug           = nvme_aio_unplug,
1361
1362     .bdrv_register_buf        = nvme_register_buf,
1363     .bdrv_unregister_buf      = nvme_unregister_buf,
1364 };
1365
1366 static void bdrv_nvme_init(void)
1367 {
1368     bdrv_register(&bdrv_nvme);
1369 }
1370
1371 block_init(bdrv_nvme_init);