hw/block/nvme.c

   1 /*
   2  * QEMU NVM Express Controller
   3  *
   4  * Copyright (c) 2012, Intel Corporation
   5  *
   6  * Written by Keith Busch <[email protected]>
   7  *
   8  * This code is licensed under the GNU GPL v2 or later.
   9  */
  10
  11 /**
  12  * Reference Specs: http://www.nvmexpress.org, 1.2, 1.1, 1.0e
  13  *
  14  *  http://www.nvmexpress.org/resources/
  15  */
  16
  17 /**
  18  * Usage: add options:
  19  *      -drive file=<file>,if=none,id=<drive_id>
  20  *      -device nvme,drive=<drive_id>,serial=<serial>,id=<id[optional]>, \
  21  *              cmb_size_mb=<cmb_size_mb[optional]>
  22  *
  23  * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
  24  * offset 0 in BAR2 and supports only WDS, RDS and SQS for now.
  25  */
  26
  27 #include "qemu/osdep.h"
  28 #include "hw/block/block.h"
  29 #include "hw/hw.h"
  30 #include "hw/pci/msix.h"
  31 #include "hw/pci/pci.h"
  32 #include "sysemu/sysemu.h"
  33 #include "qapi/error.h"
  34 #include "qapi/visitor.h"
  35 #include "sysemu/block-backend.h"
  36
  37 #include "qemu/log.h"
  38 #include "qemu/cutils.h"
  39 #include "trace.h"
  40 #include "nvme.h"
  41
  42 #define NVME_GUEST_ERR(trace, fmt, ...) \
  43     do { \
  44         (trace_##trace)(__VA_ARGS__); \
  45         qemu_log_mask(LOG_GUEST_ERROR, #trace \
  46             " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
  47     } while (0)
  48
  49 static void nvme_process_sq(void *opaque);
  50
  51 static void nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
  52 {
  53     if (n->cmbsz && addr >= n->ctrl_mem.addr &&
  54                 addr < (n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size))) {
  55         memcpy(buf, (void *)&n->cmbuf[addr - n->ctrl_mem.addr], size);
  56     } else {
  57         pci_dma_read(&n->parent_obj, addr, buf, size);
  58     }
  59 }
  60
  61 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
  62 {
  63     return sqid < n->num_queues && n->sq[sqid] != NULL ? 0 : -1;
  64 }
  65
  66 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
  67 {
  68     return cqid < n->num_queues && n->cq[cqid] != NULL ? 0 : -1;
  69 }
  70
  71 static void nvme_inc_cq_tail(NvmeCQueue *cq)
  72 {
  73     cq->tail++;
  74     if (cq->tail >= cq->size) {
  75         cq->tail = 0;
  76         cq->phase = !cq->phase;
  77     }
  78 }
  79
  80 static void nvme_inc_sq_head(NvmeSQueue *sq)
  81 {
  82     sq->head = (sq->head + 1) % sq->size;
  83 }
  84
  85 static uint8_t nvme_cq_full(NvmeCQueue *cq)
  86 {
  87     return (cq->tail + 1) % cq->size == cq->head;
  88 }
  89
  90 static uint8_t nvme_sq_empty(NvmeSQueue *sq)
  91 {
  92     return sq->head == sq->tail;
  93 }
  94
  95 static void nvme_irq_check(NvmeCtrl *n)
  96 {
  97     if (msix_enabled(&(n->parent_obj))) {
  98         return;
  99     }
 100     if (~n->bar.intms & n->irq_status) {
 101         pci_irq_assert(&n->parent_obj);
 102     } else {
 103         pci_irq_deassert(&n->parent_obj);
 104     }
 105 }
 106
 107 static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
 108 {
 109     if (cq->irq_enabled) {
 110         if (msix_enabled(&(n->parent_obj))) {
 111             trace_nvme_irq_msix(cq->vector);
 112             msix_notify(&(n->parent_obj), cq->vector);
 113         } else {
 114             trace_nvme_irq_pin();
 115             assert(cq->cqid < 64);
 116             n->irq_status |= 1 << cq->cqid;
 117             nvme_irq_check(n);
 118         }
 119     } else {
 120         trace_nvme_irq_masked();
 121     }
 122 }
 123
 124 static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
 125 {
 126     if (cq->irq_enabled) {
 127         if (msix_enabled(&(n->parent_obj))) {
 128             return;
 129         } else {
 130             assert(cq->cqid < 64);
 131             n->irq_status &= ~(1 << cq->cqid);
 132             nvme_irq_check(n);
 133         }
 134     }
 135 }
 136
 137 static uint16_t nvme_map_prp(QEMUSGList *qsg, QEMUIOVector *iov, uint64_t prp1,
 138                              uint64_t prp2, uint32_t len, NvmeCtrl *n)
 139 {
 140     hwaddr trans_len = n->page_size - (prp1 % n->page_size);
 141     trans_len = MIN(len, trans_len);
 142     int num_prps = (len >> n->page_bits) + 1;
 143
 144     if (unlikely(!prp1)) {
 145         trace_nvme_err_invalid_prp();
 146         return NVME_INVALID_FIELD | NVME_DNR;
 147     } else if (n->cmbsz && prp1 >= n->ctrl_mem.addr &&
 148                prp1 < n->ctrl_mem.addr + int128_get64(n->ctrl_mem.size)) {
 149         qsg->nsg = 0;
 150         qemu_iovec_init(iov, num_prps);
 151         qemu_iovec_add(iov, (void *)&n->cmbuf[prp1 - n->ctrl_mem.addr], trans_len);
 152     } else {
 153         pci_dma_sglist_init(qsg, &n->parent_obj, num_prps);
 154         qemu_sglist_add(qsg, prp1, trans_len);
 155     }
 156     len -= trans_len;
 157     if (len) {
 158         if (unlikely(!prp2)) {
 159             trace_nvme_err_invalid_prp2_missing();
 160             goto unmap;
 161         }
 162         if (len > n->page_size) {
 163             uint64_t prp_list[n->max_prp_ents];
 164             uint32_t nents, prp_trans;
 165             int i = 0;
 166
 167             nents = (len + n->page_size - 1) >> n->page_bits;
 168             prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
 169             nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
 170             while (len != 0) {
 171                 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
 172
 173                 if (i == n->max_prp_ents - 1 && len > n->page_size) {
 174                     if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
 175                         trace_nvme_err_invalid_prplist_ent(prp_ent);
 176                         goto unmap;
 177                     }
 178
 179                     i = 0;
 180                     nents = (len + n->page_size - 1) >> n->page_bits;
 181                     prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
 182                     nvme_addr_read(n, prp_ent, (void *)prp_list,
 183                         prp_trans);
 184                     prp_ent = le64_to_cpu(prp_list[i]);
 185                 }
 186
 187                 if (unlikely(!prp_ent || prp_ent & (n->page_size - 1))) {
 188                     trace_nvme_err_invalid_prplist_ent(prp_ent);
 189                     goto unmap;
 190                 }
 191
 192                 trans_len = MIN(len, n->page_size);
 193                 if (qsg->nsg){
 194                     qemu_sglist_add(qsg, prp_ent, trans_len);
 195                 } else {
 196                     qemu_iovec_add(iov, (void *)&n->cmbuf[prp_ent - n->ctrl_mem.addr], trans_len);
 197                 }
 198                 len -= trans_len;
 199                 i++;
 200             }
 201         } else {
 202             if (unlikely(prp2 & (n->page_size - 1))) {
 203                 trace_nvme_err_invalid_prp2_align(prp2);
 204                 goto unmap;
 205             }
 206             if (qsg->nsg) {
 207                 qemu_sglist_add(qsg, prp2, len);
 208             } else {
 209                 qemu_iovec_add(iov, (void *)&n->cmbuf[prp2 - n->ctrl_mem.addr], trans_len);
 210             }
 211         }
 212     }
 213     return NVME_SUCCESS;
 214
 215  unmap:
 216     qemu_sglist_destroy(qsg);
 217     return NVME_INVALID_FIELD | NVME_DNR;
 218 }
 219
 220 static uint16_t nvme_dma_read_prp(NvmeCtrl *n, uint8_t *ptr, uint32_t len,
 221     uint64_t prp1, uint64_t prp2)
 222 {
 223     QEMUSGList qsg;
 224     QEMUIOVector iov;
 225     uint16_t status = NVME_SUCCESS;
 226
 227     trace_nvme_dma_read(prp1, prp2);
 228
 229     if (nvme_map_prp(&qsg, &iov, prp1, prp2, len, n)) {
 230         return NVME_INVALID_FIELD | NVME_DNR;
 231     }
 232     if (qsg.nsg > 0) {
 233         if (unlikely(dma_buf_read(ptr, len, &qsg))) {
 234             trace_nvme_err_invalid_dma();
 235             status = NVME_INVALID_FIELD | NVME_DNR;
 236         }
 237         qemu_sglist_destroy(&qsg);
 238     } else {
 239         if (unlikely(qemu_iovec_to_buf(&iov, 0, ptr, len) != len)) {
 240             trace_nvme_err_invalid_dma();
 241             status = NVME_INVALID_FIELD | NVME_DNR;
 242         }
 243         qemu_iovec_destroy(&iov);
 244     }
 245     return status;
 246 }
 247
 248 static void nvme_post_cqes(void *opaque)
 249 {
 250     NvmeCQueue *cq = opaque;
 251     NvmeCtrl *n = cq->ctrl;
 252     NvmeRequest *req, *next;
 253
 254     QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
 255         NvmeSQueue *sq;
 256         hwaddr addr;
 257
 258         if (nvme_cq_full(cq)) {
 259             break;
 260         }
 261
 262         QTAILQ_REMOVE(&cq->req_list, req, entry);
 263         sq = req->sq;
 264         req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
 265         req->cqe.sq_id = cpu_to_le16(sq->sqid);
 266         req->cqe.sq_head = cpu_to_le16(sq->head);
 267         addr = cq->dma_addr + cq->tail * n->cqe_size;
 268         nvme_inc_cq_tail(cq);
 269         pci_dma_write(&n->parent_obj, addr, (void *)&req->cqe,
 270             sizeof(req->cqe));
 271         QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
 272     }
 273     nvme_irq_assert(n, cq);
 274 }
 275
 276 static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
 277 {
 278     assert(cq->cqid == req->sq->cqid);
 279     QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
 280     QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
 281     timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
 282 }
 283
 284 static void nvme_rw_cb(void *opaque, int ret)
 285 {
 286     NvmeRequest *req = opaque;
 287     NvmeSQueue *sq = req->sq;
 288     NvmeCtrl *n = sq->ctrl;
 289     NvmeCQueue *cq = n->cq[sq->cqid];
 290
 291     if (!ret) {
 292         block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
 293         req->status = NVME_SUCCESS;
 294     } else {
 295         block_acct_failed(blk_get_stats(n->conf.blk), &req->acct);
 296         req->status = NVME_INTERNAL_DEV_ERROR;
 297     }
 298     if (req->has_sg) {
 299         qemu_sglist_destroy(&req->qsg);
 300     }
 301     nvme_enqueue_req_completion(cq, req);
 302 }
 303
 304 static uint16_t nvme_flush(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
 305     NvmeRequest *req)
 306 {
 307     req->has_sg = false;
 308     block_acct_start(blk_get_stats(n->conf.blk), &req->acct, 0,
 309          BLOCK_ACCT_FLUSH);
 310     req->aiocb = blk_aio_flush(n->conf.blk, nvme_rw_cb, req);
 311
 312     return NVME_NO_COMPLETE;
 313 }
 314
 315 static uint16_t nvme_write_zeros(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
 316     NvmeRequest *req)
 317 {
 318     NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
 319     const uint8_t lba_index = NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas);
 320     const uint8_t data_shift = ns->id_ns.lbaf[lba_index].ds;
 321     uint64_t slba = le64_to_cpu(rw->slba);
 322     uint32_t nlb  = le16_to_cpu(rw->nlb) + 1;
 323     uint64_t aio_slba = slba << (data_shift - BDRV_SECTOR_BITS);
 324     uint32_t aio_nlb = nlb << (data_shift - BDRV_SECTOR_BITS);
 325
 326     if (unlikely(slba + nlb > ns->id_ns.nsze)) {
 327         trace_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
 328         return NVME_LBA_RANGE | NVME_DNR;
 329     }
 330
 331     req->has_sg = false;
 332     block_acct_start(blk_get_stats(n->conf.blk), &req->acct, 0,
 333                      BLOCK_ACCT_WRITE);
 334     req->aiocb = blk_aio_pwrite_zeroes(n->conf.blk, aio_slba, aio_nlb,
 335                                         BDRV_REQ_MAY_UNMAP, nvme_rw_cb, req);
 336     return NVME_NO_COMPLETE;
 337 }
 338
 339 static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
 340     NvmeRequest *req)
 341 {
 342     NvmeRwCmd *rw = (NvmeRwCmd *)cmd;
 343     uint32_t nlb  = le32_to_cpu(rw->nlb) + 1;
 344     uint64_t slba = le64_to_cpu(rw->slba);
 345     uint64_t prp1 = le64_to_cpu(rw->prp1);
 346     uint64_t prp2 = le64_to_cpu(rw->prp2);
 347
 348     uint8_t lba_index  = NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas);
 349     uint8_t data_shift = ns->id_ns.lbaf[lba_index].ds;
 350     uint64_t data_size = (uint64_t)nlb << data_shift;
 351     uint64_t data_offset = slba << data_shift;
 352     int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
 353     enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
 354
 355     trace_nvme_rw(is_write ? "write" : "read", nlb, data_size, slba);
 356
 357     if (unlikely((slba + nlb) > ns->id_ns.nsze)) {
 358         block_acct_invalid(blk_get_stats(n->conf.blk), acct);
 359         trace_nvme_err_invalid_lba_range(slba, nlb, ns->id_ns.nsze);
 360         return NVME_LBA_RANGE | NVME_DNR;
 361     }
 362
 363     if (nvme_map_prp(&req->qsg, &req->iov, prp1, prp2, data_size, n)) {
 364         block_acct_invalid(blk_get_stats(n->conf.blk), acct);
 365         return NVME_INVALID_FIELD | NVME_DNR;
 366     }
 367
 368     dma_acct_start(n->conf.blk, &req->acct, &req->qsg, acct);
 369     if (req->qsg.nsg > 0) {
 370         req->has_sg = true;
 371         req->aiocb = is_write ?
 372             dma_blk_write(n->conf.blk, &req->qsg, data_offset, BDRV_SECTOR_SIZE,
 373                           nvme_rw_cb, req) :
 374             dma_blk_read(n->conf.blk, &req->qsg, data_offset, BDRV_SECTOR_SIZE,
 375                          nvme_rw_cb, req);
 376     } else {
 377         req->has_sg = false;
 378         req->aiocb = is_write ?
 379             blk_aio_pwritev(n->conf.blk, data_offset, &req->iov, 0, nvme_rw_cb,
 380                             req) :
 381             blk_aio_preadv(n->conf.blk, data_offset, &req->iov, 0, nvme_rw_cb,
 382                            req);
 383     }
 384
 385     return NVME_NO_COMPLETE;
 386 }
 387
 388 static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
 389 {
 390     NvmeNamespace *ns;
 391     uint32_t nsid = le32_to_cpu(cmd->nsid);
 392
 393     if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
 394         trace_nvme_err_invalid_ns(nsid, n->num_namespaces);
 395         return NVME_INVALID_NSID | NVME_DNR;
 396     }
 397
 398     ns = &n->namespaces[nsid - 1];
 399     switch (cmd->opcode) {
 400     case NVME_CMD_FLUSH:
 401         return nvme_flush(n, ns, cmd, req);
 402     case NVME_CMD_WRITE_ZEROS:
 403         return nvme_write_zeros(n, ns, cmd, req);
 404     case NVME_CMD_WRITE:
 405     case NVME_CMD_READ:
 406         return nvme_rw(n, ns, cmd, req);
 407     default:
 408         trace_nvme_err_invalid_opc(cmd->opcode);
 409         return NVME_INVALID_OPCODE | NVME_DNR;
 410     }
 411 }
 412
 413 static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
 414 {
 415     n->sq[sq->sqid] = NULL;
 416     timer_del(sq->timer);
 417     timer_free(sq->timer);
 418     g_free(sq->io_req);
 419     if (sq->sqid) {
 420         g_free(sq);
 421     }
 422 }
 423
 424 static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeCmd *cmd)
 425 {
 426     NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
 427     NvmeRequest *req, *next;
 428     NvmeSQueue *sq;
 429     NvmeCQueue *cq;
 430     uint16_t qid = le16_to_cpu(c->qid);
 431
 432     if (unlikely(!qid || nvme_check_sqid(n, qid))) {
 433         trace_nvme_err_invalid_del_sq(qid);
 434         return NVME_INVALID_QID | NVME_DNR;
 435     }
 436
 437     trace_nvme_del_sq(qid);
 438
 439     sq = n->sq[qid];
 440     while (!QTAILQ_EMPTY(&sq->out_req_list)) {
 441         req = QTAILQ_FIRST(&sq->out_req_list);
 442         assert(req->aiocb);
 443         blk_aio_cancel(req->aiocb);
 444     }
 445     if (!nvme_check_cqid(n, sq->cqid)) {
 446         cq = n->cq[sq->cqid];
 447         QTAILQ_REMOVE(&cq->sq_list, sq, entry);
 448
 449         nvme_post_cqes(cq);
 450         QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
 451             if (req->sq == sq) {
 452                 QTAILQ_REMOVE(&cq->req_list, req, entry);
 453                 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
 454             }
 455         }
 456     }
 457
 458     nvme_free_sq(sq, n);
 459     return NVME_SUCCESS;
 460 }
 461
 462 static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
 463     uint16_t sqid, uint16_t cqid, uint16_t size)
 464 {
 465     int i;
 466     NvmeCQueue *cq;
 467
 468     sq->ctrl = n;
 469     sq->dma_addr = dma_addr;
 470     sq->sqid = sqid;
 471     sq->size = size;
 472     sq->cqid = cqid;
 473     sq->head = sq->tail = 0;
 474     sq->io_req = g_new(NvmeRequest, sq->size);
 475
 476     QTAILQ_INIT(&sq->req_list);
 477     QTAILQ_INIT(&sq->out_req_list);
 478     for (i = 0; i < sq->size; i++) {
 479         sq->io_req[i].sq = sq;
 480         QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
 481     }
 482     sq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_process_sq, sq);
 483
 484     assert(n->cq[cqid]);
 485     cq = n->cq[cqid];
 486     QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
 487     n->sq[sqid] = sq;
 488 }
 489
 490 static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeCmd *cmd)
 491 {
 492     NvmeSQueue *sq;
 493     NvmeCreateSq *c = (NvmeCreateSq *)cmd;
 494
 495     uint16_t cqid = le16_to_cpu(c->cqid);
 496     uint16_t sqid = le16_to_cpu(c->sqid);
 497     uint16_t qsize = le16_to_cpu(c->qsize);
 498     uint16_t qflags = le16_to_cpu(c->sq_flags);
 499     uint64_t prp1 = le64_to_cpu(c->prp1);
 500
 501     trace_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
 502
 503     if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
 504         trace_nvme_err_invalid_create_sq_cqid(cqid);
 505         return NVME_INVALID_CQID | NVME_DNR;
 506     }
 507     if (unlikely(!sqid || !nvme_check_sqid(n, sqid))) {
 508         trace_nvme_err_invalid_create_sq_sqid(sqid);
 509         return NVME_INVALID_QID | NVME_DNR;
 510     }
 511     if (unlikely(!qsize || qsize > NVME_CAP_MQES(n->bar.cap))) {
 512         trace_nvme_err_invalid_create_sq_size(qsize);
 513         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
 514     }
 515     if (unlikely(!prp1 || prp1 & (n->page_size - 1))) {
 516         trace_nvme_err_invalid_create_sq_addr(prp1);
 517         return NVME_INVALID_FIELD | NVME_DNR;
 518     }
 519     if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
 520         trace_nvme_err_invalid_create_sq_qflags(NVME_SQ_FLAGS_PC(qflags));
 521         return NVME_INVALID_FIELD | NVME_DNR;
 522     }
 523     sq = g_malloc0(sizeof(*sq));
 524     nvme_init_sq(sq, n, prp1, sqid, cqid, qsize + 1);
 525     return NVME_SUCCESS;
 526 }
 527
 528 static void nvme_free_cq(NvmeCQueue *cq, NvmeCtrl *n)
 529 {
 530     n->cq[cq->cqid] = NULL;
 531     timer_del(cq->timer);
 532     timer_free(cq->timer);
 533     msix_vector_unuse(&n->parent_obj, cq->vector);
 534     if (cq->cqid) {
 535         g_free(cq);
 536     }
 537 }
 538
 539 static uint16_t nvme_del_cq(NvmeCtrl *n, NvmeCmd *cmd)
 540 {
 541     NvmeDeleteQ *c = (NvmeDeleteQ *)cmd;
 542     NvmeCQueue *cq;
 543     uint16_t qid = le16_to_cpu(c->qid);
 544
 545     if (unlikely(!qid || nvme_check_cqid(n, qid))) {
 546         trace_nvme_err_invalid_del_cq_cqid(qid);
 547         return NVME_INVALID_CQID | NVME_DNR;
 548     }
 549
 550     cq = n->cq[qid];
 551     if (unlikely(!QTAILQ_EMPTY(&cq->sq_list))) {
 552         trace_nvme_err_invalid_del_cq_notempty(qid);
 553         return NVME_INVALID_QUEUE_DEL;
 554     }
 555     trace_nvme_del_cq(qid);
 556     nvme_free_cq(cq, n);
 557     return NVME_SUCCESS;
 558 }
 559
 560 static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr,
 561     uint16_t cqid, uint16_t vector, uint16_t size, uint16_t irq_enabled)
 562 {
 563     cq->ctrl = n;
 564     cq->cqid = cqid;
 565     cq->size = size;
 566     cq->dma_addr = dma_addr;
 567     cq->phase = 1;
 568     cq->irq_enabled = irq_enabled;
 569     cq->vector = vector;
 570     cq->head = cq->tail = 0;
 571     QTAILQ_INIT(&cq->req_list);
 572     QTAILQ_INIT(&cq->sq_list);
 573     msix_vector_use(&n->parent_obj, cq->vector);
 574     n->cq[cqid] = cq;
 575     cq->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, nvme_post_cqes, cq);
 576 }
 577
 578 static uint16_t nvme_create_cq(NvmeCtrl *n, NvmeCmd *cmd)
 579 {
 580     NvmeCQueue *cq;
 581     NvmeCreateCq *c = (NvmeCreateCq *)cmd;
 582     uint16_t cqid = le16_to_cpu(c->cqid);
 583     uint16_t vector = le16_to_cpu(c->irq_vector);
 584     uint16_t qsize = le16_to_cpu(c->qsize);
 585     uint16_t qflags = le16_to_cpu(c->cq_flags);
 586     uint64_t prp1 = le64_to_cpu(c->prp1);
 587
 588     trace_nvme_create_cq(prp1, cqid, vector, qsize, qflags,
 589                          NVME_CQ_FLAGS_IEN(qflags) != 0);
 590
 591     if (unlikely(!cqid || !nvme_check_cqid(n, cqid))) {
 592         trace_nvme_err_invalid_create_cq_cqid(cqid);
 593         return NVME_INVALID_CQID | NVME_DNR;
 594     }
 595     if (unlikely(!qsize || qsize > NVME_CAP_MQES(n->bar.cap))) {
 596         trace_nvme_err_invalid_create_cq_size(qsize);
 597         return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
 598     }
 599     if (unlikely(!prp1)) {
 600         trace_nvme_err_invalid_create_cq_addr(prp1);
 601         return NVME_INVALID_FIELD | NVME_DNR;
 602     }
 603     if (unlikely(vector > n->num_queues)) {
 604         trace_nvme_err_invalid_create_cq_vector(vector);
 605         return NVME_INVALID_IRQ_VECTOR | NVME_DNR;
 606     }
 607     if (unlikely(!(NVME_CQ_FLAGS_PC(qflags)))) {
 608         trace_nvme_err_invalid_create_cq_qflags(NVME_CQ_FLAGS_PC(qflags));
 609         return NVME_INVALID_FIELD | NVME_DNR;
 610     }
 611
 612     cq = g_malloc0(sizeof(*cq));
 613     nvme_init_cq(cq, n, prp1, cqid, vector, qsize + 1,
 614         NVME_CQ_FLAGS_IEN(qflags));
 615     return NVME_SUCCESS;
 616 }
 617
 618 static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeIdentify *c)
 619 {
 620     uint64_t prp1 = le64_to_cpu(c->prp1);
 621     uint64_t prp2 = le64_to_cpu(c->prp2);
 622
 623     trace_nvme_identify_ctrl();
 624
 625     return nvme_dma_read_prp(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl),
 626         prp1, prp2);
 627 }
 628
 629 static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeIdentify *c)
 630 {
 631     NvmeNamespace *ns;
 632     uint32_t nsid = le32_to_cpu(c->nsid);
 633     uint64_t prp1 = le64_to_cpu(c->prp1);
 634     uint64_t prp2 = le64_to_cpu(c->prp2);
 635
 636     trace_nvme_identify_ns(nsid);
 637
 638     if (unlikely(nsid == 0 || nsid > n->num_namespaces)) {
 639         trace_nvme_err_invalid_ns(nsid, n->num_namespaces);
 640         return NVME_INVALID_NSID | NVME_DNR;
 641     }
 642
 643     ns = &n->namespaces[nsid - 1];
 644
 645     return nvme_dma_read_prp(n, (uint8_t *)&ns->id_ns, sizeof(ns->id_ns),
 646         prp1, prp2);
 647 }
 648
 649 static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeIdentify *c)
 650 {
 651     static const int data_len = 4096;
 652     uint32_t min_nsid = le32_to_cpu(c->nsid);
 653     uint64_t prp1 = le64_to_cpu(c->prp1);
 654     uint64_t prp2 = le64_to_cpu(c->prp2);
 655     uint32_t *list;
 656     uint16_t ret;
 657     int i, j = 0;
 658
 659     trace_nvme_identify_nslist(min_nsid);
 660
 661     list = g_malloc0(data_len);
 662     for (i = 0; i < n->num_namespaces; i++) {
 663         if (i < min_nsid) {
 664             continue;
 665         }
 666         list[j++] = cpu_to_le32(i + 1);
 667         if (j == data_len / sizeof(uint32_t)) {
 668             break;
 669         }
 670     }
 671     ret = nvme_dma_read_prp(n, (uint8_t *)list, data_len, prp1, prp2);
 672     g_free(list);
 673     return ret;
 674 }
 675
 676
 677 static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd)
 678 {
 679     NvmeIdentify *c = (NvmeIdentify *)cmd;
 680
 681     switch (le32_to_cpu(c->cns)) {
 682     case 0x00:
 683         return nvme_identify_ns(n, c);
 684     case 0x01:
 685         return nvme_identify_ctrl(n, c);
 686     case 0x02:
 687         return nvme_identify_nslist(n, c);
 688     default:
 689         trace_nvme_err_invalid_identify_cns(le32_to_cpu(c->cns));
 690         return NVME_INVALID_FIELD | NVME_DNR;
 691     }
 692 }
 693
 694 static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
 695 {
 696     uint32_t dw10 = le32_to_cpu(cmd->cdw10);
 697     uint32_t result;
 698
 699     switch (dw10) {
 700     case NVME_VOLATILE_WRITE_CACHE:
 701         result = blk_enable_write_cache(n->conf.blk);
 702         trace_nvme_getfeat_vwcache(result ? "enabled" : "disabled");
 703         break;
 704     case NVME_NUMBER_OF_QUEUES:
 705         result = cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 16));
 706         trace_nvme_getfeat_numq(result);
 707         break;
 708     default:
 709         trace_nvme_err_invalid_getfeat(dw10);
 710         return NVME_INVALID_FIELD | NVME_DNR;
 711     }
 712
 713     req->cqe.result = result;
 714     return NVME_SUCCESS;
 715 }
 716
 717 static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
 718 {
 719     uint32_t dw10 = le32_to_cpu(cmd->cdw10);
 720     uint32_t dw11 = le32_to_cpu(cmd->cdw11);
 721
 722     switch (dw10) {
 723     case NVME_VOLATILE_WRITE_CACHE:
 724         blk_set_enable_write_cache(n->conf.blk, dw11 & 1);
 725         break;
 726     case NVME_NUMBER_OF_QUEUES:
 727         trace_nvme_setfeat_numq((dw11 & 0xFFFF) + 1,
 728                                 ((dw11 >> 16) & 0xFFFF) + 1,
 729                                 n->num_queues - 1, n->num_queues - 1);
 730         req->cqe.result =
 731             cpu_to_le32((n->num_queues - 2) | ((n->num_queues - 2) << 16));
 732         break;
 733     default:
 734         trace_nvme_err_invalid_setfeat(dw10);
 735         return NVME_INVALID_FIELD | NVME_DNR;
 736     }
 737     return NVME_SUCCESS;
 738 }
 739
 740 static uint16_t nvme_admin_cmd(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
 741 {
 742     switch (cmd->opcode) {
 743     case NVME_ADM_CMD_DELETE_SQ:
 744         return nvme_del_sq(n, cmd);
 745     case NVME_ADM_CMD_CREATE_SQ:
 746         return nvme_create_sq(n, cmd);
 747     case NVME_ADM_CMD_DELETE_CQ:
 748         return nvme_del_cq(n, cmd);
 749     case NVME_ADM_CMD_CREATE_CQ:
 750         return nvme_create_cq(n, cmd);
 751     case NVME_ADM_CMD_IDENTIFY:
 752         return nvme_identify(n, cmd);
 753     case NVME_ADM_CMD_SET_FEATURES:
 754         return nvme_set_feature(n, cmd, req);
 755     case NVME_ADM_CMD_GET_FEATURES:
 756         return nvme_get_feature(n, cmd, req);
 757     default:
 758         trace_nvme_err_invalid_admin_opc(cmd->opcode);
 759         return NVME_INVALID_OPCODE | NVME_DNR;
 760     }
 761 }
 762
 763 static void nvme_process_sq(void *opaque)
 764 {
 765     NvmeSQueue *sq = opaque;
 766     NvmeCtrl *n = sq->ctrl;
 767     NvmeCQueue *cq = n->cq[sq->cqid];
 768
 769     uint16_t status;
 770     hwaddr addr;
 771     NvmeCmd cmd;
 772     NvmeRequest *req;
 773
 774     while (!(nvme_sq_empty(sq) || QTAILQ_EMPTY(&sq->req_list))) {
 775         addr = sq->dma_addr + sq->head * n->sqe_size;
 776         nvme_addr_read(n, addr, (void *)&cmd, sizeof(cmd));
 777         nvme_inc_sq_head(sq);
 778
 779         req = QTAILQ_FIRST(&sq->req_list);
 780         QTAILQ_REMOVE(&sq->req_list, req, entry);
 781         QTAILQ_INSERT_TAIL(&sq->out_req_list, req, entry);
 782         memset(&req->cqe, 0, sizeof(req->cqe));
 783         req->cqe.cid = cmd.cid;
 784
 785         status = sq->sqid ? nvme_io_cmd(n, &cmd, req) :
 786             nvme_admin_cmd(n, &cmd, req);
 787         if (status != NVME_NO_COMPLETE) {
 788             req->status = status;
 789             nvme_enqueue_req_completion(cq, req);
 790         }
 791     }
 792 }
 793
 794 static void nvme_clear_ctrl(NvmeCtrl *n)
 795 {
 796     int i;
 797
 798     for (i = 0; i < n->num_queues; i++) {
 799         if (n->sq[i] != NULL) {
 800             nvme_free_sq(n->sq[i], n);
 801         }
 802     }
 803     for (i = 0; i < n->num_queues; i++) {
 804         if (n->cq[i] != NULL) {
 805             nvme_free_cq(n->cq[i], n);
 806         }
 807     }
 808
 809     blk_flush(n->conf.blk);
 810     n->bar.cc = 0;
 811 }
 812
 813 static int nvme_start_ctrl(NvmeCtrl *n)
 814 {
 815     uint32_t page_bits = NVME_CC_MPS(n->bar.cc) + 12;
 816     uint32_t page_size = 1 << page_bits;
 817
 818     if (unlikely(n->cq[0])) {
 819         trace_nvme_err_startfail_cq();
 820         return -1;
 821     }
 822     if (unlikely(n->sq[0])) {
 823         trace_nvme_err_startfail_sq();
 824         return -1;
 825     }
 826     if (unlikely(!n->bar.asq)) {
 827         trace_nvme_err_startfail_nbarasq();
 828         return -1;
 829     }
 830     if (unlikely(!n->bar.acq)) {
 831         trace_nvme_err_startfail_nbaracq();
 832         return -1;
 833     }
 834     if (unlikely(n->bar.asq & (page_size - 1))) {
 835         trace_nvme_err_startfail_asq_misaligned(n->bar.asq);
 836         return -1;
 837     }
 838     if (unlikely(n->bar.acq & (page_size - 1))) {
 839         trace_nvme_err_startfail_acq_misaligned(n->bar.acq);
 840         return -1;
 841     }
 842     if (unlikely(NVME_CC_MPS(n->bar.cc) <
 843                  NVME_CAP_MPSMIN(n->bar.cap))) {
 844         trace_nvme_err_startfail_page_too_small(
 845                     NVME_CC_MPS(n->bar.cc),
 846                     NVME_CAP_MPSMIN(n->bar.cap));
 847         return -1;
 848     }
 849     if (unlikely(NVME_CC_MPS(n->bar.cc) >
 850                  NVME_CAP_MPSMAX(n->bar.cap))) {
 851         trace_nvme_err_startfail_page_too_large(
 852                     NVME_CC_MPS(n->bar.cc),
 853                     NVME_CAP_MPSMAX(n->bar.cap));
 854         return -1;
 855     }
 856     if (unlikely(NVME_CC_IOCQES(n->bar.cc) <
 857                  NVME_CTRL_CQES_MIN(n->id_ctrl.cqes))) {
 858         trace_nvme_err_startfail_cqent_too_small(
 859                     NVME_CC_IOCQES(n->bar.cc),
 860                     NVME_CTRL_CQES_MIN(n->bar.cap));
 861         return -1;
 862     }
 863     if (unlikely(NVME_CC_IOCQES(n->bar.cc) >
 864                  NVME_CTRL_CQES_MAX(n->id_ctrl.cqes))) {
 865         trace_nvme_err_startfail_cqent_too_large(
 866                     NVME_CC_IOCQES(n->bar.cc),
 867                     NVME_CTRL_CQES_MAX(n->bar.cap));
 868         return -1;
 869     }
 870     if (unlikely(NVME_CC_IOSQES(n->bar.cc) <
 871                  NVME_CTRL_SQES_MIN(n->id_ctrl.sqes))) {
 872         trace_nvme_err_startfail_sqent_too_small(
 873                     NVME_CC_IOSQES(n->bar.cc),
 874                     NVME_CTRL_SQES_MIN(n->bar.cap));
 875         return -1;
 876     }
 877     if (unlikely(NVME_CC_IOSQES(n->bar.cc) >
 878                  NVME_CTRL_SQES_MAX(n->id_ctrl.sqes))) {
 879         trace_nvme_err_startfail_sqent_too_large(
 880                     NVME_CC_IOSQES(n->bar.cc),
 881                     NVME_CTRL_SQES_MAX(n->bar.cap));
 882         return -1;
 883     }
 884     if (unlikely(!NVME_AQA_ASQS(n->bar.aqa))) {
 885         trace_nvme_err_startfail_asqent_sz_zero();
 886         return -1;
 887     }
 888     if (unlikely(!NVME_AQA_ACQS(n->bar.aqa))) {
 889         trace_nvme_err_startfail_acqent_sz_zero();
 890         return -1;
 891     }
 892
 893     n->page_bits = page_bits;
 894     n->page_size = page_size;
 895     n->max_prp_ents = n->page_size / sizeof(uint64_t);
 896     n->cqe_size = 1 << NVME_CC_IOCQES(n->bar.cc);
 897     n->sqe_size = 1 << NVME_CC_IOSQES(n->bar.cc);
 898     nvme_init_cq(&n->admin_cq, n, n->bar.acq, 0, 0,
 899         NVME_AQA_ACQS(n->bar.aqa) + 1, 1);
 900     nvme_init_sq(&n->admin_sq, n, n->bar.asq, 0, 0,
 901         NVME_AQA_ASQS(n->bar.aqa) + 1);
 902
 903     return 0;
 904 }
 905
 906 static void nvme_write_bar(NvmeCtrl *n, hwaddr offset, uint64_t data,
 907     unsigned size)
 908 {
 909     if (unlikely(offset & (sizeof(uint32_t) - 1))) {
 910         NVME_GUEST_ERR(nvme_ub_mmiowr_misaligned32,
 911                        "MMIO write not 32-bit aligned,"
 912                        " offset=0x%"PRIx64"", offset);
 913         /* should be ignored, fall through for now */
 914     }
 915
 916     if (unlikely(size < sizeof(uint32_t))) {
 917         NVME_GUEST_ERR(nvme_ub_mmiowr_toosmall,
 918                        "MMIO write smaller than 32-bits,"
 919                        " offset=0x%"PRIx64", size=%u",
 920                        offset, size);
 921         /* should be ignored, fall through for now */
 922     }
 923
 924     switch (offset) {
 925     case 0xc:   /* INTMS */
 926         if (unlikely(msix_enabled(&(n->parent_obj)))) {
 927             NVME_GUEST_ERR(nvme_ub_mmiowr_intmask_with_msix,
 928                            "undefined access to interrupt mask set"
 929                            " when MSI-X is enabled");
 930             /* should be ignored, fall through for now */
 931         }
 932         n->bar.intms |= data & 0xffffffff;
 933         n->bar.intmc = n->bar.intms;
 934         trace_nvme_mmio_intm_set(data & 0xffffffff,
 935                                  n->bar.intmc);
 936         nvme_irq_check(n);
 937         break;
 938     case 0x10:  /* INTMC */
 939         if (unlikely(msix_enabled(&(n->parent_obj)))) {
 940             NVME_GUEST_ERR(nvme_ub_mmiowr_intmask_with_msix,
 941                            "undefined access to interrupt mask clr"
 942                            " when MSI-X is enabled");
 943             /* should be ignored, fall through for now */
 944         }
 945         n->bar.intms &= ~(data & 0xffffffff);
 946         n->bar.intmc = n->bar.intms;
 947         trace_nvme_mmio_intm_clr(data & 0xffffffff,
 948                                  n->bar.intmc);
 949         nvme_irq_check(n);
 950         break;
 951     case 0x14:  /* CC */
 952         trace_nvme_mmio_cfg(data & 0xffffffff);
 953         /* Windows first sends data, then sends enable bit */
 954         if (!NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc) &&
 955             !NVME_CC_SHN(data) && !NVME_CC_SHN(n->bar.cc))
 956         {
 957             n->bar.cc = data;
 958         }
 959
 960         if (NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc)) {
 961             n->bar.cc = data;
 962             if (unlikely(nvme_start_ctrl(n))) {
 963                 trace_nvme_err_startfail();
 964                 n->bar.csts = NVME_CSTS_FAILED;
 965             } else {
 966                 trace_nvme_mmio_start_success();
 967                 n->bar.csts = NVME_CSTS_READY;
 968             }
 969         } else if (!NVME_CC_EN(data) && NVME_CC_EN(n->bar.cc)) {
 970             trace_nvme_mmio_stopped();
 971             nvme_clear_ctrl(n);
 972             n->bar.csts &= ~NVME_CSTS_READY;
 973         }
 974         if (NVME_CC_SHN(data) && !(NVME_CC_SHN(n->bar.cc))) {
 975             trace_nvme_mmio_shutdown_set();
 976             nvme_clear_ctrl(n);
 977             n->bar.cc = data;
 978             n->bar.csts |= NVME_CSTS_SHST_COMPLETE;
 979         } else if (!NVME_CC_SHN(data) && NVME_CC_SHN(n->bar.cc)) {
 980             trace_nvme_mmio_shutdown_cleared();
 981             n->bar.csts &= ~NVME_CSTS_SHST_COMPLETE;
 982             n->bar.cc = data;
 983         }
 984         break;
 985     case 0x1C:  /* CSTS */
 986         if (data & (1 << 4)) {
 987             NVME_GUEST_ERR(nvme_ub_mmiowr_ssreset_w1c_unsupported,
 988                            "attempted to W1C CSTS.NSSRO"
 989                            " but CAP.NSSRS is zero (not supported)");
 990         } else if (data != 0) {
 991             NVME_GUEST_ERR(nvme_ub_mmiowr_ro_csts,
 992                            "attempted to set a read only bit"
 993                            " of controller status");
 994         }
 995         break;
 996     case 0x20:  /* NSSR */
 997         if (data == 0x4E564D65) {
 998             trace_nvme_ub_mmiowr_ssreset_unsupported();
 999         } else {
1000             /* The spec says that writes of other values have no effect */
1001             return;
1002         }
1003         break;
1004     case 0x24:  /* AQA */
1005         n->bar.aqa = data & 0xffffffff;
1006         trace_nvme_mmio_aqattr(data & 0xffffffff);
1007         break;
1008     case 0x28:  /* ASQ */
1009         n->bar.asq = data;
1010         trace_nvme_mmio_asqaddr(data);
1011         break;
1012     case 0x2c:  /* ASQ hi */
1013         n->bar.asq |= data << 32;
1014         trace_nvme_mmio_asqaddr_hi(data, n->bar.asq);
1015         break;
1016     case 0x30:  /* ACQ */
1017         trace_nvme_mmio_acqaddr(data);
1018         n->bar.acq = data;
1019         break;
1020     case 0x34:  /* ACQ hi */
1021         n->bar.acq |= data << 32;
1022         trace_nvme_mmio_acqaddr_hi(data, n->bar.acq);
1023         break;
1024     case 0x38:  /* CMBLOC */
1025         NVME_GUEST_ERR(nvme_ub_mmiowr_cmbloc_reserved,
1026                        "invalid write to reserved CMBLOC"
1027                        " when CMBSZ is zero, ignored");
1028         return;
1029     case 0x3C:  /* CMBSZ */
1030         NVME_GUEST_ERR(nvme_ub_mmiowr_cmbsz_readonly,
1031                        "invalid write to read only CMBSZ, ignored");
1032         return;
1033     default:
1034         NVME_GUEST_ERR(nvme_ub_mmiowr_invalid,
1035                        "invalid MMIO write,"
1036                        " offset=0x%"PRIx64", data=%"PRIx64"",
1037                        offset, data);
1038         break;
1039     }
1040 }
1041
1042 static uint64_t nvme_mmio_read(void *opaque, hwaddr addr, unsigned size)
1043 {
1044     NvmeCtrl *n = (NvmeCtrl *)opaque;
1045     uint8_t *ptr = (uint8_t *)&n->bar;
1046     uint64_t val = 0;
1047
1048     if (unlikely(addr & (sizeof(uint32_t) - 1))) {
1049         NVME_GUEST_ERR(nvme_ub_mmiord_misaligned32,
1050                        "MMIO read not 32-bit aligned,"
1051                        " offset=0x%"PRIx64"", addr);
1052         /* should RAZ, fall through for now */
1053     } else if (unlikely(size < sizeof(uint32_t))) {
1054         NVME_GUEST_ERR(nvme_ub_mmiord_toosmall,
1055                        "MMIO read smaller than 32-bits,"
1056                        " offset=0x%"PRIx64"", addr);
1057         /* should RAZ, fall through for now */
1058     }
1059
1060     if (addr < sizeof(n->bar)) {
1061         memcpy(&val, ptr + addr, size);
1062     } else {
1063         NVME_GUEST_ERR(nvme_ub_mmiord_invalid_ofs,
1064                        "MMIO read beyond last register,"
1065                        " offset=0x%"PRIx64", returning 0", addr);
1066     }
1067
1068     return val;
1069 }
1070
1071 static void nvme_process_db(NvmeCtrl *n, hwaddr addr, int val)
1072 {
1073     uint32_t qid;
1074
1075     if (unlikely(addr & ((1 << 2) - 1))) {
1076         NVME_GUEST_ERR(nvme_ub_db_wr_misaligned,
1077                        "doorbell write not 32-bit aligned,"
1078                        " offset=0x%"PRIx64", ignoring", addr);
1079         return;
1080     }
1081
1082     if (((addr - 0x1000) >> 2) & 1) {
1083         /* Completion queue doorbell write */
1084
1085         uint16_t new_head = val & 0xffff;
1086         int start_sqs;
1087         NvmeCQueue *cq;
1088
1089         qid = (addr - (0x1000 + (1 << 2))) >> 3;
1090         if (unlikely(nvme_check_cqid(n, qid))) {
1091             NVME_GUEST_ERR(nvme_ub_db_wr_invalid_cq,
1092                            "completion queue doorbell write"
1093                            " for nonexistent queue,"
1094                            " sqid=%"PRIu32", ignoring", qid);
1095             return;
1096         }
1097
1098         cq = n->cq[qid];
1099         if (unlikely(new_head >= cq->size)) {
1100             NVME_GUEST_ERR(nvme_ub_db_wr_invalid_cqhead,
1101                            "completion queue doorbell write value"
1102                            " beyond queue size, sqid=%"PRIu32","
1103                            " new_head=%"PRIu16", ignoring",
1104                            qid, new_head);
1105             return;
1106         }
1107
1108         start_sqs = nvme_cq_full(cq) ? 1 : 0;
1109         cq->head = new_head;
1110         if (start_sqs) {
1111             NvmeSQueue *sq;
1112             QTAILQ_FOREACH(sq, &cq->sq_list, entry) {
1113                 timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
1114             }
1115             timer_mod(cq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
1116         }
1117
1118         if (cq->tail == cq->head) {
1119             nvme_irq_deassert(n, cq);
1120         }
1121     } else {
1122         /* Submission queue doorbell write */
1123
1124         uint16_t new_tail = val & 0xffff;
1125         NvmeSQueue *sq;
1126
1127         qid = (addr - 0x1000) >> 3;
1128         if (unlikely(nvme_check_sqid(n, qid))) {
1129             NVME_GUEST_ERR(nvme_ub_db_wr_invalid_sq,
1130                            "submission queue doorbell write"
1131                            " for nonexistent queue,"
1132                            " sqid=%"PRIu32", ignoring", qid);
1133             return;
1134         }
1135
1136         sq = n->sq[qid];
1137         if (unlikely(new_tail >= sq->size)) {
1138             NVME_GUEST_ERR(nvme_ub_db_wr_invalid_sqtail,
1139                            "submission queue doorbell write value"
1140                            " beyond queue size, sqid=%"PRIu32","
1141                            " new_tail=%"PRIu16", ignoring",
1142                            qid, new_tail);
1143             return;
1144         }
1145
1146         sq->tail = new_tail;
1147         timer_mod(sq->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 500);
1148     }
1149 }
1150
1151 static void nvme_mmio_write(void *opaque, hwaddr addr, uint64_t data,
1152     unsigned size)
1153 {
1154     NvmeCtrl *n = (NvmeCtrl *)opaque;
1155     if (addr < sizeof(n->bar)) {
1156         nvme_write_bar(n, addr, data, size);
1157     } else if (addr >= 0x1000) {
1158         nvme_process_db(n, addr, data);
1159     }
1160 }
1161
1162 static const MemoryRegionOps nvme_mmio_ops = {
1163     .read = nvme_mmio_read,
1164     .write = nvme_mmio_write,
1165     .endianness = DEVICE_LITTLE_ENDIAN,
1166     .impl = {
1167         .min_access_size = 2,
1168         .max_access_size = 8,
1169     },
1170 };
1171
1172 static void nvme_cmb_write(void *opaque, hwaddr addr, uint64_t data,
1173     unsigned size)
1174 {
1175     NvmeCtrl *n = (NvmeCtrl *)opaque;
1176     memcpy(&n->cmbuf[addr], &data, size);
1177 }
1178
1179 static uint64_t nvme_cmb_read(void *opaque, hwaddr addr, unsigned size)
1180 {
1181     uint64_t val;
1182     NvmeCtrl *n = (NvmeCtrl *)opaque;
1183
1184     memcpy(&val, &n->cmbuf[addr], size);
1185     return val;
1186 }
1187
1188 static const MemoryRegionOps nvme_cmb_ops = {
1189     .read = nvme_cmb_read,
1190     .write = nvme_cmb_write,
1191     .endianness = DEVICE_LITTLE_ENDIAN,
1192     .impl = {
1193         .min_access_size = 2,
1194         .max_access_size = 8,
1195     },
1196 };
1197
1198 static void nvme_realize(PCIDevice *pci_dev, Error **errp)
1199 {
1200     NvmeCtrl *n = NVME(pci_dev);
1201     NvmeIdCtrl *id = &n->id_ctrl;
1202
1203     int i;
1204     int64_t bs_size;
1205     uint8_t *pci_conf;
1206
1207     if (!n->conf.blk) {
1208         error_setg(errp, "drive property not set");
1209         return;
1210     }
1211
1212     bs_size = blk_getlength(n->conf.blk);
1213     if (bs_size < 0) {
1214         error_setg(errp, "could not get backing file size");
1215         return;
1216     }
1217
1218     blkconf_serial(&n->conf, &n->serial);
1219     if (!n->serial) {
1220         error_setg(errp, "serial property not set");
1221         return;
1222     }
1223     blkconf_blocksizes(&n->conf);
1224     if (!blkconf_apply_backend_options(&n->conf, blk_is_read_only(n->conf.blk),
1225                                        false, errp)) {
1226         return;
1227     }
1228
1229     pci_conf = pci_dev->config;
1230     pci_conf[PCI_INTERRUPT_PIN] = 1;
1231     pci_config_set_prog_interface(pci_dev->config, 0x2);
1232     pci_config_set_class(pci_dev->config, PCI_CLASS_STORAGE_EXPRESS);
1233     pcie_endpoint_cap_init(&n->parent_obj, 0x80);
1234
1235     n->num_namespaces = 1;
1236     n->num_queues = 64;
1237     n->reg_size = pow2ceil(0x1004 + 2 * (n->num_queues + 1) * 4);
1238     n->ns_size = bs_size / (uint64_t)n->num_namespaces;
1239
1240     n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
1241     n->sq = g_new0(NvmeSQueue *, n->num_queues);
1242     n->cq = g_new0(NvmeCQueue *, n->num_queues);
1243
1244     memory_region_init_io(&n->iomem, OBJECT(n), &nvme_mmio_ops, n,
1245                           "nvme", n->reg_size);
1246     pci_register_bar(&n->parent_obj, 0,
1247         PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64,
1248         &n->iomem);
1249     msix_init_exclusive_bar(&n->parent_obj, n->num_queues, 4, NULL);
1250
1251     id->vid = cpu_to_le16(pci_get_word(pci_conf + PCI_VENDOR_ID));
1252     id->ssvid = cpu_to_le16(pci_get_word(pci_conf + PCI_SUBSYSTEM_VENDOR_ID));
1253     strpadcpy((char *)id->mn, sizeof(id->mn), "QEMU NVMe Ctrl", ' ');
1254     strpadcpy((char *)id->fr, sizeof(id->fr), "1.0", ' ');
1255     strpadcpy((char *)id->sn, sizeof(id->sn), n->serial, ' ');
1256     id->rab = 6;
1257     id->ieee[0] = 0x00;
1258     id->ieee[1] = 0x02;
1259     id->ieee[2] = 0xb3;
1260     id->oacs = cpu_to_le16(0);
1261     id->frmw = 7 << 1;
1262     id->lpa = 1 << 0;
1263     id->sqes = (0x6 << 4) | 0x6;
1264     id->cqes = (0x4 << 4) | 0x4;
1265     id->nn = cpu_to_le32(n->num_namespaces);
1266     id->oncs = cpu_to_le16(NVME_ONCS_WRITE_ZEROS);
1267     id->psd[0].mp = cpu_to_le16(0x9c4);
1268     id->psd[0].enlat = cpu_to_le32(0x10);
1269     id->psd[0].exlat = cpu_to_le32(0x4);
1270     if (blk_enable_write_cache(n->conf.blk)) {
1271         id->vwc = 1;
1272     }
1273
1274     n->bar.cap = 0;
1275     NVME_CAP_SET_MQES(n->bar.cap, 0x7ff);
1276     NVME_CAP_SET_CQR(n->bar.cap, 1);
1277     NVME_CAP_SET_AMS(n->bar.cap, 1);
1278     NVME_CAP_SET_TO(n->bar.cap, 0xf);
1279     NVME_CAP_SET_CSS(n->bar.cap, 1);
1280     NVME_CAP_SET_MPSMAX(n->bar.cap, 4);
1281
1282     n->bar.vs = 0x00010200;
1283     n->bar.intmc = n->bar.intms = 0;
1284
1285     if (n->cmb_size_mb) {
1286
1287         NVME_CMBLOC_SET_BIR(n->bar.cmbloc, 2);
1288         NVME_CMBLOC_SET_OFST(n->bar.cmbloc, 0);
1289
1290         NVME_CMBSZ_SET_SQS(n->bar.cmbsz, 1);
1291         NVME_CMBSZ_SET_CQS(n->bar.cmbsz, 0);
1292         NVME_CMBSZ_SET_LISTS(n->bar.cmbsz, 0);
1293         NVME_CMBSZ_SET_RDS(n->bar.cmbsz, 1);
1294         NVME_CMBSZ_SET_WDS(n->bar.cmbsz, 1);
1295         NVME_CMBSZ_SET_SZU(n->bar.cmbsz, 2); /* MBs */
1296         NVME_CMBSZ_SET_SZ(n->bar.cmbsz, n->cmb_size_mb);
1297
1298         n->cmbloc = n->bar.cmbloc;
1299         n->cmbsz = n->bar.cmbsz;
1300
1301         n->cmbuf = g_malloc0(NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
1302         memory_region_init_io(&n->ctrl_mem, OBJECT(n), &nvme_cmb_ops, n,
1303                               "nvme-cmb", NVME_CMBSZ_GETSIZE(n->bar.cmbsz));
1304         pci_register_bar(&n->parent_obj, NVME_CMBLOC_BIR(n->bar.cmbloc),
1305             PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64 |
1306             PCI_BASE_ADDRESS_MEM_PREFETCH, &n->ctrl_mem);
1307
1308     }
1309
1310     for (i = 0; i < n->num_namespaces; i++) {
1311         NvmeNamespace *ns = &n->namespaces[i];
1312         NvmeIdNs *id_ns = &ns->id_ns;
1313         id_ns->nsfeat = 0;
1314         id_ns->nlbaf = 0;
1315         id_ns->flbas = 0;
1316         id_ns->mc = 0;
1317         id_ns->dpc = 0;
1318         id_ns->dps = 0;
1319         id_ns->lbaf[0].ds = BDRV_SECTOR_BITS;
1320         id_ns->ncap  = id_ns->nuse = id_ns->nsze =
1321             cpu_to_le64(n->ns_size >>
1322                 id_ns->lbaf[NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas)].ds);
1323     }
1324 }
1325
1326 static void nvme_exit(PCIDevice *pci_dev)
1327 {
1328     NvmeCtrl *n = NVME(pci_dev);
1329
1330     nvme_clear_ctrl(n);
1331     g_free(n->namespaces);
1332     g_free(n->cq);
1333     g_free(n->sq);
1334     if (n->cmbsz) {
1335         memory_region_unref(&n->ctrl_mem);
1336     }
1337
1338     msix_uninit_exclusive_bar(pci_dev);
1339 }
1340
1341 static Property nvme_props[] = {
1342     DEFINE_BLOCK_PROPERTIES(NvmeCtrl, conf),
1343     DEFINE_PROP_STRING("serial", NvmeCtrl, serial),
1344     DEFINE_PROP_UINT32("cmb_size_mb", NvmeCtrl, cmb_size_mb, 0),
1345     DEFINE_PROP_END_OF_LIST(),
1346 };
1347
1348 static const VMStateDescription nvme_vmstate = {
1349     .name = "nvme",
1350     .unmigratable = 1,
1351 };
1352
1353 static void nvme_class_init(ObjectClass *oc, void *data)
1354 {
1355     DeviceClass *dc = DEVICE_CLASS(oc);
1356     PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
1357
1358     pc->realize = nvme_realize;
1359     pc->exit = nvme_exit;
1360     pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
1361     pc->vendor_id = PCI_VENDOR_ID_INTEL;
1362     pc->device_id = 0x5845;
1363     pc->revision = 2;
1364
1365     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
1366     dc->desc = "Non-Volatile Memory Express";
1367     dc->props = nvme_props;
1368     dc->vmsd = &nvme_vmstate;
1369 }
1370
1371 static void nvme_instance_init(Object *obj)
1372 {
1373     NvmeCtrl *s = NVME(obj);
1374
1375     device_add_bootindex_property(obj, &s->conf.bootindex,
1376                                   "bootindex", "/namespace@1,0",
1377                                   DEVICE(obj), &error_abort);
1378 }
1379
1380 static const TypeInfo nvme_info = {
1381     .name          = "nvme",
1382     .parent        = TYPE_PCI_DEVICE,
1383     .instance_size = sizeof(NvmeCtrl),
1384     .class_init    = nvme_class_init,
1385     .instance_init = nvme_instance_init,
1386     .interfaces = (InterfaceInfo[]) {
1387         { INTERFACE_PCIE_DEVICE },
1388         { }
1389     },
1390 };
1391
1392 static void nvme_register_types(void)
1393 {
1394     type_register_static(&nvme_info);
1395 }
1396
1397 type_init(nvme_register_types)