#include "qapi/qmp/qdict.h"
#include "qapi/qmp/qstring.h"
#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
+#include "qemu/module.h"
#include "qemu/cutils.h"
#include "qemu/option.h"
#include "qemu/vfio-helpers.h"
uint8_t reserved1[0xec0];
uint8_t cmd_set_specfic[0x100];
uint32_t doorbells[];
-} QEMU_PACKED NVMeRegs;
+} NVMeRegs;
QEMU_BUILD_BUG_ON(offsetof(NVMeRegs, doorbells) != 0x1000);
size_t doorbell_scale;
bool write_cache_supported;
EventNotifier irq_notifier;
+
uint64_t nsze; /* Namespace size reported by identify command */
int nsid; /* The namespace id to read/write data. */
+ int blkshift;
+
uint64_t max_transfer;
- int plugged;
+ bool plugged;
CoMutex dma_map_lock;
CoQueue dma_flush_queue;
/* Total size of mapped qiov, accessed under dma_map_lock */
int dma_map_count;
+
+ /* PCI address (required for nvme_refresh_filename()) */
+ char *device;
} BDRVNVMeState;
#define NVME_BLOCK_OPT_DEVICE "device"
error_propagate(errp, local_err);
goto fail;
}
- q->cq.doorbell = &s->regs->doorbells[idx * 2 * s->doorbell_scale + 1];
+ q->cq.doorbell = &s->regs->doorbells[(idx * 2 + 1) * s->doorbell_scale];
return q;
fail:
while (q->inflight) {
int16_t cid;
c = (NvmeCqe *)&q->cq.queue[q->cq.head * NVME_CQ_ENTRY_BYTES];
- if (!c->cid || (le16_to_cpu(c->status) & 0x1) == q->cq_phase) {
+ if ((le16_to_cpu(c->status) & 0x1) == q->cq_phase) {
break;
}
q->cq.head = (q->cq.head + 1) % NVME_QUEUE_SIZE;
qemu_mutex_unlock(&q->lock);
req.cb(req.opaque, nvme_translate_error(c));
qemu_mutex_lock(&q->lock);
- c->cid = cpu_to_le16(0);
q->inflight--;
- /* Flip Phase Tag bit. */
- c->status = cpu_to_le16(le16_to_cpu(c->status) ^ 0x1);
progress = true;
}
if (progress) {
{
int *pret = opaque;
*pret = ret;
+ aio_wait_kick();
}
static int nvme_cmd_sync(BlockDriverState *bs, NVMeQueuePair *q,
BDRVNVMeState *s = bs->opaque;
NvmeIdCtrl *idctrl;
NvmeIdNs *idns;
+ NvmeLBAF *lbaf;
uint8_t *resp;
int r;
uint64_t iova;
}
s->nsze = le64_to_cpu(idns->nsze);
+ lbaf = &idns->lbaf[NVME_ID_NS_FLBAS_INDEX(idns->flbas)];
+
+ if (lbaf->ms) {
+ error_setg(errp, "Namespaces with metadata are not yet supported");
+ goto out;
+ }
+
+ if (lbaf->ds < BDRV_SECTOR_BITS || lbaf->ds > 12 ||
+ (1 << lbaf->ds) > s->page_size)
+ {
+ error_setg(errp, "Namespace has unsupported block size (2^%d)",
+ lbaf->ds);
+ goto out;
+ }
+ s->blkshift = lbaf->ds;
out:
qemu_vfio_dma_unmap(s->vfio, resp);
qemu_vfree(resp);
BDRVNVMeState *s = container_of(n, BDRVNVMeState, irq_notifier);
trace_nvme_handle_event(s);
- aio_context_acquire(s->aio_context);
event_notifier_test_and_clear(n);
nvme_poll_queues(s);
- aio_context_release(s->aio_context);
}
static bool nvme_add_io_queue(BlockDriverState *bs, Error **errp)
qemu_co_mutex_init(&s->dma_map_lock);
qemu_co_queue_init(&s->dma_flush_queue);
+ s->device = g_strdup(device);
s->nsid = namespace;
s->aio_context = bdrv_get_aio_context(bs);
ret = event_notifier_init(&s->irq_notifier, 0);
s->vfio = qemu_vfio_open_pci(device, errp);
if (!s->vfio) {
ret = -EINVAL;
- goto fail;
+ goto out;
}
s->regs = qemu_vfio_pci_map_bar(s->vfio, 0, 0, NVME_BAR_SIZE, errp);
if (!s->regs) {
ret = -EINVAL;
- goto fail;
+ goto out;
}
/* Perform initialize sequence as described in NVMe spec "7.6.1
if (!(cap & (1ULL << 37))) {
error_setg(errp, "Device doesn't support NVMe command set");
ret = -EINVAL;
- goto fail;
+ goto out;
}
s->page_size = MAX(4096, 1 << (12 + ((cap >> 48) & 0xF)));
PRId64 " ms)",
timeout_ms);
ret = -ETIMEDOUT;
- goto fail;
+ goto out;
}
}
/* Set up admin queue. */
s->queues = g_new(NVMeQueuePair *, 1);
- s->nr_queues = 1;
s->queues[0] = nvme_create_queue_pair(bs, 0, NVME_QUEUE_SIZE, errp);
if (!s->queues[0]) {
ret = -EINVAL;
- goto fail;
+ goto out;
}
+ s->nr_queues = 1;
QEMU_BUILD_BUG_ON(NVME_QUEUE_SIZE & 0xF000);
s->regs->aqa = cpu_to_le32((NVME_QUEUE_SIZE << 16) | NVME_QUEUE_SIZE);
s->regs->asq = cpu_to_le64(s->queues[0]->sq.iova);
PRId64 " ms)",
timeout_ms);
ret = -ETIMEDOUT;
- goto fail_queue;
+ goto out;
}
}
ret = qemu_vfio_pci_init_irq(s->vfio, &s->irq_notifier,
VFIO_PCI_MSIX_IRQ_INDEX, errp);
if (ret) {
- goto fail_queue;
+ goto out;
}
aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
false, nvme_handle_event, nvme_poll_cb);
if (local_err) {
error_propagate(errp, local_err);
ret = -EIO;
- goto fail_handler;
+ goto out;
}
/* Set up command queues. */
if (!nvme_add_io_queue(bs, errp)) {
ret = -EIO;
- goto fail_handler;
}
- return 0;
-
-fail_handler:
- aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
- false, NULL, NULL);
-fail_queue:
- nvme_free_queue_pair(bs, s->queues[0]);
-fail:
- g_free(s->queues);
- if (s->regs) {
- qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->regs, 0, NVME_BAR_SIZE);
- }
- if (s->vfio) {
- qemu_vfio_close(s->vfio);
- }
- event_notifier_cleanup(&s->irq_notifier);
+out:
+ /* Cleaning up is done in nvme_file_open() upon error. */
return ret;
}
unsigned long ns;
const char *slash = strchr(tmp, '/');
if (!slash) {
- qdict_put(options, NVME_BLOCK_OPT_DEVICE,
- qstring_from_str(tmp));
+ qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, tmp);
return;
}
device = g_strndup(tmp, slash - tmp);
- qdict_put(options, NVME_BLOCK_OPT_DEVICE, qstring_from_str(device));
+ qdict_put_str(options, NVME_BLOCK_OPT_DEVICE, device);
g_free(device);
namespace = slash + 1;
if (*namespace && qemu_strtoul(namespace, NULL, 10, &ns)) {
namespace);
return;
}
- qdict_put(options, NVME_BLOCK_OPT_NAMESPACE,
- qstring_from_str(*namespace ? namespace : "1"));
+ qdict_put_str(options, NVME_BLOCK_OPT_NAMESPACE,
+ *namespace ? namespace : "1");
}
}
for (i = 0; i < s->nr_queues; ++i) {
nvme_free_queue_pair(bs, s->queues[i]);
}
+ g_free(s->queues);
aio_set_event_notifier(bdrv_get_aio_context(bs), &s->irq_notifier,
false, NULL, NULL);
+ event_notifier_cleanup(&s->irq_notifier);
qemu_vfio_pci_unmap_bar(s->vfio, 0, (void *)s->regs, 0, NVME_BAR_SIZE);
qemu_vfio_close(s->vfio);
+
+ g_free(s->device);
}
static int nvme_file_open(BlockDriverState *bs, QDict *options, int flags,
static int64_t nvme_getlength(BlockDriverState *bs)
{
BDRVNVMeState *s = bs->opaque;
+ return s->nsze << s->blkshift;
+}
- return s->nsze << BDRV_SECTOR_BITS;
+static uint32_t nvme_get_blocksize(BlockDriverState *bs)
+{
+ BDRVNVMeState *s = bs->opaque;
+ assert(s->blkshift >= BDRV_SECTOR_BITS && s->blkshift <= 12);
+ return UINT32_C(1) << s->blkshift;
+}
+
+static int nvme_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
+{
+ uint32_t blocksize = nvme_get_blocksize(bs);
+ bsz->phys = blocksize;
+ bsz->log = blocksize;
+ return 0;
}
/* Called with s->dma_map_lock */
}
for (j = 0; j < qiov->iov[i].iov_len / s->page_size; j++) {
- pagelist[entries++] = iova + j * s->page_size;
+ pagelist[entries++] = cpu_to_le64(iova + j * s->page_size);
}
trace_nvme_cmd_map_qiov_iov(s, i, qiov->iov[i].iov_base,
qiov->iov[i].iov_len / s->page_size);
case 0:
abort();
case 1:
- cmd->prp1 = cpu_to_le64(pagelist[0]);
+ cmd->prp1 = pagelist[0];
cmd->prp2 = 0;
break;
case 2:
- cmd->prp1 = cpu_to_le64(pagelist[0]);
- cmd->prp2 = cpu_to_le64(pagelist[1]);;
+ cmd->prp1 = pagelist[0];
+ cmd->prp2 = pagelist[1];
break;
default:
- cmd->prp1 = cpu_to_le64(pagelist[0]);
- cmd->prp2 = cpu_to_le64(req->prp_list_iova);
- for (i = 0; i < entries - 1; ++i) {
- pagelist[i] = cpu_to_le64(pagelist[i + 1]);
- }
- pagelist[entries - 1] = 0;
+ cmd->prp1 = pagelist[0];
+ cmd->prp2 = cpu_to_le64(req->prp_list_iova + sizeof(uint64_t));
break;
}
trace_nvme_cmd_map_qiov(s, cmd, req, qiov, entries);
BDRVNVMeState *s = bs->opaque;
NVMeQueuePair *ioq = s->queues[1];
NVMeRequest *req;
- uint32_t cdw12 = (((bytes >> BDRV_SECTOR_BITS) - 1) & 0xFFFF) |
+
+ uint32_t cdw12 = (((bytes >> s->blkshift) - 1) & 0xFFFF) |
(flags & BDRV_REQ_FUA ? 1 << 30 : 0);
NvmeCmd cmd = {
.opcode = is_write ? NVME_CMD_WRITE : NVME_CMD_READ,
.nsid = cpu_to_le32(s->nsid),
- .cdw10 = cpu_to_le32((offset >> BDRV_SECTOR_BITS) & 0xFFFFFFFF),
- .cdw11 = cpu_to_le32(((offset >> BDRV_SECTOR_BITS) >> 32) & 0xFFFFFFFF),
+ .cdw10 = cpu_to_le32((offset >> s->blkshift) & 0xFFFFFFFF),
+ .cdw11 = cpu_to_le32(((offset >> s->blkshift) >> 32) & 0xFFFFFFFF),
.cdw12 = cpu_to_le32(cdw12),
};
NVMeCoData data = {
return 0;
}
-static void nvme_refresh_filename(BlockDriverState *bs, QDict *opts)
+static void nvme_refresh_filename(BlockDriverState *bs)
{
- QINCREF(opts);
- qdict_del(opts, "filename");
-
- if (!qdict_size(opts)) {
- snprintf(bs->exact_filename, sizeof(bs->exact_filename), "%s://",
- bs->drv->format_name);
- }
+ BDRVNVMeState *s = bs->opaque;
- qdict_put(opts, "driver", qstring_from_str(bs->drv->format_name));
- bs->full_open_options = opts;
+ snprintf(bs->exact_filename, sizeof(bs->exact_filename), "nvme://%s/%i",
+ s->device, s->nsid);
}
static void nvme_refresh_limits(BlockDriverState *bs, Error **errp)
static void nvme_aio_plug(BlockDriverState *bs)
{
BDRVNVMeState *s = bs->opaque;
- s->plugged++;
+ assert(!s->plugged);
+ s->plugged = true;
}
static void nvme_aio_unplug(BlockDriverState *bs)
int i;
BDRVNVMeState *s = bs->opaque;
assert(s->plugged);
- if (!--s->plugged) {
- for (i = 1; i < s->nr_queues; i++) {
- NVMeQueuePair *q = s->queues[i];
- qemu_mutex_lock(&q->lock);
- nvme_kick(s, q);
- nvme_process_completion(s, q);
- qemu_mutex_unlock(&q->lock);
- }
+ s->plugged = false;
+ for (i = 1; i < s->nr_queues; i++) {
+ NVMeQueuePair *q = s->queues[i];
+ qemu_mutex_lock(&q->lock);
+ nvme_kick(s, q);
+ nvme_process_completion(s, q);
+ qemu_mutex_unlock(&q->lock);
}
}
qemu_vfio_dma_unmap(s->vfio, host);
}
+static const char *const nvme_strong_runtime_opts[] = {
+ NVME_BLOCK_OPT_DEVICE,
+ NVME_BLOCK_OPT_NAMESPACE,
+
+ NULL
+};
+
static BlockDriver bdrv_nvme = {
.format_name = "nvme",
.protocol_name = "nvme",
.bdrv_file_open = nvme_file_open,
.bdrv_close = nvme_close,
.bdrv_getlength = nvme_getlength,
+ .bdrv_probe_blocksizes = nvme_probe_blocksizes,
.bdrv_co_preadv = nvme_co_preadv,
.bdrv_co_pwritev = nvme_co_pwritev,
.bdrv_refresh_filename = nvme_refresh_filename,
.bdrv_refresh_limits = nvme_refresh_limits,
+ .strong_runtime_opts = nvme_strong_runtime_opts,
.bdrv_detach_aio_context = nvme_detach_aio_context,
.bdrv_attach_aio_context = nvme_attach_aio_context,
projects / qemu.git / blobdiff