[INTEL_GVT_HYPERVISOR_KVM] = "KVM",
};
- static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt,
- const char *name)
+ static struct intel_vgpu_type *
+ intel_gvt_find_vgpu_type(struct intel_gvt *gvt, unsigned int type_group_id)
{
- const char *driver_name =
- dev_driver_string(gvt->gt->i915->drm.dev);
- int i;
-
- name += strlen(driver_name) + 1;
- for (i = 0; i < gvt->num_types; i++) {
- struct intel_vgpu_type *t = &gvt->types[i];
-
- if (!strncmp(t->name, name, sizeof(t->name)))
- return t;
- }
-
- return NULL;
+ if (WARN_ON(type_group_id >= gvt->num_types))
+ return NULL;
+ return &gvt->types[type_group_id];
}
- static ssize_t available_instances_show(struct kobject *kobj,
- struct device *dev, char *buf)
+ static ssize_t available_instances_show(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr,
+ char *buf)
{
struct intel_vgpu_type *type;
unsigned int num = 0;
- void *gvt = kdev_to_i915(dev)->gvt;
+ void *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt;
- type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
+ type = intel_gvt_find_vgpu_type(gvt, mtype_get_type_group_id(mtype));
if (!type)
num = 0;
else
return sprintf(buf, "%u\n", num);
}
- static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
- char *buf)
+ static ssize_t device_api_show(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr, char *buf)
{
return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING);
}
- static ssize_t description_show(struct kobject *kobj, struct device *dev,
- char *buf)
+ static ssize_t description_show(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr, char *buf)
{
struct intel_vgpu_type *type;
- void *gvt = kdev_to_i915(dev)->gvt;
+ void *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt;
- type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
+ type = intel_gvt_find_vgpu_type(gvt, mtype_get_type_group_id(mtype));
if (!type)
return 0;
static void init_device_info(struct intel_gvt *gvt)
{
struct intel_gvt_device_info *info = &gvt->device_info;
- struct pci_dev *pdev = gvt->gt->i915->drm.pdev;
+ struct pci_dev *pdev = to_pci_dev(gvt->gt->i915->drm.dev);
info->max_support_vgpus = 8;
info->cfg_space_size = PCI_CFG_SPACE_EXP_SIZE;
info->msi_cap_offset = pdev->msi_cap;
}
+static void intel_gvt_test_and_emulate_vblank(struct intel_gvt *gvt)
+{
+ struct intel_vgpu *vgpu;
+ int id;
+
+ mutex_lock(&gvt->lock);
+ idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) {
+ if (test_and_clear_bit(INTEL_GVT_REQUEST_EMULATE_VBLANK + id,
+ (void *)&gvt->service_request)) {
+ if (vgpu->active)
+ intel_vgpu_emulate_vblank(vgpu);
+ }
+ }
+ mutex_unlock(&gvt->lock);
+}
+
static int gvt_service_thread(void *data)
{
struct intel_gvt *gvt = (struct intel_gvt *)data;
if (WARN_ONCE(ret, "service thread is waken up by signal.\n"))
continue;
- if (test_and_clear_bit(INTEL_GVT_REQUEST_EMULATE_VBLANK,
- (void *)&gvt->service_request))
- intel_gvt_emulate_vblank(gvt);
+ intel_gvt_test_and_emulate_vblank(gvt);
if (test_bit(INTEL_GVT_REQUEST_SCHED,
(void *)&gvt->service_request) ||
intel_gvt_clean_sched_policy(gvt);
intel_gvt_clean_workload_scheduler(gvt);
intel_gvt_clean_gtt(gvt);
- intel_gvt_clean_irq(gvt);
intel_gvt_free_firmware(gvt);
intel_gvt_clean_mmio_info(gvt);
idr_destroy(&gvt->vgpu_idr);
ret = intel_gvt_init_gtt(gvt);
if (ret)
- goto out_clean_irq;
+ goto out_free_firmware;
ret = intel_gvt_init_workload_scheduler(gvt);
if (ret)
intel_gvt_debugfs_init(gvt);
gvt_dbg_core("gvt device initialization is done\n");
- intel_gvt_host.dev = &i915->drm.pdev->dev;
+ intel_gvt_host.dev = i915->drm.dev;
intel_gvt_host.initialized = true;
return 0;
intel_gvt_clean_workload_scheduler(gvt);
out_clean_gtt:
intel_gvt_clean_gtt(gvt);
-out_clean_irq:
- intel_gvt_clean_irq(gvt);
out_free_firmware:
intel_gvt_free_firmware(gvt);
out_clean_mmio_info:
struct intel_vgpu_i2c_edid i2c_edid;
struct intel_vgpu_port ports[I915_MAX_PORTS];
struct intel_vgpu_sbi sbi;
+ enum port port_num;
};
struct vgpu_sched_ctl {
struct list_head dmabuf_obj_list_head;
struct mutex dmabuf_lock;
struct idr object_idr;
+ struct intel_vgpu_vblank_timer vblank_timer;
u32 scan_nonprivbb;
};
}
enum {
- INTEL_GVT_REQUEST_EMULATE_VBLANK = 0,
-
/* Scheduling trigger by timer */
- INTEL_GVT_REQUEST_SCHED = 1,
+ INTEL_GVT_REQUEST_SCHED = 0,
/* Scheduling trigger by event */
- INTEL_GVT_REQUEST_EVENT_SCHED = 2,
+ INTEL_GVT_REQUEST_EVENT_SCHED = 1,
+
+ /* per-vGPU vblank emulation request */
+ INTEL_GVT_REQUEST_EMULATE_VBLANK = 2,
+ INTEL_GVT_REQUEST_EMULATE_VBLANK_MAX = INTEL_GVT_REQUEST_EMULATE_VBLANK
+ + GVT_MAX_VGPU,
};
static inline void intel_gvt_request_service(struct intel_gvt *gvt,
void (*vgpu_reset)(struct intel_vgpu *);
void (*vgpu_activate)(struct intel_vgpu *);
void (*vgpu_deactivate)(struct intel_vgpu *);
- struct intel_vgpu_type *(*gvt_find_vgpu_type)(struct intel_gvt *gvt,
- const char *name);
+ struct intel_vgpu_type *(*gvt_find_vgpu_type)(
+ struct intel_gvt *gvt, unsigned int type_group_id);
bool (*get_gvt_attrs)(struct attribute_group ***intel_vgpu_type_groups);
int (*vgpu_query_plane)(struct intel_vgpu *vgpu, void *);
int (*vgpu_get_dmabuf)(struct intel_vgpu *vgpu, unsigned int);
static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
dma_addr_t *dma_addr, unsigned long size)
{
- struct device *dev = &vgpu->gvt->gt->i915->drm.pdev->dev;
+ struct device *dev = vgpu->gvt->gt->i915->drm.dev;
struct page *page = NULL;
int ret;
static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
dma_addr_t dma_addr, unsigned long size)
{
- struct device *dev = &vgpu->gvt->gt->i915->drm.pdev->dev;
+ struct device *dev = vgpu->gvt->gt->i915->drm.dev;
dma_unmap_page(dev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
gvt_unpin_guest_page(vgpu, gfn, size);
vfio_device_put(vdev->vfio_device);
}
- static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
+ static int intel_vgpu_create(struct mdev_device *mdev)
{
struct intel_vgpu *vgpu = NULL;
struct intel_vgpu_type *type;
pdev = mdev_parent_dev(mdev);
gvt = kdev_to_i915(pdev)->gvt;
- type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj));
+ type = intel_gvt_ops->gvt_find_vgpu_type(gvt,
+ mdev_get_type_group_id(mdev));
if (!type) {
- gvt_vgpu_err("failed to find type %s to create\n",
- kobject_name(kobj));
ret = -EINVAL;
goto out;
}
matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
struct ap_matrix_mdev, pqap_hook);
+ /*
+ * If the KVM pointer is in the process of being set, wait until the
+ * process has completed.
+ */
+ wait_event_cmd(matrix_mdev->wait_for_kvm,
+ !matrix_mdev->kvm_busy,
+ mutex_unlock(&matrix_dev->lock),
+ mutex_lock(&matrix_dev->lock));
+
+ /* If the there is no guest using the mdev, there is nothing to do */
+ if (!matrix_mdev->kvm)
+ goto out_unlock;
+
q = vfio_ap_get_queue(matrix_mdev, apqn);
if (!q)
goto out_unlock;
matrix->adm_max = info->apxa ? info->Nd : 15;
}
- static int vfio_ap_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
+ static int vfio_ap_mdev_create(struct mdev_device *mdev)
{
struct ap_matrix_mdev *matrix_mdev;
matrix_mdev->mdev = mdev;
vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
+ init_waitqueue_head(&matrix_mdev->wait_for_kvm);
mdev_set_drvdata(mdev, matrix_mdev);
matrix_mdev->pqap_hook.hook = handle_pqap;
matrix_mdev->pqap_hook.owner = THIS_MODULE;
{
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
- if (matrix_mdev->kvm)
+ mutex_lock(&matrix_dev->lock);
+
+ /*
+ * If the KVM pointer is in flux or the guest is running, disallow
+ * un-assignment of control domain.
+ */
+ if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
+ mutex_unlock(&matrix_dev->lock);
return -EBUSY;
+ }
- mutex_lock(&matrix_dev->lock);
vfio_ap_mdev_reset_queues(mdev);
list_del(&matrix_mdev->node);
- mutex_unlock(&matrix_dev->lock);
-
kfree(matrix_mdev);
mdev_set_drvdata(mdev, NULL);
atomic_inc(&matrix_dev->available_instances);
+ mutex_unlock(&matrix_dev->lock);
return 0;
}
- static ssize_t name_show(struct kobject *kobj, struct device *dev, char *buf)
+ static ssize_t name_show(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr, char *buf)
{
return sprintf(buf, "%s\n", VFIO_AP_MDEV_NAME_HWVIRT);
}
static MDEV_TYPE_ATTR_RO(name);
- static ssize_t available_instances_show(struct kobject *kobj,
- struct device *dev, char *buf)
+ static ssize_t available_instances_show(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr,
+ char *buf)
{
return sprintf(buf, "%d\n",
atomic_read(&matrix_dev->available_instances));
static MDEV_TYPE_ATTR_RO(available_instances);
- static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
- char *buf)
+ static ssize_t device_api_show(struct mdev_type *mtype,
+ struct mdev_type_attribute *attr, char *buf)
{
return sprintf(buf, "%s\n", VFIO_DEVICE_API_AP_STRING);
}
struct mdev_device *mdev = mdev_from_dev(dev);
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
- /* If the guest is running, disallow assignment of adapter */
- if (matrix_mdev->kvm)
- return -EBUSY;
+ mutex_lock(&matrix_dev->lock);
+
+ /*
+ * If the KVM pointer is in flux or the guest is running, disallow
+ * un-assignment of adapter
+ */
+ if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
+ ret = -EBUSY;
+ goto done;
+ }
ret = kstrtoul(buf, 0, &apid);
if (ret)
- return ret;
+ goto done;
- if (apid > matrix_mdev->matrix.apm_max)
- return -ENODEV;
+ if (apid > matrix_mdev->matrix.apm_max) {
+ ret = -ENODEV;
+ goto done;
+ }
/*
* Set the bit in the AP mask (APM) corresponding to the AP adapter
* number (APID). The bits in the mask, from most significant to least
* significant bit, correspond to APIDs 0-255.
*/
- mutex_lock(&matrix_dev->lock);
-
ret = vfio_ap_mdev_verify_queues_reserved_for_apid(matrix_mdev, apid);
if (ret)
goto done;
struct mdev_device *mdev = mdev_from_dev(dev);
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
- /* If the guest is running, disallow un-assignment of adapter */
- if (matrix_mdev->kvm)
- return -EBUSY;
+ mutex_lock(&matrix_dev->lock);
+
+ /*
+ * If the KVM pointer is in flux or the guest is running, disallow
+ * un-assignment of adapter
+ */
+ if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
+ ret = -EBUSY;
+ goto done;
+ }
ret = kstrtoul(buf, 0, &apid);
if (ret)
- return ret;
+ goto done;
- if (apid > matrix_mdev->matrix.apm_max)
- return -ENODEV;
+ if (apid > matrix_mdev->matrix.apm_max) {
+ ret = -ENODEV;
+ goto done;
+ }
- mutex_lock(&matrix_dev->lock);
clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
+ ret = count;
+done:
mutex_unlock(&matrix_dev->lock);
-
- return count;
+ return ret;
}
static DEVICE_ATTR_WO(unassign_adapter);
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
unsigned long max_apqi = matrix_mdev->matrix.aqm_max;
- /* If the guest is running, disallow assignment of domain */
- if (matrix_mdev->kvm)
- return -EBUSY;
+ mutex_lock(&matrix_dev->lock);
+
+ /*
+ * If the KVM pointer is in flux or the guest is running, disallow
+ * assignment of domain
+ */
+ if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
+ ret = -EBUSY;
+ goto done;
+ }
ret = kstrtoul(buf, 0, &apqi);
if (ret)
- return ret;
- if (apqi > max_apqi)
- return -ENODEV;
-
- mutex_lock(&matrix_dev->lock);
+ goto done;
+ if (apqi > max_apqi) {
+ ret = -ENODEV;
+ goto done;
+ }
ret = vfio_ap_mdev_verify_queues_reserved_for_apqi(matrix_mdev, apqi);
if (ret)
struct mdev_device *mdev = mdev_from_dev(dev);
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
- /* If the guest is running, disallow un-assignment of domain */
- if (matrix_mdev->kvm)
- return -EBUSY;
+ mutex_lock(&matrix_dev->lock);
+
+ /*
+ * If the KVM pointer is in flux or the guest is running, disallow
+ * un-assignment of domain
+ */
+ if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
+ ret = -EBUSY;
+ goto done;
+ }
ret = kstrtoul(buf, 0, &apqi);
if (ret)
- return ret;
+ goto done;
- if (apqi > matrix_mdev->matrix.aqm_max)
- return -ENODEV;
+ if (apqi > matrix_mdev->matrix.aqm_max) {
+ ret = -ENODEV;
+ goto done;
+ }
- mutex_lock(&matrix_dev->lock);
clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);
- mutex_unlock(&matrix_dev->lock);
+ ret = count;
- return count;
+done:
+ mutex_unlock(&matrix_dev->lock);
+ return ret;
}
static DEVICE_ATTR_WO(unassign_domain);
struct mdev_device *mdev = mdev_from_dev(dev);
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
- /* If the guest is running, disallow assignment of control domain */
- if (matrix_mdev->kvm)
- return -EBUSY;
+ mutex_lock(&matrix_dev->lock);
+
+ /*
+ * If the KVM pointer is in flux or the guest is running, disallow
+ * assignment of control domain.
+ */
+ if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
+ ret = -EBUSY;
+ goto done;
+ }
ret = kstrtoul(buf, 0, &id);
if (ret)
- return ret;
+ goto done;
- if (id > matrix_mdev->matrix.adm_max)
- return -ENODEV;
+ if (id > matrix_mdev->matrix.adm_max) {
+ ret = -ENODEV;
+ goto done;
+ }
/* Set the bit in the ADM (bitmask) corresponding to the AP control
* domain number (id). The bits in the mask, from most significant to
* least significant, correspond to IDs 0 up to the one less than the
* number of control domains that can be assigned.
*/
- mutex_lock(&matrix_dev->lock);
set_bit_inv(id, matrix_mdev->matrix.adm);
+ ret = count;
+done:
mutex_unlock(&matrix_dev->lock);
-
- return count;
+ return ret;
}
static DEVICE_ATTR_WO(assign_control_domain);
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
unsigned long max_domid = matrix_mdev->matrix.adm_max;
- /* If the guest is running, disallow un-assignment of control domain */
- if (matrix_mdev->kvm)
- return -EBUSY;
+ mutex_lock(&matrix_dev->lock);
+
+ /*
+ * If the KVM pointer is in flux or the guest is running, disallow
+ * un-assignment of control domain.
+ */
+ if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
+ ret = -EBUSY;
+ goto done;
+ }
ret = kstrtoul(buf, 0, &domid);
if (ret)
- return ret;
- if (domid > max_domid)
- return -ENODEV;
+ goto done;
+ if (domid > max_domid) {
+ ret = -ENODEV;
+ goto done;
+ }
- mutex_lock(&matrix_dev->lock);
clear_bit_inv(domid, matrix_mdev->matrix.adm);
+ ret = count;
+done:
mutex_unlock(&matrix_dev->lock);
-
- return count;
+ return ret;
}
static DEVICE_ATTR_WO(unassign_control_domain);
* @matrix_mdev: a mediated matrix device
* @kvm: reference to KVM instance
*
- * Verifies no other mediated matrix device has @kvm and sets a reference to
- * it in @matrix_mdev->kvm.
+ * Sets all data for @matrix_mdev that are needed to manage AP resources
+ * for the guest whose state is represented by @kvm.
+ *
+ * Note: The matrix_dev->lock must be taken prior to calling
+ * this function; however, the lock will be temporarily released while the
+ * guest's AP configuration is set to avoid a potential lockdep splat.
+ * The kvm->lock is taken to set the guest's AP configuration which, under
+ * certain circumstances, will result in a circular lock dependency if this is
+ * done under the @matrix_mdev->lock.
*
* Return 0 if no other mediated matrix device has a reference to @kvm;
* otherwise, returns an -EPERM.
{
struct ap_matrix_mdev *m;
- list_for_each_entry(m, &matrix_dev->mdev_list, node) {
- if ((m != matrix_mdev) && (m->kvm == kvm))
- return -EPERM;
- }
+ if (kvm->arch.crypto.crycbd) {
+ list_for_each_entry(m, &matrix_dev->mdev_list, node) {
+ if (m != matrix_mdev && m->kvm == kvm)
+ return -EPERM;
+ }
- matrix_mdev->kvm = kvm;
- kvm_get_kvm(kvm);
- kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
+ kvm_get_kvm(kvm);
+ matrix_mdev->kvm_busy = true;
+ mutex_unlock(&matrix_dev->lock);
+ kvm_arch_crypto_set_masks(kvm,
+ matrix_mdev->matrix.apm,
+ matrix_mdev->matrix.aqm,
+ matrix_mdev->matrix.adm);
+ mutex_lock(&matrix_dev->lock);
+ kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
+ matrix_mdev->kvm = kvm;
+ matrix_mdev->kvm_busy = false;
+ wake_up_all(&matrix_mdev->wait_for_kvm);
+ }
return 0;
}
return NOTIFY_DONE;
}
+/**
+ * vfio_ap_mdev_unset_kvm
+ *
+ * @matrix_mdev: a matrix mediated device
+ *
+ * Performs clean-up of resources no longer needed by @matrix_mdev.
+ *
+ * Note: The matrix_dev->lock must be taken prior to calling
+ * this function; however, the lock will be temporarily released while the
+ * guest's AP configuration is cleared to avoid a potential lockdep splat.
+ * The kvm->lock is taken to clear the guest's AP configuration which, under
+ * certain circumstances, will result in a circular lock dependency if this is
+ * done under the @matrix_mdev->lock.
+ *
+ */
static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)
{
- kvm_arch_crypto_clear_masks(matrix_mdev->kvm);
- matrix_mdev->kvm->arch.crypto.pqap_hook = NULL;
- vfio_ap_mdev_reset_queues(matrix_mdev->mdev);
- kvm_put_kvm(matrix_mdev->kvm);
- matrix_mdev->kvm = NULL;
+ /*
+ * If the KVM pointer is in the process of being set, wait until the
+ * process has completed.
+ */
+ wait_event_cmd(matrix_mdev->wait_for_kvm,
+ !matrix_mdev->kvm_busy,
+ mutex_unlock(&matrix_dev->lock),
+ mutex_lock(&matrix_dev->lock));
+
+ if (matrix_mdev->kvm) {
+ matrix_mdev->kvm_busy = true;
+ mutex_unlock(&matrix_dev->lock);
+ kvm_arch_crypto_clear_masks(matrix_mdev->kvm);
+ mutex_lock(&matrix_dev->lock);
+ vfio_ap_mdev_reset_queues(matrix_mdev->mdev);
+ matrix_mdev->kvm->arch.crypto.pqap_hook = NULL;
+ kvm_put_kvm(matrix_mdev->kvm);
+ matrix_mdev->kvm = NULL;
+ matrix_mdev->kvm_busy = false;
+ wake_up_all(&matrix_mdev->wait_for_kvm);
+ }
}
static int vfio_ap_mdev_group_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
- int ret, notify_rc = NOTIFY_OK;
+ int notify_rc = NOTIFY_OK;
struct ap_matrix_mdev *matrix_mdev;
if (action != VFIO_GROUP_NOTIFY_SET_KVM)
return NOTIFY_OK;
- matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);
mutex_lock(&matrix_dev->lock);
+ matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);
- if (!data) {
- if (matrix_mdev->kvm)
- vfio_ap_mdev_unset_kvm(matrix_mdev);
- goto notify_done;
- }
-
- ret = vfio_ap_mdev_set_kvm(matrix_mdev, data);
- if (ret) {
- notify_rc = NOTIFY_DONE;
- goto notify_done;
- }
-
- /* If there is no CRYCB pointer, then we can't copy the masks */
- if (!matrix_mdev->kvm->arch.crypto.crycbd) {
+ if (!data)
+ vfio_ap_mdev_unset_kvm(matrix_mdev);
+ else if (vfio_ap_mdev_set_kvm(matrix_mdev, data))
notify_rc = NOTIFY_DONE;
- goto notify_done;
- }
-
- kvm_arch_crypto_set_masks(matrix_mdev->kvm, matrix_mdev->matrix.apm,
- matrix_mdev->matrix.aqm,
- matrix_mdev->matrix.adm);
-notify_done:
mutex_unlock(&matrix_dev->lock);
+
return notify_rc;
}
struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
mutex_lock(&matrix_dev->lock);
- if (matrix_mdev->kvm)
- vfio_ap_mdev_unset_kvm(matrix_mdev);
+ vfio_ap_mdev_unset_kvm(matrix_mdev);
mutex_unlock(&matrix_dev->lock);
vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
unsigned int cmd, unsigned long arg)
{
int ret;
+ struct ap_matrix_mdev *matrix_mdev;
mutex_lock(&matrix_dev->lock);
switch (cmd) {
ret = vfio_ap_mdev_get_device_info(arg);
break;
case VFIO_DEVICE_RESET:
+ matrix_mdev = mdev_get_drvdata(mdev);
+ if (WARN(!matrix_mdev, "Driver data missing from mdev!!")) {
+ ret = -EINVAL;
+ break;
+ }
+
+ /*
+ * If the KVM pointer is in the process of being set, wait until
+ * the process has completed.
+ */
+ wait_event_cmd(matrix_mdev->wait_for_kvm,
+ !matrix_mdev->kvm_busy,
+ mutex_unlock(&matrix_dev->lock),
+ mutex_lock(&matrix_dev->lock));
+
ret = vfio_ap_mdev_reset_queues(mdev);
break;
default:
if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
vdev->has_vga = true;
-
if (vfio_pci_is_vga(pdev) &&
pdev->vendor == PCI_VENDOR_ID_INTEL &&
IS_ENABLED(CONFIG_VFIO_PCI_IGD)) {
}
}
- if (pdev->vendor == PCI_VENDOR_ID_NVIDIA &&
- IS_ENABLED(CONFIG_VFIO_PCI_NVLINK2)) {
- ret = vfio_pci_nvdia_v100_nvlink2_init(vdev);
- if (ret && ret != -ENODEV) {
- pci_warn(pdev, "Failed to setup NVIDIA NV2 RAM region\n");
- goto disable_exit;
- }
- }
-
- if (pdev->vendor == PCI_VENDOR_ID_IBM &&
- IS_ENABLED(CONFIG_VFIO_PCI_NVLINK2)) {
- ret = vfio_pci_ibm_npu2_init(vdev);
- if (ret && ret != -ENODEV) {
- pci_warn(pdev, "Failed to setup NVIDIA NV2 ATSD region\n");
- goto disable_exit;
- }
- }
-
vfio_pci_probe_mmaps(vdev);
return 0;
static struct pci_driver vfio_pci_driver;
- static struct vfio_pci_device *get_pf_vdev(struct vfio_pci_device *vdev,
- struct vfio_device **pf_dev)
+ static struct vfio_pci_device *get_pf_vdev(struct vfio_pci_device *vdev)
{
struct pci_dev *physfn = pci_physfn(vdev->pdev);
+ struct vfio_device *pf_dev;
if (!vdev->pdev->is_virtfn)
return NULL;
- *pf_dev = vfio_device_get_from_dev(&physfn->dev);
- if (!*pf_dev)
+ pf_dev = vfio_device_get_from_dev(&physfn->dev);
+ if (!pf_dev)
return NULL;
if (pci_dev_driver(physfn) != &vfio_pci_driver) {
- vfio_device_put(*pf_dev);
+ vfio_device_put(pf_dev);
return NULL;
}
- return vfio_device_data(*pf_dev);
+ return container_of(pf_dev, struct vfio_pci_device, vdev);
}
static void vfio_pci_vf_token_user_add(struct vfio_pci_device *vdev, int val)
{
- struct vfio_device *pf_dev;
- struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev, &pf_dev);
+ struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev);
if (!pf_vdev)
return;
WARN_ON(pf_vdev->vf_token->users < 0);
mutex_unlock(&pf_vdev->vf_token->lock);
- vfio_device_put(pf_dev);
+ vfio_device_put(&pf_vdev->vdev);
}
- static void vfio_pci_release(void *device_data)
+ static void vfio_pci_release(struct vfio_device *core_vdev)
{
- struct vfio_pci_device *vdev = device_data;
+ struct vfio_pci_device *vdev =
+ container_of(core_vdev, struct vfio_pci_device, vdev);
mutex_lock(&vdev->reflck->lock);
module_put(THIS_MODULE);
}
- static int vfio_pci_open(void *device_data)
+ static int vfio_pci_open(struct vfio_device *core_vdev)
{
- struct vfio_pci_device *vdev = device_data;
+ struct vfio_pci_device *vdev =
+ container_of(core_vdev, struct vfio_pci_device, vdev);
int ret = 0;
if (!try_module_get(THIS_MODULE))
}
struct vfio_devices {
- struct vfio_device **devices;
+ struct vfio_pci_device **devices;
int cur_index;
int max_index;
};
- static long vfio_pci_ioctl(void *device_data,
+ static long vfio_pci_ioctl(struct vfio_device *core_vdev,
unsigned int cmd, unsigned long arg)
{
- struct vfio_pci_device *vdev = device_data;
+ struct vfio_pci_device *vdev =
+ container_of(core_vdev, struct vfio_pci_device, vdev);
unsigned long minsz;
if (cmd == VFIO_DEVICE_GET_INFO) {
goto hot_reset_release;
for (; mem_idx < devs.cur_index; mem_idx++) {
- struct vfio_pci_device *tmp;
-
- tmp = vfio_device_data(devs.devices[mem_idx]);
+ struct vfio_pci_device *tmp = devs.devices[mem_idx];
ret = down_write_trylock(&tmp->memory_lock);
if (!ret) {
hot_reset_release:
for (i = 0; i < devs.cur_index; i++) {
- struct vfio_device *device;
- struct vfio_pci_device *tmp;
-
- device = devs.devices[i];
- tmp = vfio_device_data(device);
+ struct vfio_pci_device *tmp = devs.devices[i];
if (i < mem_idx)
up_write(&tmp->memory_lock);
else
mutex_unlock(&tmp->vma_lock);
- vfio_device_put(device);
+ vfio_device_put(&tmp->vdev);
}
kfree(devs.devices);
return -ENOTTY;
}
- static ssize_t vfio_pci_rw(void *device_data, char __user *buf,
+ static ssize_t vfio_pci_rw(struct vfio_pci_device *vdev, char __user *buf,
size_t count, loff_t *ppos, bool iswrite)
{
unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
- struct vfio_pci_device *vdev = device_data;
if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
return -EINVAL;
return -EINVAL;
}
- static ssize_t vfio_pci_read(void *device_data, char __user *buf,
+ static ssize_t vfio_pci_read(struct vfio_device *core_vdev, char __user *buf,
size_t count, loff_t *ppos)
{
+ struct vfio_pci_device *vdev =
+ container_of(core_vdev, struct vfio_pci_device, vdev);
+
if (!count)
return 0;
- return vfio_pci_rw(device_data, buf, count, ppos, false);
+ return vfio_pci_rw(vdev, buf, count, ppos, false);
}
- static ssize_t vfio_pci_write(void *device_data, const char __user *buf,
+ static ssize_t vfio_pci_write(struct vfio_device *core_vdev, const char __user *buf,
size_t count, loff_t *ppos)
{
+ struct vfio_pci_device *vdev =
+ container_of(core_vdev, struct vfio_pci_device, vdev);
+
if (!count)
return 0;
- return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true);
+ return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true);
}
/* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */
.fault = vfio_pci_mmap_fault,
};
- static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
+ static int vfio_pci_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
{
- struct vfio_pci_device *vdev = device_data;
+ struct vfio_pci_device *vdev =
+ container_of(core_vdev, struct vfio_pci_device, vdev);
struct pci_dev *pdev = vdev->pdev;
unsigned int index;
u64 phys_len, req_len, pgoff, req_start;
index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
+ if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
+ return -EINVAL;
if (vma->vm_end < vma->vm_start)
return -EINVAL;
if ((vma->vm_flags & VM_SHARED) == 0)
int regnum = index - VFIO_PCI_NUM_REGIONS;
struct vfio_pci_region *region = vdev->region + regnum;
- if (region && region->ops && region->ops->mmap &&
+ if (region->ops && region->ops->mmap &&
(region->flags & VFIO_REGION_INFO_FLAG_MMAP))
return region->ops->mmap(vdev, region, vma);
return -EINVAL;
return 0;
}
- static void vfio_pci_request(void *device_data, unsigned int count)
+ static void vfio_pci_request(struct vfio_device *core_vdev, unsigned int count)
{
- struct vfio_pci_device *vdev = device_data;
+ struct vfio_pci_device *vdev =
+ container_of(core_vdev, struct vfio_pci_device, vdev);
struct pci_dev *pdev = vdev->pdev;
mutex_lock(&vdev->igate);
return 0; /* No VF token provided or required */
if (vdev->pdev->is_virtfn) {
- struct vfio_device *pf_dev;
- struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev, &pf_dev);
+ struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev);
bool match;
if (!pf_vdev) {
}
if (!vf_token) {
- vfio_device_put(pf_dev);
+ vfio_device_put(&pf_vdev->vdev);
pci_info_ratelimited(vdev->pdev,
"VF token required to access device\n");
return -EACCES;
match = uuid_equal(uuid, &pf_vdev->vf_token->uuid);
mutex_unlock(&pf_vdev->vf_token->lock);
- vfio_device_put(pf_dev);
+ vfio_device_put(&pf_vdev->vdev);
if (!match) {
pci_info_ratelimited(vdev->pdev,
#define VF_TOKEN_ARG "vf_token="
- static int vfio_pci_match(void *device_data, char *buf)
+ static int vfio_pci_match(struct vfio_device *core_vdev, char *buf)
{
- struct vfio_pci_device *vdev = device_data;
+ struct vfio_pci_device *vdev =
+ container_of(core_vdev, struct vfio_pci_device, vdev);
bool vf_token = false;
uuid_t uuid;
int ret;
return 0;
}
+ static int vfio_pci_vf_init(struct vfio_pci_device *vdev)
+ {
+ struct pci_dev *pdev = vdev->pdev;
+ int ret;
+
+ if (!pdev->is_physfn)
+ return 0;
+
+ vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);
+ if (!vdev->vf_token)
+ return -ENOMEM;
+
+ mutex_init(&vdev->vf_token->lock);
+ uuid_gen(&vdev->vf_token->uuid);
+
+ vdev->nb.notifier_call = vfio_pci_bus_notifier;
+ ret = bus_register_notifier(&pci_bus_type, &vdev->nb);
+ if (ret) {
+ kfree(vdev->vf_token);
+ return ret;
+ }
+ return 0;
+ }
+
+ static void vfio_pci_vf_uninit(struct vfio_pci_device *vdev)
+ {
+ if (!vdev->vf_token)
+ return;
+
+ bus_unregister_notifier(&pci_bus_type, &vdev->nb);
+ WARN_ON(vdev->vf_token->users);
+ mutex_destroy(&vdev->vf_token->lock);
+ kfree(vdev->vf_token);
+ }
+
+ static int vfio_pci_vga_init(struct vfio_pci_device *vdev)
+ {
+ struct pci_dev *pdev = vdev->pdev;
+ int ret;
+
+ if (!vfio_pci_is_vga(pdev))
+ return 0;
+
+ ret = vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);
+ if (ret)
+ return ret;
+ vga_set_legacy_decoding(pdev, vfio_pci_set_vga_decode(vdev, false));
+ return 0;
+ }
+
+ static void vfio_pci_vga_uninit(struct vfio_pci_device *vdev)
+ {
+ struct pci_dev *pdev = vdev->pdev;
+
+ if (!vfio_pci_is_vga(pdev))
+ return;
+ vga_client_register(pdev, NULL, NULL, NULL);
+ vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
+ VGA_RSRC_LEGACY_IO |
+ VGA_RSRC_LEGACY_MEM);
+ }
+
static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct vfio_pci_device *vdev;
goto out_group_put;
}
+ vfio_init_group_dev(&vdev->vdev, &pdev->dev, &vfio_pci_ops);
vdev->pdev = pdev;
vdev->irq_type = VFIO_PCI_NUM_IRQS;
mutex_init(&vdev->igate);
INIT_LIST_HEAD(&vdev->vma_list);
init_rwsem(&vdev->memory_lock);
- ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
+ ret = vfio_pci_reflck_attach(vdev);
if (ret)
goto out_free;
-
- ret = vfio_pci_reflck_attach(vdev);
+ ret = vfio_pci_vf_init(vdev);
if (ret)
- goto out_del_group_dev;
-
- if (pdev->is_physfn) {
- vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);
- if (!vdev->vf_token) {
- ret = -ENOMEM;
- goto out_reflck;
- }
-
- mutex_init(&vdev->vf_token->lock);
- uuid_gen(&vdev->vf_token->uuid);
-
- vdev->nb.notifier_call = vfio_pci_bus_notifier;
- ret = bus_register_notifier(&pci_bus_type, &vdev->nb);
- if (ret)
- goto out_vf_token;
- }
-
- if (vfio_pci_is_vga(pdev)) {
- vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);
- vga_set_legacy_decoding(pdev,
- vfio_pci_set_vga_decode(vdev, false));
- }
+ goto out_reflck;
+ ret = vfio_pci_vga_init(vdev);
+ if (ret)
+ goto out_vf;
vfio_pci_probe_power_state(vdev);
vfio_pci_set_power_state(vdev, PCI_D3hot);
}
- return ret;
+ ret = vfio_register_group_dev(&vdev->vdev);
+ if (ret)
+ goto out_power;
+ dev_set_drvdata(&pdev->dev, vdev);
+ return 0;
- out_vf_token:
- kfree(vdev->vf_token);
+ out_power:
+ if (!disable_idle_d3)
+ vfio_pci_set_power_state(vdev, PCI_D0);
+ out_vf:
+ vfio_pci_vf_uninit(vdev);
out_reflck:
vfio_pci_reflck_put(vdev->reflck);
- out_del_group_dev:
- vfio_del_group_dev(&pdev->dev);
out_free:
+ kfree(vdev->pm_save);
kfree(vdev);
out_group_put:
vfio_iommu_group_put(group, &pdev->dev);
static void vfio_pci_remove(struct pci_dev *pdev)
{
- struct vfio_pci_device *vdev;
+ struct vfio_pci_device *vdev = dev_get_drvdata(&pdev->dev);
pci_disable_sriov(pdev);
- vdev = vfio_del_group_dev(&pdev->dev);
- if (!vdev)
- return;
-
- if (vdev->vf_token) {
- WARN_ON(vdev->vf_token->users);
- mutex_destroy(&vdev->vf_token->lock);
- kfree(vdev->vf_token);
- }
-
- if (vdev->nb.notifier_call)
- bus_unregister_notifier(&pci_bus_type, &vdev->nb);
+ vfio_unregister_group_dev(&vdev->vdev);
+ vfio_pci_vf_uninit(vdev);
vfio_pci_reflck_put(vdev->reflck);
+ vfio_pci_vga_uninit(vdev);
vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
- kfree(vdev->region);
- mutex_destroy(&vdev->ioeventfds_lock);
if (!disable_idle_d3)
vfio_pci_set_power_state(vdev, PCI_D0);
+ mutex_destroy(&vdev->ioeventfds_lock);
+ kfree(vdev->region);
kfree(vdev->pm_save);
kfree(vdev);
-
- if (vfio_pci_is_vga(pdev)) {
- vga_client_register(pdev, NULL, NULL, NULL);
- vga_set_legacy_decoding(pdev,
- VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
- VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM);
- }
}
static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
if (device == NULL)
return PCI_ERS_RESULT_DISCONNECT;
- vdev = vfio_device_data(device);
- if (vdev == NULL) {
- vfio_device_put(device);
- return PCI_ERS_RESULT_DISCONNECT;
- }
+ vdev = container_of(device, struct vfio_pci_device, vdev);
mutex_lock(&vdev->igate);
static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
{
- struct vfio_pci_device *vdev;
struct vfio_device *device;
int ret = 0;
if (!device)
return -ENODEV;
- vdev = vfio_device_data(device);
- if (!vdev) {
- vfio_device_put(device);
- return -ENODEV;
- }
-
if (nr_virtfn == 0)
pci_disable_sriov(pdev);
else
return 0;
}
- vdev = vfio_device_data(device);
+ vdev = container_of(device, struct vfio_pci_device, vdev);
if (vdev->reflck) {
vfio_pci_reflck_get(vdev->reflck);
return -EBUSY;
}
- vdev = vfio_device_data(device);
+ vdev = container_of(device, struct vfio_pci_device, vdev);
/* Fault if the device is not unused */
if (vdev->refcnt) {
return -EBUSY;
}
- devs->devices[devs->cur_index++] = device;
+ devs->devices[devs->cur_index++] = vdev;
return 0;
}
return -EBUSY;
}
- vdev = vfio_device_data(device);
+ vdev = container_of(device, struct vfio_pci_device, vdev);
/*
* Locking multiple devices is prone to deadlock, runaway and
return -EBUSY;
}
- devs->devices[devs->cur_index++] = device;
+ devs->devices[devs->cur_index++] = vdev;
return 0;
}
/* Does at least one need a reset? */
for (i = 0; i < devs.cur_index; i++) {
- tmp = vfio_device_data(devs.devices[i]);
+ tmp = devs.devices[i];
if (tmp->needs_reset) {
ret = pci_reset_bus(vdev->pdev);
break;
put_devs:
for (i = 0; i < devs.cur_index; i++) {
- tmp = vfio_device_data(devs.devices[i]);
+ tmp = devs.devices[i];
/*
* If reset was successful, affected devices no longer need
vfio_pci_set_power_state(tmp, PCI_D3hot);
}
- vfio_device_put(devs.devices[i]);
+ vfio_device_put(&tmp->vdev);
}
kfree(devs.devices);
{
int ret;
- /* Allocate shared config space permision data used by all devices */
+ /* Allocate shared config space permission data used by all devices */
ret = vfio_pci_init_perm_bits();
if (ret)
return ret;
projects / linux.git / commitdiff