[linux.git] / virt / kvm / vfio.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * VFIO-KVM bridge pseudo device
 *
 * Copyright (C) 2013 Red Hat, Inc.  All rights reserved.
 *     Author: Alex Williamson <[email protected]>
 */

#include <linux/errno.h>
#include <linux/file.h>
#include <linux/kvm_host.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/vfio.h>
#include "vfio.h"

#ifdef CONFIG_SPAPR_TCE_IOMMU
#include <asm/kvm_ppc.h>
#endif

struct kvm_vfio_group {
	struct list_head node;
	struct vfio_group *vfio_group;
};

struct kvm_vfio {
	struct list_head group_list;
	struct mutex lock;
	bool noncoherent;
};

static struct vfio_group *kvm_vfio_group_get_external_user(struct file *filep)
{
	struct vfio_group *vfio_group;
	struct vfio_group *(*fn)(struct file *);

	fn = symbol_get(vfio_group_get_external_user);
	if (!fn)
		return ERR_PTR(-EINVAL);

	vfio_group = fn(filep);

	symbol_put(vfio_group_get_external_user);

	return vfio_group;
}

static bool kvm_vfio_external_group_match_file(struct vfio_group *group,
					       struct file *filep)
{
	bool ret, (*fn)(struct vfio_group *, struct file *);

	fn = symbol_get(vfio_external_group_match_file);
	if (!fn)
		return false;

	ret = fn(group, filep);

	symbol_put(vfio_external_group_match_file);

	return ret;
}

static void kvm_vfio_group_put_external_user(struct vfio_group *vfio_group)
{
	void (*fn)(struct vfio_group *);

	fn = symbol_get(vfio_group_put_external_user);
	if (!fn)
		return;

	fn(vfio_group);

	symbol_put(vfio_group_put_external_user);
}

static void kvm_vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm)
{
	void (*fn)(struct vfio_group *, struct kvm *);

	fn = symbol_get(vfio_group_set_kvm);
	if (!fn)
		return;

	fn(group, kvm);

	symbol_put(vfio_group_set_kvm);
}

static bool kvm_vfio_group_is_coherent(struct vfio_group *vfio_group)
{
	long (*fn)(struct vfio_group *, unsigned long);
	long ret;

	fn = symbol_get(vfio_external_check_extension);
	if (!fn)
		return false;

	ret = fn(vfio_group, VFIO_DMA_CC_IOMMU);

	symbol_put(vfio_external_check_extension);

	return ret > 0;
}

#ifdef CONFIG_SPAPR_TCE_IOMMU
static int kvm_vfio_external_user_iommu_id(struct vfio_group *vfio_group)
{
	int (*fn)(struct vfio_group *);
	int ret = -EINVAL;

	fn = symbol_get(vfio_external_user_iommu_id);
	if (!fn)
		return ret;

	ret = fn(vfio_group);

	symbol_put(vfio_external_user_iommu_id);

	return ret;
}

static struct iommu_group *kvm_vfio_group_get_iommu_group(
		struct vfio_group *group)
{
	int group_id = kvm_vfio_external_user_iommu_id(group);

	if (group_id < 0)
		return NULL;

	return iommu_group_get_by_id(group_id);
}

static void kvm_spapr_tce_release_vfio_group(struct kvm *kvm,
		struct vfio_group *vfio_group)
{
	struct iommu_group *grp = kvm_vfio_group_get_iommu_group(vfio_group);

	if (WARN_ON_ONCE(!grp))
		return;

	kvm_spapr_tce_release_iommu_group(kvm, grp);
	iommu_group_put(grp);
}
#endif

/*
 * Groups can use the same or different IOMMU domains.  If the same then
 * adding a new group may change the coherency of groups we've previously
 * been told about.  We don't want to care about any of that so we retest
 * each group and bail as soon as we find one that's noncoherent.  This
 * means we only ever [un]register_noncoherent_dma once for the whole device.
 */
static void kvm_vfio_update_coherency(struct kvm_device *dev)
{
	struct kvm_vfio *kv = dev->private;
	bool noncoherent = false;
	struct kvm_vfio_group *kvg;

	mutex_lock(&kv->lock);

	list_for_each_entry(kvg, &kv->group_list, node) {
		if (!kvm_vfio_group_is_coherent(kvg->vfio_group)) {
			noncoherent = true;
			break;
		}
	}

	if (noncoherent != kv->noncoherent) {
		kv->noncoherent = noncoherent;

		if (kv->noncoherent)
			kvm_arch_register_noncoherent_dma(dev->kvm);
		else
			kvm_arch_unregister_noncoherent_dma(dev->kvm);
	}

	mutex_unlock(&kv->lock);
}

static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg)
{
	struct kvm_vfio *kv = dev->private;
	struct vfio_group *vfio_group;
	struct kvm_vfio_group *kvg;
	int32_t __user *argp = (int32_t __user *)(unsigned long)arg;
	struct fd f;
	int32_t fd;
	int ret;

	switch (attr) {
	case KVM_DEV_VFIO_GROUP_ADD:
		if (get_user(fd, argp))
			return -EFAULT;

		f = fdget(fd);
		if (!f.file)
			return -EBADF;

		vfio_group = kvm_vfio_group_get_external_user(f.file);
		fdput(f);

		if (IS_ERR(vfio_group))
			return PTR_ERR(vfio_group);

		mutex_lock(&kv->lock);

		list_for_each_entry(kvg, &kv->group_list, node) {
			if (kvg->vfio_group == vfio_group) {
				mutex_unlock(&kv->lock);
				kvm_vfio_group_put_external_user(vfio_group);
				return -EEXIST;
			}
		}

		kvg = kzalloc(sizeof(*kvg), GFP_KERNEL_ACCOUNT);
		if (!kvg) {
			mutex_unlock(&kv->lock);
			kvm_vfio_group_put_external_user(vfio_group);
			return -ENOMEM;
		}

		list_add_tail(&kvg->node, &kv->group_list);
		kvg->vfio_group = vfio_group;

		kvm_arch_start_assignment(dev->kvm);

		mutex_unlock(&kv->lock);

		kvm_vfio_group_set_kvm(vfio_group, dev->kvm);

		kvm_vfio_update_coherency(dev);

		return 0;

	case KVM_DEV_VFIO_GROUP_DEL:
		if (get_user(fd, argp))
			return -EFAULT;

		f = fdget(fd);
		if (!f.file)
			return -EBADF;

		ret = -ENOENT;

		mutex_lock(&kv->lock);

		list_for_each_entry(kvg, &kv->group_list, node) {
			if (!kvm_vfio_external_group_match_file(kvg->vfio_group,
								f.file))
				continue;

			list_del(&kvg->node);
			kvm_arch_end_assignment(dev->kvm);
#ifdef CONFIG_SPAPR_TCE_IOMMU
			kvm_spapr_tce_release_vfio_group(dev->kvm,
							 kvg->vfio_group);
#endif
			kvm_vfio_group_set_kvm(kvg->vfio_group, NULL);
			kvm_vfio_group_put_external_user(kvg->vfio_group);
			kfree(kvg);
			ret = 0;
			break;
		}

		mutex_unlock(&kv->lock);

		fdput(f);

		kvm_vfio_update_coherency(dev);

		return ret;

#ifdef CONFIG_SPAPR_TCE_IOMMU
	case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE: {
		struct kvm_vfio_spapr_tce param;
		struct kvm_vfio *kv = dev->private;
		struct vfio_group *vfio_group;
		struct kvm_vfio_group *kvg;
		struct fd f;
		struct iommu_group *grp;

		if (copy_from_user(&param, (void __user *)arg,
				sizeof(struct kvm_vfio_spapr_tce)))
			return -EFAULT;

		f = fdget(param.groupfd);
		if (!f.file)
			return -EBADF;

		vfio_group = kvm_vfio_group_get_external_user(f.file);
		fdput(f);

		if (IS_ERR(vfio_group))
			return PTR_ERR(vfio_group);

		grp = kvm_vfio_group_get_iommu_group(vfio_group);
		if (WARN_ON_ONCE(!grp)) {
			kvm_vfio_group_put_external_user(vfio_group);
			return -EIO;
		}

		ret = -ENOENT;

		mutex_lock(&kv->lock);

		list_for_each_entry(kvg, &kv->group_list, node) {
			if (kvg->vfio_group != vfio_group)
				continue;

			ret = kvm_spapr_tce_attach_iommu_group(dev->kvm,
					param.tablefd, grp);
			break;
		}

		mutex_unlock(&kv->lock);

		iommu_group_put(grp);
		kvm_vfio_group_put_external_user(vfio_group);

		return ret;
	}
#endif /* CONFIG_SPAPR_TCE_IOMMU */
	}

	return -ENXIO;
}

static int kvm_vfio_set_attr(struct kvm_device *dev,
			     struct kvm_device_attr *attr)
{
	switch (attr->group) {
	case KVM_DEV_VFIO_GROUP:
		return kvm_vfio_set_group(dev, attr->attr, attr->addr);
	}

	return -ENXIO;
}

static int kvm_vfio_has_attr(struct kvm_device *dev,
			     struct kvm_device_attr *attr)
{
	switch (attr->group) {
	case KVM_DEV_VFIO_GROUP:
		switch (attr->attr) {
		case KVM_DEV_VFIO_GROUP_ADD:
		case KVM_DEV_VFIO_GROUP_DEL:
#ifdef CONFIG_SPAPR_TCE_IOMMU
		case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE:
#endif
			return 0;
		}

		break;
	}

	return -ENXIO;
}

static void kvm_vfio_destroy(struct kvm_device *dev)
{
	struct kvm_vfio *kv = dev->private;
	struct kvm_vfio_group *kvg, *tmp;

	list_for_each_entry_safe(kvg, tmp, &kv->group_list, node) {
#ifdef CONFIG_SPAPR_TCE_IOMMU
		kvm_spapr_tce_release_vfio_group(dev->kvm, kvg->vfio_group);
#endif
		kvm_vfio_group_set_kvm(kvg->vfio_group, NULL);
		kvm_vfio_group_put_external_user(kvg->vfio_group);
		list_del(&kvg->node);
		kfree(kvg);
		kvm_arch_end_assignment(dev->kvm);
	}

	kvm_vfio_update_coherency(dev);

	kfree(kv);
	kfree(dev); /* alloc by kvm_ioctl_create_device, free by .destroy */
}

static int kvm_vfio_create(struct kvm_device *dev, u32 type);

static struct kvm_device_ops kvm_vfio_ops = {
	.name = "kvm-vfio",
	.create = kvm_vfio_create,
	.destroy = kvm_vfio_destroy,
	.set_attr = kvm_vfio_set_attr,
	.has_attr = kvm_vfio_has_attr,
};

static int kvm_vfio_create(struct kvm_device *dev, u32 type)
{
	struct kvm_device *tmp;
	struct kvm_vfio *kv;

	/* Only one VFIO "device" per VM */
	list_for_each_entry(tmp, &dev->kvm->devices, vm_node)
		if (tmp->ops == &kvm_vfio_ops)
			return -EBUSY;

	kv = kzalloc(sizeof(*kv), GFP_KERNEL_ACCOUNT);
	if (!kv)
		return -ENOMEM;

	INIT_LIST_HEAD(&kv->group_list);
	mutex_init(&kv->lock);

	dev->private = kv;

	return 0;
}

int kvm_vfio_ops_init(void)
{
	return kvm_register_device_ops(&kvm_vfio_ops, KVM_DEV_TYPE_VFIO);
}

void kvm_vfio_ops_exit(void)
{
	kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO);
}
Commit	Line	Data
d2912cb1	1	// SPDX-License-Identifier: GPL-2.0-only
ec53500f AW	2	/*
	3	* VFIO-KVM bridge pseudo device
	4	*
	5	* Copyright (C) 2013 Red Hat, Inc. All rights reserved.
	6	* Author: Alex Williamson <[email protected]>
ec53500f AW	7	*/
	8
	9	#include <linux/errno.h>
	10	#include <linux/file.h>
	11	#include <linux/kvm_host.h>
	12	#include <linux/list.h>
	13	#include <linux/module.h>
	14	#include <linux/mutex.h>
	15	#include <linux/slab.h>
	16	#include <linux/uaccess.h>
	17	#include <linux/vfio.h>
3c3c29fd	18	#include "vfio.h"
ec53500f	19
121f80ba AK	20	#ifdef CONFIG_SPAPR_TCE_IOMMU
	21	#include <asm/kvm_ppc.h>
	22	#endif
	23
ec53500f AW	24	struct kvm_vfio_group {
	25	struct list_head node;
	26	struct vfio_group *vfio_group;
	27	};
	28
	29	struct kvm_vfio {
	30	struct list_head group_list;
	31	struct mutex lock;
e0f0bbc5	32	bool noncoherent;
ec53500f AW	33	};
	34
	35	static struct vfio_group kvm_vfio_group_get_external_user(struct file filep)
	36	{
	37	struct vfio_group *vfio_group;
	38	struct vfio_group (fn)(struct file *);
	39
	40	fn = symbol_get(vfio_group_get_external_user);
	41	if (!fn)
	42	return ERR_PTR(-EINVAL);
	43
	44	vfio_group = fn(filep);
	45
	46	symbol_put(vfio_group_get_external_user);
	47
	48	return vfio_group;
	49	}
	50
5d6dee80 AW	51	static bool kvm_vfio_external_group_match_file(struct vfio_group *group,
	52	struct file *filep)
	53	{
	54	bool ret, (fn)(struct vfio_group , struct file *);
	55
	56	fn = symbol_get(vfio_external_group_match_file);
	57	if (!fn)
	58	return false;
	59
	60	ret = fn(group, filep);
	61
	62	symbol_put(vfio_external_group_match_file);
	63
	64	return ret;
	65	}
	66
ec53500f AW	67	static void kvm_vfio_group_put_external_user(struct vfio_group *vfio_group)
	68	{
	69	void (fn)(struct vfio_group );
	70
	71	fn = symbol_get(vfio_group_put_external_user);
	72	if (!fn)
	73	return;
	74
	75	fn(vfio_group);
	76
	77	symbol_put(vfio_group_put_external_user);
	78	}
	79
2fc1bec1 JS	80	static void kvm_vfio_group_set_kvm(struct vfio_group group, struct kvm kvm)
	81	{
	82	void (fn)(struct vfio_group , struct kvm *);
	83
	84	fn = symbol_get(vfio_group_set_kvm);
	85	if (!fn)
	86	return;
	87
	88	fn(group, kvm);
	89
	90	symbol_put(vfio_group_set_kvm);
	91	}
	92
9d830d47 AW	93	static bool kvm_vfio_group_is_coherent(struct vfio_group *vfio_group)
	94	{
	95	long (fn)(struct vfio_group , unsigned long);
	96	long ret;
	97
	98	fn = symbol_get(vfio_external_check_extension);
	99	if (!fn)
	100	return false;
	101
	102	ret = fn(vfio_group, VFIO_DMA_CC_IOMMU);
	103
	104	symbol_put(vfio_external_check_extension);
	105
	106	return ret > 0;
	107	}
	108
121f80ba AK	109	#ifdef CONFIG_SPAPR_TCE_IOMMU
	110	static int kvm_vfio_external_user_iommu_id(struct vfio_group *vfio_group)
	111	{
	112	int (fn)(struct vfio_group );
	113	int ret = -EINVAL;
	114
	115	fn = symbol_get(vfio_external_user_iommu_id);
	116	if (!fn)
	117	return ret;
	118
	119	ret = fn(vfio_group);
	120
	121	symbol_put(vfio_external_user_iommu_id);
	122
	123	return ret;
	124	}
	125
	126	static struct iommu_group *kvm_vfio_group_get_iommu_group(
	127	struct vfio_group *group)
	128	{
	129	int group_id = kvm_vfio_external_user_iommu_id(group);
	130
	131	if (group_id < 0)
	132	return NULL;
	133
	134	return iommu_group_get_by_id(group_id);
	135	}
	136
	137	static void kvm_spapr_tce_release_vfio_group(struct kvm *kvm,
	138	struct vfio_group *vfio_group)
	139	{
	140	struct iommu_group *grp = kvm_vfio_group_get_iommu_group(vfio_group);
	141
	142	if (WARN_ON_ONCE(!grp))
	143	return;
	144
	145	kvm_spapr_tce_release_iommu_group(kvm, grp);
	146	iommu_group_put(grp);
	147	}
	148	#endif
	149
e0f0bbc5 AW	150	/*
	151	* Groups can use the same or different IOMMU domains. If the same then
	152	* adding a new group may change the coherency of groups we've previously
	153	* been told about. We don't want to care about any of that so we retest
	154	* each group and bail as soon as we find one that's noncoherent. This
	155	* means we only ever [un]register_noncoherent_dma once for the whole device.
	156	*/
	157	static void kvm_vfio_update_coherency(struct kvm_device *dev)
	158	{
	159	struct kvm_vfio *kv = dev->private;
	160	bool noncoherent = false;
	161	struct kvm_vfio_group *kvg;
	162
	163	mutex_lock(&kv->lock);
	164
	165	list_for_each_entry(kvg, &kv->group_list, node) {
9d830d47 AW	166	if (!kvm_vfio_group_is_coherent(kvg->vfio_group)) {
	167	noncoherent = true;
	168	break;
	169	}
e0f0bbc5 AW	170	}
	171
	172	if (noncoherent != kv->noncoherent) {
	173	kv->noncoherent = noncoherent;
	174
	175	if (kv->noncoherent)
	176	kvm_arch_register_noncoherent_dma(dev->kvm);
	177	else
	178	kvm_arch_unregister_noncoherent_dma(dev->kvm);
	179	}
	180
	181	mutex_unlock(&kv->lock);
	182	}
	183
ec53500f AW	184	static int kvm_vfio_set_group(struct kvm_device *dev, long attr, u64 arg)
	185	{
	186	struct kvm_vfio *kv = dev->private;
	187	struct vfio_group *vfio_group;
	188	struct kvm_vfio_group *kvg;
e81d1ad3	189	int32_t __user argp = (int32_t __user )(unsigned long)arg;
ec53500f AW	190	struct fd f;
	191	int32_t fd;
	192	int ret;
	193
	194	switch (attr) {
	195	case KVM_DEV_VFIO_GROUP_ADD:
e81d1ad3	196	if (get_user(fd, argp))
ec53500f AW	197	return -EFAULT;
	198
	199	f = fdget(fd);
	200	if (!f.file)
	201	return -EBADF;
	202
	203	vfio_group = kvm_vfio_group_get_external_user(f.file);
	204	fdput(f);
	205
	206	if (IS_ERR(vfio_group))
	207	return PTR_ERR(vfio_group);
	208
	209	mutex_lock(&kv->lock);
	210
	211	list_for_each_entry(kvg, &kv->group_list, node) {
	212	if (kvg->vfio_group == vfio_group) {
	213	mutex_unlock(&kv->lock);
	214	kvm_vfio_group_put_external_user(vfio_group);
	215	return -EEXIST;
	216	}
	217	}
	218
b12ce36a	219	kvg = kzalloc(sizeof(*kvg), GFP_KERNEL_ACCOUNT);
ec53500f AW	220	if (!kvg) {
	221	mutex_unlock(&kv->lock);
	222	kvm_vfio_group_put_external_user(vfio_group);
	223	return -ENOMEM;
	224	}
	225
	226	list_add_tail(&kvg->node, &kv->group_list);
	227	kvg->vfio_group = vfio_group;
	228
5544eb9b PB	229	kvm_arch_start_assignment(dev->kvm);
5544eb9b PB	230
ec53500f AW	231	mutex_unlock(&kv->lock);
ec53500f AW	232
2fc1bec1 JS	233	kvm_vfio_group_set_kvm(vfio_group, dev->kvm);
2fc1bec1 JS	234
e0f0bbc5 AW	235	kvm_vfio_update_coherency(dev);
e0f0bbc5 AW	236
ec53500f AW	237	return 0;
	238
	239	case KVM_DEV_VFIO_GROUP_DEL:
e81d1ad3	240	if (get_user(fd, argp))
ec53500f AW	241	return -EFAULT;
	242
	243	f = fdget(fd);
	244	if (!f.file)
	245	return -EBADF;
	246
ec53500f AW	247	ret = -ENOENT;
	248
	249	mutex_lock(&kv->lock);
	250
	251	list_for_each_entry(kvg, &kv->group_list, node) {
5d6dee80 AW	252	if (!kvm_vfio_external_group_match_file(kvg->vfio_group,
5d6dee80 AW	253	f.file))
ec53500f AW	254	continue;
	255
	256	list_del(&kvg->node);
e323369b AW	257	kvm_arch_end_assignment(dev->kvm);
	258	#ifdef CONFIG_SPAPR_TCE_IOMMU
	259	kvm_spapr_tce_release_vfio_group(dev->kvm,
	260	kvg->vfio_group);
	261	#endif
	262	kvm_vfio_group_set_kvm(kvg->vfio_group, NULL);
ec53500f AW	263	kvm_vfio_group_put_external_user(kvg->vfio_group);
	264	kfree(kvg);
	265	ret = 0;
	266	break;
	267	}
	268
	269	mutex_unlock(&kv->lock);
	270
5d6dee80	271	fdput(f);
ec53500f	272
e0f0bbc5 AW	273	kvm_vfio_update_coherency(dev);
e0f0bbc5 AW	274
ec53500f	275	return ret;
121f80ba AK	276
	277	#ifdef CONFIG_SPAPR_TCE_IOMMU
	278	case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE: {
	279	struct kvm_vfio_spapr_tce param;
	280	struct kvm_vfio *kv = dev->private;
	281	struct vfio_group *vfio_group;
	282	struct kvm_vfio_group *kvg;
	283	struct fd f;
	284	struct iommu_group *grp;
	285
	286	if (copy_from_user(&param, (void __user *)arg,
	287	sizeof(struct kvm_vfio_spapr_tce)))
	288	return -EFAULT;
	289
	290	f = fdget(param.groupfd);
	291	if (!f.file)
	292	return -EBADF;
	293
	294	vfio_group = kvm_vfio_group_get_external_user(f.file);
	295	fdput(f);
	296
	297	if (IS_ERR(vfio_group))
	298	return PTR_ERR(vfio_group);
	299
	300	grp = kvm_vfio_group_get_iommu_group(vfio_group);
	301	if (WARN_ON_ONCE(!grp)) {
	302	kvm_vfio_group_put_external_user(vfio_group);
	303	return -EIO;
	304	}
	305
	306	ret = -ENOENT;
	307
	308	mutex_lock(&kv->lock);
	309
	310	list_for_each_entry(kvg, &kv->group_list, node) {
	311	if (kvg->vfio_group != vfio_group)
	312	continue;
	313
	314	ret = kvm_spapr_tce_attach_iommu_group(dev->kvm,
	315	param.tablefd, grp);
	316	break;
	317	}
	318
	319	mutex_unlock(&kv->lock);
	320
	321	iommu_group_put(grp);
	322	kvm_vfio_group_put_external_user(vfio_group);
	323
	324	return ret;
	325	}
	326	#endif /* CONFIG_SPAPR_TCE_IOMMU */
ec53500f AW	327	}
	328
	329	return -ENXIO;
	330	}
	331
	332	static int kvm_vfio_set_attr(struct kvm_device *dev,
	333	struct kvm_device_attr *attr)
	334	{
	335	switch (attr->group) {
	336	case KVM_DEV_VFIO_GROUP:
	337	return kvm_vfio_set_group(dev, attr->attr, attr->addr);
	338	}
	339
	340	return -ENXIO;
	341	}
	342
	343	static int kvm_vfio_has_attr(struct kvm_device *dev,
	344	struct kvm_device_attr *attr)
	345	{
	346	switch (attr->group) {
	347	case KVM_DEV_VFIO_GROUP:
	348	switch (attr->attr) {
	349	case KVM_DEV_VFIO_GROUP_ADD:
	350	case KVM_DEV_VFIO_GROUP_DEL:
121f80ba AK	351	#ifdef CONFIG_SPAPR_TCE_IOMMU
	352	case KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE:
	353	#endif
ec53500f AW	354	return 0;
	355	}
	356
	357	break;
	358	}
	359
	360	return -ENXIO;
	361	}
	362
	363	static void kvm_vfio_destroy(struct kvm_device *dev)
	364	{
	365	struct kvm_vfio *kv = dev->private;
	366	struct kvm_vfio_group kvg, tmp;
	367
	368	list_for_each_entry_safe(kvg, tmp, &kv->group_list, node) {
121f80ba AK	369	#ifdef CONFIG_SPAPR_TCE_IOMMU
	370	kvm_spapr_tce_release_vfio_group(dev->kvm, kvg->vfio_group);
	371	#endif
2fc1bec1	372	kvm_vfio_group_set_kvm(kvg->vfio_group, NULL);
ec53500f AW	373	kvm_vfio_group_put_external_user(kvg->vfio_group);
	374	list_del(&kvg->node);
	375	kfree(kvg);
5544eb9b	376	kvm_arch_end_assignment(dev->kvm);
ec53500f AW	377	}
ec53500f AW	378
e0f0bbc5 AW	379	kvm_vfio_update_coherency(dev);
e0f0bbc5 AW	380
ec53500f AW	381	kfree(kv);
	382	kfree(dev); /* alloc by kvm_ioctl_create_device, free by .destroy */
	383	}
	384
80ce1639 WD	385	static int kvm_vfio_create(struct kvm_device *dev, u32 type);
	386
	387	static struct kvm_device_ops kvm_vfio_ops = {
	388	.name = "kvm-vfio",
	389	.create = kvm_vfio_create,
	390	.destroy = kvm_vfio_destroy,
	391	.set_attr = kvm_vfio_set_attr,
	392	.has_attr = kvm_vfio_has_attr,
	393	};
	394
ec53500f AW	395	static int kvm_vfio_create(struct kvm_device *dev, u32 type)
	396	{
	397	struct kvm_device *tmp;
	398	struct kvm_vfio *kv;
	399
	400	/* Only one VFIO "device" per VM */
	401	list_for_each_entry(tmp, &dev->kvm->devices, vm_node)
	402	if (tmp->ops == &kvm_vfio_ops)
	403	return -EBUSY;
	404
b12ce36a	405	kv = kzalloc(sizeof(*kv), GFP_KERNEL_ACCOUNT);
ec53500f AW	406	if (!kv)
	407	return -ENOMEM;
	408
	409	INIT_LIST_HEAD(&kv->group_list);
	410	mutex_init(&kv->lock);
	411
	412	dev->private = kv;
	413
	414	return 0;
	415	}
	416
3c3c29fd	417	int kvm_vfio_ops_init(void)
80ce1639 WD	418	{
	419	return kvm_register_device_ops(&kvm_vfio_ops, KVM_DEV_TYPE_VFIO);
	420	}
571ee1b6 WL	421
	422	void kvm_vfio_ops_exit(void)
	423	{
	424	kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO);
	425	}