[linux.git] / drivers / vfio / container.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
 *
 * VFIO container (/dev/vfio/vfio)
 */
#include <linux/file.h>
#include <linux/slab.h>
#include <linux/fs.h>
#include <linux/capability.h>
#include <linux/iommu.h>
#include <linux/miscdevice.h>
#include <linux/vfio.h>
#include <uapi/linux/vfio.h>

#include "vfio.h"

struct vfio_container {
	struct kref			kref;
	struct list_head		group_list;
	struct rw_semaphore		group_lock;
	struct vfio_iommu_driver	*iommu_driver;
	void				*iommu_data;
	bool				noiommu;
};

static struct vfio {
	struct list_head		iommu_drivers_list;
	struct mutex			iommu_drivers_lock;
} vfio;

#ifdef CONFIG_VFIO_NOIOMMU
bool vfio_noiommu __read_mostly;
module_param_named(enable_unsafe_noiommu_mode,
		   vfio_noiommu, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode.  This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel.  If you do not know what this is for, step away. (default: false)");
#endif

static void *vfio_noiommu_open(unsigned long arg)
{
	if (arg != VFIO_NOIOMMU_IOMMU)
		return ERR_PTR(-EINVAL);
	if (!capable(CAP_SYS_RAWIO))
		return ERR_PTR(-EPERM);

	return NULL;
}

static void vfio_noiommu_release(void *iommu_data)
{
}

static long vfio_noiommu_ioctl(void *iommu_data,
			       unsigned int cmd, unsigned long arg)
{
	if (cmd == VFIO_CHECK_EXTENSION)
		return vfio_noiommu && (arg == VFIO_NOIOMMU_IOMMU) ? 1 : 0;

	return -ENOTTY;
}

static int vfio_noiommu_attach_group(void *iommu_data,
		struct iommu_group *iommu_group, enum vfio_group_type type)
{
	return 0;
}

static void vfio_noiommu_detach_group(void *iommu_data,
				      struct iommu_group *iommu_group)
{
}

static const struct vfio_iommu_driver_ops vfio_noiommu_ops = {
	.name = "vfio-noiommu",
	.owner = THIS_MODULE,
	.open = vfio_noiommu_open,
	.release = vfio_noiommu_release,
	.ioctl = vfio_noiommu_ioctl,
	.attach_group = vfio_noiommu_attach_group,
	.detach_group = vfio_noiommu_detach_group,
};

/*
 * Only noiommu containers can use vfio-noiommu and noiommu containers can only
 * use vfio-noiommu.
 */
static bool vfio_iommu_driver_allowed(struct vfio_container *container,
				      const struct vfio_iommu_driver *driver)
{
	if (!IS_ENABLED(CONFIG_VFIO_NOIOMMU))
		return true;
	return container->noiommu == (driver->ops == &vfio_noiommu_ops);
}

/*
 * IOMMU driver registration
 */
int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops)
{
	struct vfio_iommu_driver *driver, *tmp;

	if (WARN_ON(!ops->register_device != !ops->unregister_device))
		return -EINVAL;

	driver = kzalloc(sizeof(*driver), GFP_KERNEL);
	if (!driver)
		return -ENOMEM;

	driver->ops = ops;

	mutex_lock(&vfio.iommu_drivers_lock);

	/* Check for duplicates */
	list_for_each_entry(tmp, &vfio.iommu_drivers_list, vfio_next) {
		if (tmp->ops == ops) {
			mutex_unlock(&vfio.iommu_drivers_lock);
			kfree(driver);
			return -EINVAL;
		}
	}

	list_add(&driver->vfio_next, &vfio.iommu_drivers_list);

	mutex_unlock(&vfio.iommu_drivers_lock);

	return 0;
}
EXPORT_SYMBOL_GPL(vfio_register_iommu_driver);

void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops)
{
	struct vfio_iommu_driver *driver;

	mutex_lock(&vfio.iommu_drivers_lock);
	list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
		if (driver->ops == ops) {
			list_del(&driver->vfio_next);
			mutex_unlock(&vfio.iommu_drivers_lock);
			kfree(driver);
			return;
		}
	}
	mutex_unlock(&vfio.iommu_drivers_lock);
}
EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver);

/*
 * Container objects - containers are created when /dev/vfio/vfio is
 * opened, but their lifecycle extends until the last user is done, so
 * it's freed via kref.  Must support container/group/device being
 * closed in any order.
 */
static void vfio_container_release(struct kref *kref)
{
	struct vfio_container *container;
	container = container_of(kref, struct vfio_container, kref);

	kfree(container);
}

static void vfio_container_get(struct vfio_container *container)
{
	kref_get(&container->kref);
}

static void vfio_container_put(struct vfio_container *container)
{
	kref_put(&container->kref, vfio_container_release);
}

void vfio_device_container_register(struct vfio_device *device)
{
	struct vfio_iommu_driver *iommu_driver =
		device->group->container->iommu_driver;

	if (iommu_driver && iommu_driver->ops->register_device)
		iommu_driver->ops->register_device(
			device->group->container->iommu_data, device);
}

void vfio_device_container_unregister(struct vfio_device *device)
{
	struct vfio_iommu_driver *iommu_driver =
		device->group->container->iommu_driver;

	if (iommu_driver && iommu_driver->ops->unregister_device)
		iommu_driver->ops->unregister_device(
			device->group->container->iommu_data, device);
}

static long
vfio_container_ioctl_check_extension(struct vfio_container *container,
				     unsigned long arg)
{
	struct vfio_iommu_driver *driver;
	long ret = 0;

	down_read(&container->group_lock);

	driver = container->iommu_driver;

	switch (arg) {
		/* No base extensions yet */
	default:
		/*
		 * If no driver is set, poll all registered drivers for
		 * extensions and return the first positive result.  If
		 * a driver is already set, further queries will be passed
		 * only to that driver.
		 */
		if (!driver) {
			mutex_lock(&vfio.iommu_drivers_lock);
			list_for_each_entry(driver, &vfio.iommu_drivers_list,
					    vfio_next) {

				if (!list_empty(&container->group_list) &&
				    !vfio_iommu_driver_allowed(container,
							       driver))
					continue;
				if (!try_module_get(driver->ops->owner))
					continue;

				ret = driver->ops->ioctl(NULL,
							 VFIO_CHECK_EXTENSION,
							 arg);
				module_put(driver->ops->owner);
				if (ret > 0)
					break;
			}
			mutex_unlock(&vfio.iommu_drivers_lock);
		} else
			ret = driver->ops->ioctl(container->iommu_data,
						 VFIO_CHECK_EXTENSION, arg);
	}

	up_read(&container->group_lock);

	return ret;
}

/* hold write lock on container->group_lock */
static int __vfio_container_attach_groups(struct vfio_container *container,
					  struct vfio_iommu_driver *driver,
					  void *data)
{
	struct vfio_group *group;
	int ret = -ENODEV;

	list_for_each_entry(group, &container->group_list, container_next) {
		ret = driver->ops->attach_group(data, group->iommu_group,
						group->type);
		if (ret)
			goto unwind;
	}

	return ret;

unwind:
	list_for_each_entry_continue_reverse(group, &container->group_list,
					     container_next) {
		driver->ops->detach_group(data, group->iommu_group);
	}

	return ret;
}

static long vfio_ioctl_set_iommu(struct vfio_container *container,
				 unsigned long arg)
{
	struct vfio_iommu_driver *driver;
	long ret = -ENODEV;

	down_write(&container->group_lock);

	/*
	 * The container is designed to be an unprivileged interface while
	 * the group can be assigned to specific users.  Therefore, only by
	 * adding a group to a container does the user get the privilege of
	 * enabling the iommu, which may allocate finite resources.  There
	 * is no unset_iommu, but by removing all the groups from a container,
	 * the container is deprivileged and returns to an unset state.
	 */
	if (list_empty(&container->group_list) || container->iommu_driver) {
		up_write(&container->group_lock);
		return -EINVAL;
	}

	mutex_lock(&vfio.iommu_drivers_lock);
	list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
		void *data;

		if (!vfio_iommu_driver_allowed(container, driver))
			continue;
		if (!try_module_get(driver->ops->owner))
			continue;

		/*
		 * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
		 * so test which iommu driver reported support for this
		 * extension and call open on them.  We also pass them the
		 * magic, allowing a single driver to support multiple
		 * interfaces if they'd like.
		 */
		if (driver->ops->ioctl(NULL, VFIO_CHECK_EXTENSION, arg) <= 0) {
			module_put(driver->ops->owner);
			continue;
		}

		data = driver->ops->open(arg);
		if (IS_ERR(data)) {
			ret = PTR_ERR(data);
			module_put(driver->ops->owner);
			continue;
		}

		ret = __vfio_container_attach_groups(container, driver, data);
		if (ret) {
			driver->ops->release(data);
			module_put(driver->ops->owner);
			continue;
		}

		container->iommu_driver = driver;
		container->iommu_data = data;
		break;
	}

	mutex_unlock(&vfio.iommu_drivers_lock);
	up_write(&container->group_lock);

	return ret;
}

static long vfio_fops_unl_ioctl(struct file *filep,
				unsigned int cmd, unsigned long arg)
{
	struct vfio_container *container = filep->private_data;
	struct vfio_iommu_driver *driver;
	void *data;
	long ret = -EINVAL;

	if (!container)
		return ret;

	switch (cmd) {
	case VFIO_GET_API_VERSION:
		ret = VFIO_API_VERSION;
		break;
	case VFIO_CHECK_EXTENSION:
		ret = vfio_container_ioctl_check_extension(container, arg);
		break;
	case VFIO_SET_IOMMU:
		ret = vfio_ioctl_set_iommu(container, arg);
		break;
	default:
		driver = container->iommu_driver;
		data = container->iommu_data;

		if (driver) /* passthrough all unrecognized ioctls */
			ret = driver->ops->ioctl(data, cmd, arg);
	}

	return ret;
}

static int vfio_fops_open(struct inode *inode, struct file *filep)
{
	struct vfio_container *container;

	container = kzalloc(sizeof(*container), GFP_KERNEL);
	if (!container)
		return -ENOMEM;

	INIT_LIST_HEAD(&container->group_list);
	init_rwsem(&container->group_lock);
	kref_init(&container->kref);

	filep->private_data = container;

	return 0;
}

static int vfio_fops_release(struct inode *inode, struct file *filep)
{
	struct vfio_container *container = filep->private_data;
	struct vfio_iommu_driver *driver = container->iommu_driver;

	if (driver && driver->ops->notify)
		driver->ops->notify(container->iommu_data,
				    VFIO_IOMMU_CONTAINER_CLOSE);

	filep->private_data = NULL;

	vfio_container_put(container);

	return 0;
}

static const struct file_operations vfio_fops = {
	.owner		= THIS_MODULE,
	.open		= vfio_fops_open,
	.release	= vfio_fops_release,
	.unlocked_ioctl	= vfio_fops_unl_ioctl,
	.compat_ioctl	= compat_ptr_ioctl,
};

struct vfio_container *vfio_container_from_file(struct file *file)
{
	struct vfio_container *container;

	/* Sanity check, is this really our fd? */
	if (file->f_op != &vfio_fops)
		return NULL;

	container = file->private_data;
	WARN_ON(!container); /* fget ensures we don't race vfio_release */
	return container;
}

static struct miscdevice vfio_dev = {
	.minor = VFIO_MINOR,
	.name = "vfio",
	.fops = &vfio_fops,
	.nodename = "vfio/vfio",
	.mode = S_IRUGO | S_IWUGO,
};

int vfio_container_attach_group(struct vfio_container *container,
				struct vfio_group *group)
{
	struct vfio_iommu_driver *driver;
	int ret = 0;

	lockdep_assert_held(&group->group_lock);

	if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))
		return -EPERM;

	down_write(&container->group_lock);

	/* Real groups and fake groups cannot mix */
	if (!list_empty(&container->group_list) &&
	    container->noiommu != (group->type == VFIO_NO_IOMMU)) {
		ret = -EPERM;
		goto out_unlock_container;
	}

	if (group->type == VFIO_IOMMU) {
		ret = iommu_group_claim_dma_owner(group->iommu_group, group);
		if (ret)
			goto out_unlock_container;
	}

	driver = container->iommu_driver;
	if (driver) {
		ret = driver->ops->attach_group(container->iommu_data,
						group->iommu_group,
						group->type);
		if (ret) {
			if (group->type == VFIO_IOMMU)
				iommu_group_release_dma_owner(
					group->iommu_group);
			goto out_unlock_container;
		}
	}

	group->container = container;
	group->container_users = 1;
	container->noiommu = (group->type == VFIO_NO_IOMMU);
	list_add(&group->container_next, &container->group_list);

	/* Get a reference on the container and mark a user within the group */
	vfio_container_get(container);

out_unlock_container:
	up_write(&container->group_lock);
	return ret;
}

void vfio_group_detach_container(struct vfio_group *group)
{
	struct vfio_container *container = group->container;
	struct vfio_iommu_driver *driver;

	lockdep_assert_held(&group->group_lock);
	WARN_ON(group->container_users != 1);

	down_write(&container->group_lock);

	driver = container->iommu_driver;
	if (driver)
		driver->ops->detach_group(container->iommu_data,
					  group->iommu_group);

	if (group->type == VFIO_IOMMU)
		iommu_group_release_dma_owner(group->iommu_group);

	group->container = NULL;
	group->container_users = 0;
	list_del(&group->container_next);

	/* Detaching the last group deprivileges a container, remove iommu */
	if (driver && list_empty(&container->group_list)) {
		driver->ops->release(container->iommu_data);
		module_put(driver->ops->owner);
		container->iommu_driver = NULL;
		container->iommu_data = NULL;
	}

	up_write(&container->group_lock);

	vfio_container_put(container);
}

int vfio_group_use_container(struct vfio_group *group)
{
	lockdep_assert_held(&group->group_lock);

	/*
	 * The container fd has been assigned with VFIO_GROUP_SET_CONTAINER but
	 * VFIO_SET_IOMMU hasn't been done yet.
	 */
	if (!group->container->iommu_driver)
		return -EINVAL;

	if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))
		return -EPERM;

	get_file(group->opened_file);
	group->container_users++;
	return 0;
}

void vfio_group_unuse_container(struct vfio_group *group)
{
	lockdep_assert_held(&group->group_lock);

	WARN_ON(group->container_users <= 1);
	group->container_users--;
	fput(group->opened_file);
}

int vfio_container_pin_pages(struct vfio_container *container,
			     struct iommu_group *iommu_group, dma_addr_t iova,
			     int npage, int prot, struct page **pages)
{
	struct vfio_iommu_driver *driver = container->iommu_driver;

	if (npage > VFIO_PIN_PAGES_MAX_ENTRIES)
		return -E2BIG;

	if (unlikely(!driver || !driver->ops->pin_pages))
		return -ENOTTY;
	return driver->ops->pin_pages(container->iommu_data, iommu_group, iova,
				      npage, prot, pages);
}

void vfio_container_unpin_pages(struct vfio_container *container,
				dma_addr_t iova, int npage)
{
	if (WARN_ON(npage <= 0 || npage > VFIO_PIN_PAGES_MAX_ENTRIES))
		return;

	container->iommu_driver->ops->unpin_pages(container->iommu_data, iova,
						  npage);
}

int vfio_container_dma_rw(struct vfio_container *container, dma_addr_t iova,
			  void *data, size_t len, bool write)
{
	struct vfio_iommu_driver *driver = container->iommu_driver;

	if (unlikely(!driver || !driver->ops->dma_rw))
		return -ENOTTY;
	return driver->ops->dma_rw(container->iommu_data, iova, data, len,
				   write);
}

int __init vfio_container_init(void)
{
	int ret;

	mutex_init(&vfio.iommu_drivers_lock);
	INIT_LIST_HEAD(&vfio.iommu_drivers_list);

	ret = misc_register(&vfio_dev);
	if (ret) {
		pr_err("vfio: misc device register failed\n");
		return ret;
	}

	if (IS_ENABLED(CONFIG_VFIO_NOIOMMU)) {
		ret = vfio_register_iommu_driver(&vfio_noiommu_ops);
		if (ret)
			goto err_misc;
	}
	return 0;

err_misc:
	misc_deregister(&vfio_dev);
	return ret;
}

void vfio_container_cleanup(void)
{
	if (IS_ENABLED(CONFIG_VFIO_NOIOMMU))
		vfio_unregister_iommu_driver(&vfio_noiommu_ops);
	misc_deregister(&vfio_dev);
	mutex_destroy(&vfio.iommu_drivers_lock);
}

MODULE_ALIAS_MISCDEV(VFIO_MINOR);
MODULE_ALIAS("devname:vfio/vfio");
Commit	Line	Data
cdc71fe4 JG	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
	4	*
	5	* VFIO container (/dev/vfio/vfio)
	6	*/
	7	#include <linux/file.h>
	8	#include <linux/slab.h>
	9	#include <linux/fs.h>
	10	#include <linux/capability.h>
	11	#include <linux/iommu.h>
	12	#include <linux/miscdevice.h>
	13	#include <linux/vfio.h>
	14	#include <uapi/linux/vfio.h>
	15
	16	#include "vfio.h"
	17
	18	struct vfio_container {
	19	struct kref kref;
	20	struct list_head group_list;
	21	struct rw_semaphore group_lock;
	22	struct vfio_iommu_driver *iommu_driver;
	23	void *iommu_data;
	24	bool noiommu;
	25	};
	26
	27	static struct vfio {
	28	struct list_head iommu_drivers_list;
	29	struct mutex iommu_drivers_lock;
	30	} vfio;
	31
	32	#ifdef CONFIG_VFIO_NOIOMMU
	33	bool vfio_noiommu __read_mostly;
	34	module_param_named(enable_unsafe_noiommu_mode,
	35	vfio_noiommu, bool, S_IRUGO \| S_IWUSR);
	36	MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)");
	37	#endif
	38
	39	static void *vfio_noiommu_open(unsigned long arg)
	40	{
	41	if (arg != VFIO_NOIOMMU_IOMMU)
	42	return ERR_PTR(-EINVAL);
	43	if (!capable(CAP_SYS_RAWIO))
	44	return ERR_PTR(-EPERM);
	45
	46	return NULL;
	47	}
	48
	49	static void vfio_noiommu_release(void *iommu_data)
	50	{
	51	}
	52
	53	static long vfio_noiommu_ioctl(void *iommu_data,
	54	unsigned int cmd, unsigned long arg)
	55	{
	56	if (cmd == VFIO_CHECK_EXTENSION)
	57	return vfio_noiommu && (arg == VFIO_NOIOMMU_IOMMU) ? 1 : 0;
	58
	59	return -ENOTTY;
	60	}
	61
	62	static int vfio_noiommu_attach_group(void *iommu_data,
	63	struct iommu_group *iommu_group, enum vfio_group_type type)
	64	{
65	return 0;
66	}
67
68	static void vfio_noiommu_detach_group(void *iommu_data,
69	struct iommu_group *iommu_group)
70	{
71	}
72
73	static const struct vfio_iommu_driver_ops vfio_noiommu_ops = {
74	.name = "vfio-noiommu",
75	.owner = THIS_MODULE,
76	.open = vfio_noiommu_open,
77	.release = vfio_noiommu_release,
78	.ioctl = vfio_noiommu_ioctl,
79	.attach_group = vfio_noiommu_attach_group,
80	.detach_group = vfio_noiommu_detach_group,
81	};
82
83	/*
84	* Only noiommu containers can use vfio-noiommu and noiommu containers can only
85	* use vfio-noiommu.
86	*/
87	static bool vfio_iommu_driver_allowed(struct vfio_container *container,
88	const struct vfio_iommu_driver *driver)
89	{
90	if (!IS_ENABLED(CONFIG_VFIO_NOIOMMU))
91	return true;
92	return container->noiommu == (driver->ops == &vfio_noiommu_ops);
93	}
94
95	/*
96	* IOMMU driver registration
97	*/
98	int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops)
99	{
100	struct vfio_iommu_driver driver, tmp;
101
102	if (WARN_ON(!ops->register_device != !ops->unregister_device))
103	return -EINVAL;
104
105	driver = kzalloc(sizeof(*driver), GFP_KERNEL);
106	if (!driver)
107	return -ENOMEM;
108
109	driver->ops = ops;
110
111	mutex_lock(&vfio.iommu_drivers_lock);
112
113	/* Check for duplicates */
114	list_for_each_entry(tmp, &vfio.iommu_drivers_list, vfio_next) {
115	if (tmp->ops == ops) {
116	mutex_unlock(&vfio.iommu_drivers_lock);
117	kfree(driver);
118	return -EINVAL;
119	}
120	}
121
122	list_add(&driver->vfio_next, &vfio.iommu_drivers_list);
123
124	mutex_unlock(&vfio.iommu_drivers_lock);
125
126	return 0;
127	}
128	EXPORT_SYMBOL_GPL(vfio_register_iommu_driver);
129
130	void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops)
131	{
132	struct vfio_iommu_driver *driver;
133
134	mutex_lock(&vfio.iommu_drivers_lock);
135	list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
136	if (driver->ops == ops) {
137	list_del(&driver->vfio_next);
138	mutex_unlock(&vfio.iommu_drivers_lock);
139	kfree(driver);
140	return;
141	}
142	}
143	mutex_unlock(&vfio.iommu_drivers_lock);
144	}
145	EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver);
146
147	/*
148	* Container objects - containers are created when /dev/vfio/vfio is
149	* opened, but their lifecycle extends until the last user is done, so
150	* it's freed via kref. Must support container/group/device being
151	* closed in any order.
152	*/
153	static void vfio_container_release(struct kref *kref)
154	{
155	struct vfio_container *container;
156	container = container_of(kref, struct vfio_container, kref);
157
158	kfree(container);
159	}
160
161	static void vfio_container_get(struct vfio_container *container)
162	{
163	kref_get(&container->kref);
164	}
165
166	static void vfio_container_put(struct vfio_container *container)
167	{
168	kref_put(&container->kref, vfio_container_release);
169	}
170
171	void vfio_device_container_register(struct vfio_device *device)
172	{
173	struct vfio_iommu_driver *iommu_driver =
174	device->group->container->iommu_driver;
175
176	if (iommu_driver && iommu_driver->ops->register_device)
177	iommu_driver->ops->register_device(
178	device->group->container->iommu_data, device);
179	}
180
181	void vfio_device_container_unregister(struct vfio_device *device)
182	{
183	struct vfio_iommu_driver *iommu_driver =
184	device->group->container->iommu_driver;
185
186	if (iommu_driver && iommu_driver->ops->unregister_device)
187	iommu_driver->ops->unregister_device(
188	device->group->container->iommu_data, device);
189	}
190
0d8227b6 JG	191	static long
	192	vfio_container_ioctl_check_extension(struct vfio_container *container,
	193	unsigned long arg)
cdc71fe4 JG	194	{
	195	struct vfio_iommu_driver *driver;
	196	long ret = 0;
	197
	198	down_read(&container->group_lock);
	199
	200	driver = container->iommu_driver;
	201
	202	switch (arg) {
	203	/* No base extensions yet */
	204	default:
	205	/*
	206	* If no driver is set, poll all registered drivers for
	207	* extensions and return the first positive result. If
	208	* a driver is already set, further queries will be passed
	209	* only to that driver.
	210	*/
	211	if (!driver) {
	212	mutex_lock(&vfio.iommu_drivers_lock);
	213	list_for_each_entry(driver, &vfio.iommu_drivers_list,
	214	vfio_next) {
	215
	216	if (!list_empty(&container->group_list) &&
	217	!vfio_iommu_driver_allowed(container,
	218	driver))
	219	continue;
	220	if (!try_module_get(driver->ops->owner))
	221	continue;
	222
	223	ret = driver->ops->ioctl(NULL,
	224	VFIO_CHECK_EXTENSION,
	225	arg);
	226	module_put(driver->ops->owner);
	227	if (ret > 0)
	228	break;
	229	}
	230	mutex_unlock(&vfio.iommu_drivers_lock);
	231	} else
	232	ret = driver->ops->ioctl(container->iommu_data,
	233	VFIO_CHECK_EXTENSION, arg);
	234	}
	235
	236	up_read(&container->group_lock);
	237
	238	return ret;
	239	}
	240
	241	/* hold write lock on container->group_lock */
	242	static int __vfio_container_attach_groups(struct vfio_container *container,
	243	struct vfio_iommu_driver *driver,
	244	void *data)
	245	{
	246	struct vfio_group *group;
	247	int ret = -ENODEV;
	248
	249	list_for_each_entry(group, &container->group_list, container_next) {
	250	ret = driver->ops->attach_group(data, group->iommu_group,
	251	group->type);
	252	if (ret)
	253	goto unwind;
	254	}
	255
	256	return ret;
	257
258	unwind:
259	list_for_each_entry_continue_reverse(group, &container->group_list,
260	container_next) {
261	driver->ops->detach_group(data, group->iommu_group);
262	}
263
264	return ret;
265	}
266
267	static long vfio_ioctl_set_iommu(struct vfio_container *container,
268	unsigned long arg)
269	{
270	struct vfio_iommu_driver *driver;
271	long ret = -ENODEV;
272
273	down_write(&container->group_lock);
274
275	/*
276	* The container is designed to be an unprivileged interface while
277	* the group can be assigned to specific users. Therefore, only by
278	* adding a group to a container does the user get the privilege of
279	* enabling the iommu, which may allocate finite resources. There
280	* is no unset_iommu, but by removing all the groups from a container,
281	* the container is deprivileged and returns to an unset state.
282	*/
283	if (list_empty(&container->group_list) \|\| container->iommu_driver) {
284	up_write(&container->group_lock);
285	return -EINVAL;
286	}
287
288	mutex_lock(&vfio.iommu_drivers_lock);
289	list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) {
290	void *data;
291
292	if (!vfio_iommu_driver_allowed(container, driver))
293	continue;
294	if (!try_module_get(driver->ops->owner))
295	continue;
296
297	/*
298	* The arg magic for SET_IOMMU is the same as CHECK_EXTENSION,
299	* so test which iommu driver reported support for this
300	* extension and call open on them. We also pass them the
301	* magic, allowing a single driver to support multiple
302	* interfaces if they'd like.
303	*/
304	if (driver->ops->ioctl(NULL, VFIO_CHECK_EXTENSION, arg) <= 0) {
305	module_put(driver->ops->owner);
306	continue;
307	}
308
309	data = driver->ops->open(arg);
310	if (IS_ERR(data)) {
311	ret = PTR_ERR(data);
312	module_put(driver->ops->owner);
313	continue;
314	}
315
316	ret = __vfio_container_attach_groups(container, driver, data);
317	if (ret) {
318	driver->ops->release(data);
319	module_put(driver->ops->owner);
320	continue;
321	}
322
323	container->iommu_driver = driver;
324	container->iommu_data = data;
325	break;
326	}
327
328	mutex_unlock(&vfio.iommu_drivers_lock);
329	up_write(&container->group_lock);
330
331	return ret;
332	}
333
334	static long vfio_fops_unl_ioctl(struct file *filep,
335	unsigned int cmd, unsigned long arg)
336	{
337	struct vfio_container *container = filep->private_data;
338	struct vfio_iommu_driver *driver;
339	void *data;
340	long ret = -EINVAL;
341
342	if (!container)
343	return ret;
344
345	switch (cmd) {
346	case VFIO_GET_API_VERSION:
347	ret = VFIO_API_VERSION;
348	break;
349	case VFIO_CHECK_EXTENSION:
350	ret = vfio_container_ioctl_check_extension(container, arg);
351	break;
352	case VFIO_SET_IOMMU:
353	ret = vfio_ioctl_set_iommu(container, arg);
354	break;
355	default:
356	driver = container->iommu_driver;
357	data = container->iommu_data;
358
359	if (driver) /* passthrough all unrecognized ioctls */
360	ret = driver->ops->ioctl(data, cmd, arg);
361	}
362
363	return ret;
364	}
365
366	static int vfio_fops_open(struct inode inode, struct file filep)
367	{
368	struct vfio_container *container;
369
370	container = kzalloc(sizeof(*container), GFP_KERNEL);
371	if (!container)
372	return -ENOMEM;
373
374	INIT_LIST_HEAD(&container->group_list);
375	init_rwsem(&container->group_lock);
376	kref_init(&container->kref);
377
378	filep->private_data = container;
379
380	return 0;
381	}
382
383	static int vfio_fops_release(struct inode inode, struct file filep)
384	{
385	struct vfio_container *container = filep->private_data;
386	struct vfio_iommu_driver *driver = container->iommu_driver;
387
388	if (driver && driver->ops->notify)
389	driver->ops->notify(container->iommu_data,
390	VFIO_IOMMU_CONTAINER_CLOSE);
391
392	filep->private_data = NULL;
393
394	vfio_container_put(container);
395
396	return 0;
397	}
398
399	static const struct file_operations vfio_fops = {
400	.owner = THIS_MODULE,
401	.open = vfio_fops_open,
402	.release = vfio_fops_release,
403	.unlocked_ioctl = vfio_fops_unl_ioctl,
404	.compat_ioctl = compat_ptr_ioctl,
405	};
406
407	struct vfio_container vfio_container_from_file(struct file file)
408	{
409	struct vfio_container *container;
410
411	/* Sanity check, is this really our fd? */
412	if (file->f_op != &vfio_fops)
413	return NULL;
414
415	container = file->private_data;
416	WARN_ON(!container); /* fget ensures we don't race vfio_release */
417	return container;
418	}
419
420	static struct miscdevice vfio_dev = {
421	.minor = VFIO_MINOR,
422	.name = "vfio",
423	.fops = &vfio_fops,
424	.nodename = "vfio/vfio",
425	.mode = S_IRUGO \| S_IWUGO,
426	};
427
428	int vfio_container_attach_group(struct vfio_container *container,
429	struct vfio_group *group)
430	{
431	struct vfio_iommu_driver *driver;
432	int ret = 0;
433
c82e81ab	434	lockdep_assert_held(&group->group_lock);
cdc71fe4 JG	435
	436	if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))
	437	return -EPERM;
	438
	439	down_write(&container->group_lock);
	440
	441	/* Real groups and fake groups cannot mix */
	442	if (!list_empty(&container->group_list) &&
	443	container->noiommu != (group->type == VFIO_NO_IOMMU)) {
	444	ret = -EPERM;
	445	goto out_unlock_container;
	446	}
	447
	448	if (group->type == VFIO_IOMMU) {
	449	ret = iommu_group_claim_dma_owner(group->iommu_group, group);
	450	if (ret)
	451	goto out_unlock_container;
	452	}
	453
	454	driver = container->iommu_driver;
	455	if (driver) {
	456	ret = driver->ops->attach_group(container->iommu_data,
	457	group->iommu_group,
	458	group->type);
	459	if (ret) {
	460	if (group->type == VFIO_IOMMU)
	461	iommu_group_release_dma_owner(
	462	group->iommu_group);
	463	goto out_unlock_container;
	464	}
	465	}
	466
	467	group->container = container;
	468	group->container_users = 1;
	469	container->noiommu = (group->type == VFIO_NO_IOMMU);
	470	list_add(&group->container_next, &container->group_list);
	471
	472	/* Get a reference on the container and mark a user within the group */
	473	vfio_container_get(container);
	474
	475	out_unlock_container:
	476	up_write(&container->group_lock);
	477	return ret;
	478	}
	479
	480	void vfio_group_detach_container(struct vfio_group *group)
	481	{
	482	struct vfio_container *container = group->container;
	483	struct vfio_iommu_driver *driver;
	484
c82e81ab	485	lockdep_assert_held(&group->group_lock);
cdc71fe4 JG	486	WARN_ON(group->container_users != 1);
	487
	488	down_write(&container->group_lock);
	489
	490	driver = container->iommu_driver;
	491	if (driver)
	492	driver->ops->detach_group(container->iommu_data,
	493	group->iommu_group);
	494
	495	if (group->type == VFIO_IOMMU)
	496	iommu_group_release_dma_owner(group->iommu_group);
	497
	498	group->container = NULL;
	499	group->container_users = 0;
	500	list_del(&group->container_next);
	501
	502	/* Detaching the last group deprivileges a container, remove iommu */
	503	if (driver && list_empty(&container->group_list)) {
	504	driver->ops->release(container->iommu_data);
	505	module_put(driver->ops->owner);
	506	container->iommu_driver = NULL;
	507	container->iommu_data = NULL;
	508	}
	509
	510	up_write(&container->group_lock);
	511
	512	vfio_container_put(container);
	513	}
	514
04f930c3	515	int vfio_group_use_container(struct vfio_group *group)
cdc71fe4	516	{
c82e81ab	517	lockdep_assert_held(&group->group_lock);
cdc71fe4	518
2a3dab19 JG	519	/*
	520	* The container fd has been assigned with VFIO_GROUP_SET_CONTAINER but
	521	* VFIO_SET_IOMMU hasn't been done yet.
	522	*/
	523	if (!group->container->iommu_driver)
cdc71fe4 JG	524	return -EINVAL;
	525
	526	if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO))
	527	return -EPERM;
	528
	529	get_file(group->opened_file);
	530	group->container_users++;
	531	return 0;
	532	}
	533
04f930c3	534	void vfio_group_unuse_container(struct vfio_group *group)
cdc71fe4	535	{
04f930c3	536	lockdep_assert_held(&group->group_lock);
bab6fabc	537
04f930c3 JG	538	WARN_ON(group->container_users <= 1);
	539	group->container_users--;
	540	fput(group->opened_file);
cdc71fe4 JG	541	}
cdc71fe4 JG	542
4741f2e9 JG	543	int vfio_container_pin_pages(struct vfio_container *container,
	544	struct iommu_group *iommu_group, dma_addr_t iova,
	545	int npage, int prot, struct page **pages)
cdc71fe4	546	{
4741f2e9	547	struct vfio_iommu_driver *driver = container->iommu_driver;
cdc71fe4 JG	548
	549	if (npage > VFIO_PIN_PAGES_MAX_ENTRIES)
	550	return -E2BIG;
	551
4741f2e9 JG	552	if (unlikely(!driver \|\| !driver->ops->pin_pages))
	553	return -ENOTTY;
	554	return driver->ops->pin_pages(container->iommu_data, iommu_group, iova,
	555	npage, prot, pages);
cdc71fe4	556	}
cdc71fe4	557
4741f2e9 JG	558	void vfio_container_unpin_pages(struct vfio_container *container,
4741f2e9 JG	559	dma_addr_t iova, int npage)
cdc71fe4	560	{
cdc71fe4 JG	561	if (WARN_ON(npage <= 0 \|\| npage > VFIO_PIN_PAGES_MAX_ENTRIES))
	562	return;
	563
4741f2e9 JG	564	container->iommu_driver->ops->unpin_pages(container->iommu_data, iova,
4741f2e9 JG	565	npage);
cdc71fe4	566	}
cdc71fe4	567
4741f2e9 JG	568	int vfio_container_dma_rw(struct vfio_container *container, dma_addr_t iova,
4741f2e9 JG	569	void *data, size_t len, bool write)
cdc71fe4	570	{
4741f2e9	571	struct vfio_iommu_driver *driver = container->iommu_driver;
cdc71fe4	572
4741f2e9 JG	573	if (unlikely(!driver \|\| !driver->ops->dma_rw))
	574	return -ENOTTY;
	575	return driver->ops->dma_rw(container->iommu_data, iova, data, len,
	576	write);
cdc71fe4	577	}
cdc71fe4 JG	578
	579	int __init vfio_container_init(void)
	580	{
	581	int ret;
	582
	583	mutex_init(&vfio.iommu_drivers_lock);
	584	INIT_LIST_HEAD(&vfio.iommu_drivers_list);
	585
	586	ret = misc_register(&vfio_dev);
	587	if (ret) {
	588	pr_err("vfio: misc device register failed\n");
	589	return ret;
	590	}
	591
	592	if (IS_ENABLED(CONFIG_VFIO_NOIOMMU)) {
	593	ret = vfio_register_iommu_driver(&vfio_noiommu_ops);
	594	if (ret)
	595	goto err_misc;
	596	}
	597	return 0;
	598
	599	err_misc:
	600	misc_deregister(&vfio_dev);
	601	return ret;
	602	}
	603
	604	void vfio_container_cleanup(void)
	605	{
	606	if (IS_ENABLED(CONFIG_VFIO_NOIOMMU))
	607	vfio_unregister_iommu_driver(&vfio_noiommu_ops);
	608	misc_deregister(&vfio_dev);
	609	mutex_destroy(&vfio.iommu_drivers_lock);
	610	}
81ab9890 JG	611
	612	MODULE_ALIAS_MISCDEV(VFIO_MINOR);
	613	MODULE_ALIAS("devname:vfio/vfio");