[linux.git] / drivers / gpu / drm / msm / msm_gem_submit.c

/*
 * Copyright (C) 2013 Red Hat
 * Author: Rob Clark <[email protected]>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published by
 * the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "msm_drv.h"
#include "msm_gpu.h"
#include "msm_gem.h"

/*
 * Cmdstream submission:
 */

/* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
#define BO_VALID    0x8000
#define BO_LOCKED   0x4000
#define BO_PINNED   0x2000

static inline void __user *to_user_ptr(u64 address)
{
	return (void __user *)(uintptr_t)address;
}

static struct msm_gem_submit *submit_create(struct drm_device *dev,
		struct msm_gpu *gpu, int nr)
{
	struct msm_gem_submit *submit;
	int sz = sizeof(*submit) + (nr * sizeof(submit->bos[0]));

	submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
	if (submit) {
		submit->dev = dev;
		submit->gpu = gpu;

		/* initially, until copy_from_user() and bo lookup succeeds: */
		submit->nr_bos = 0;
		submit->nr_cmds = 0;

		INIT_LIST_HEAD(&submit->bo_list);
		ww_acquire_init(&submit->ticket, &reservation_ww_class);
	}

	return submit;
}

static int submit_lookup_objects(struct msm_gem_submit *submit,
		struct drm_msm_gem_submit *args, struct drm_file *file)
{
	unsigned i;
	int ret = 0;

	spin_lock(&file->table_lock);

	for (i = 0; i < args->nr_bos; i++) {
		struct drm_msm_gem_submit_bo submit_bo;
		struct drm_gem_object *obj;
		struct msm_gem_object *msm_obj;
		void __user *userptr =
			to_user_ptr(args->bos + (i * sizeof(submit_bo)));

		ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
		if (ret) {
			ret = -EFAULT;
			goto out_unlock;
		}

		if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
			DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
			ret = -EINVAL;
			goto out_unlock;
		}

		submit->bos[i].flags = submit_bo.flags;
		/* in validate_objects() we figure out if this is true: */
		submit->bos[i].iova  = submit_bo.presumed;

		/* normally use drm_gem_object_lookup(), but for bulk lookup
		 * all under single table_lock just hit object_idr directly:
		 */
		obj = idr_find(&file->object_idr, submit_bo.handle);
		if (!obj) {
			DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
			ret = -EINVAL;
			goto out_unlock;
		}

		msm_obj = to_msm_bo(obj);

		if (!list_empty(&msm_obj->submit_entry)) {
			DRM_ERROR("handle %u at index %u already on submit list\n",
					submit_bo.handle, i);
			ret = -EINVAL;
			goto out_unlock;
		}

		drm_gem_object_reference(obj);

		submit->bos[i].obj = msm_obj;

		list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
	}

out_unlock:
	submit->nr_bos = i;
	spin_unlock(&file->table_lock);

	return ret;
}

static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
{
	struct msm_gem_object *msm_obj = submit->bos[i].obj;

	if (submit->bos[i].flags & BO_PINNED)
		msm_gem_put_iova(&msm_obj->base, submit->gpu->id);

	if (submit->bos[i].flags & BO_LOCKED)
		ww_mutex_unlock(&msm_obj->resv->lock);

	if (!(submit->bos[i].flags & BO_VALID))
		submit->bos[i].iova = 0;

	submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);
}

/* This is where we make sure all the bo's are reserved and pin'd: */
static int submit_validate_objects(struct msm_gem_submit *submit)
{
	int contended, slow_locked = -1, i, ret = 0;

retry:
	submit->valid = true;

	for (i = 0; i < submit->nr_bos; i++) {
		struct msm_gem_object *msm_obj = submit->bos[i].obj;
		uint32_t iova;

		if (slow_locked == i)
			slow_locked = -1;

		contended = i;

		if (!(submit->bos[i].flags & BO_LOCKED)) {
			ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
					&submit->ticket);
			if (ret)
				goto fail;
			submit->bos[i].flags |= BO_LOCKED;
		}


		/* if locking succeeded, pin bo: */
		ret = msm_gem_get_iova_locked(&msm_obj->base,
				submit->gpu->id, &iova);

		/* this would break the logic in the fail path.. there is no
		 * reason for this to happen, but just to be on the safe side
		 * let's notice if this starts happening in the future:
		 */
		WARN_ON(ret == -EDEADLK);

		if (ret)
			goto fail;

		submit->bos[i].flags |= BO_PINNED;

		if (iova == submit->bos[i].iova) {
			submit->bos[i].flags |= BO_VALID;
		} else {
			submit->bos[i].iova = iova;
			submit->bos[i].flags &= ~BO_VALID;
			submit->valid = false;
		}
	}

	ww_acquire_done(&submit->ticket);

	return 0;

fail:
	for (; i >= 0; i--)
		submit_unlock_unpin_bo(submit, i);

	if (slow_locked > 0)
		submit_unlock_unpin_bo(submit, slow_locked);

	if (ret == -EDEADLK) {
		struct msm_gem_object *msm_obj = submit->bos[contended].obj;
		/* we lost out in a seqno race, lock and retry.. */
		ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
				&submit->ticket);
		if (!ret) {
			submit->bos[contended].flags |= BO_LOCKED;
			slow_locked = contended;
			goto retry;
		}
	}

	return ret;
}

static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
		struct msm_gem_object **obj, uint32_t *iova, bool *valid)
{
	if (idx >= submit->nr_bos) {
		DRM_ERROR("invalid buffer index: %u (out of %u)\n",
				idx, submit->nr_bos);
		return -EINVAL;
	}

	if (obj)
		*obj = submit->bos[idx].obj;
	if (iova)
		*iova = submit->bos[idx].iova;
	if (valid)
		*valid = !!(submit->bos[idx].flags & BO_VALID);

	return 0;
}

/* process the reloc's and patch up the cmdstream as needed: */
static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
		uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
{
	uint32_t i, last_offset = 0;
	uint32_t *ptr;
	int ret;

	if (offset % 4) {
		DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
		return -EINVAL;
	}

	/* For now, just map the entire thing.  Eventually we probably
	 * to do it page-by-page, w/ kmap() if not vmap()d..
	 */
	ptr = msm_gem_vaddr_locked(&obj->base);

	if (IS_ERR(ptr)) {
		ret = PTR_ERR(ptr);
		DBG("failed to map: %d", ret);
		return ret;
	}

	for (i = 0; i < nr_relocs; i++) {
		struct drm_msm_gem_submit_reloc submit_reloc;
		void __user *userptr =
			to_user_ptr(relocs + (i * sizeof(submit_reloc)));
		uint32_t iova, off;
		bool valid;

		ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
		if (ret)
			return -EFAULT;

		if (submit_reloc.submit_offset % 4) {
			DRM_ERROR("non-aligned reloc offset: %u\n",
					submit_reloc.submit_offset);
			return -EINVAL;
		}

		/* offset in dwords: */
		off = submit_reloc.submit_offset / 4;

		if ((off >= (obj->base.size / 4)) ||
				(off < last_offset)) {
			DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
			return -EINVAL;
		}

		ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
		if (ret)
			return ret;

		if (valid)
			continue;

		iova += submit_reloc.reloc_offset;

		if (submit_reloc.shift < 0)
			iova >>= -submit_reloc.shift;
		else
			iova <<= submit_reloc.shift;

		ptr[off] = iova | submit_reloc.or;

		last_offset = off;
	}

	return 0;
}

static void submit_cleanup(struct msm_gem_submit *submit, bool fail)
{
	unsigned i;

	for (i = 0; i < submit->nr_bos; i++) {
		struct msm_gem_object *msm_obj = submit->bos[i].obj;
		submit_unlock_unpin_bo(submit, i);
		list_del_init(&msm_obj->submit_entry);
		drm_gem_object_unreference(&msm_obj->base);
	}

	ww_acquire_fini(&submit->ticket);
	kfree(submit);
}

int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
		struct drm_file *file)
{
	struct msm_drm_private *priv = dev->dev_private;
	struct drm_msm_gem_submit *args = data;
	struct msm_file_private *ctx = file->driver_priv;
	struct msm_gem_submit *submit;
	struct msm_gpu *gpu;
	unsigned i;
	int ret;

	/* for now, we just have 3d pipe.. eventually this would need to
	 * be more clever to dispatch to appropriate gpu module:
	 */
	if (args->pipe != MSM_PIPE_3D0)
		return -EINVAL;

	gpu = priv->gpu;

	if (args->nr_cmds > MAX_CMDS)
		return -EINVAL;

	mutex_lock(&dev->struct_mutex);

	submit = submit_create(dev, gpu, args->nr_bos);
	if (!submit) {
		ret = -ENOMEM;
		goto out;
	}

	ret = submit_lookup_objects(submit, args, file);
	if (ret)
		goto out;

	ret = submit_validate_objects(submit);
	if (ret)
		goto out;

	for (i = 0; i < args->nr_cmds; i++) {
		struct drm_msm_gem_submit_cmd submit_cmd;
		void __user *userptr =
			to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
		struct msm_gem_object *msm_obj;
		uint32_t iova;

		ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
		if (ret) {
			ret = -EFAULT;
			goto out;
		}

		/* validate input from userspace: */
		switch (submit_cmd.type) {
		case MSM_SUBMIT_CMD_BUF:
		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
			break;
		default:
			DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
			ret = -EINVAL;
			goto out;
		}

		ret = submit_bo(submit, submit_cmd.submit_idx,
				&msm_obj, &iova, NULL);
		if (ret)
			goto out;

		if (submit_cmd.size % 4) {
			DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
					submit_cmd.size);
			ret = -EINVAL;
			goto out;
		}

		if ((submit_cmd.size + submit_cmd.submit_offset) >=
				msm_obj->base.size) {
			DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
			ret = -EINVAL;
			goto out;
		}

		submit->cmd[i].type = submit_cmd.type;
		submit->cmd[i].size = submit_cmd.size / 4;
		submit->cmd[i].iova = iova + submit_cmd.submit_offset;
		submit->cmd[i].idx  = submit_cmd.submit_idx;

		if (submit->valid)
			continue;

		ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
				submit_cmd.nr_relocs, submit_cmd.relocs);
		if (ret)
			goto out;
	}

	submit->nr_cmds = i;

	ret = msm_gpu_submit(gpu, submit, ctx);

	args->fence = submit->fence;

out:
	if (submit)
		submit_cleanup(submit, !!ret);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}
Commit	Line	Data
7198e6b0 RC	1	/*
	2	* Copyright (C) 2013 Red Hat
	3	* Author: Rob Clark <[email protected]>
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms of the GNU General Public License version 2 as published by
	7	* the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful, but WITHOUT
	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	12	* more details.
	13	*
	14	* You should have received a copy of the GNU General Public License along with
	15	* this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
	18	#include "msm_drv.h"
	19	#include "msm_gpu.h"
	20	#include "msm_gem.h"
	21
	22	/*
	23	* Cmdstream submission:
	24	*/
	25
7198e6b0 RC	26	/* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
	27	#define BO_VALID 0x8000
	28	#define BO_LOCKED 0x4000
	29	#define BO_PINNED 0x2000
	30
	31	static inline void __user *to_user_ptr(u64 address)
	32	{
	33	return (void __user *)(uintptr_t)address;
	34	}
	35
	36	static struct msm_gem_submit submit_create(struct drm_device dev,
	37	struct msm_gpu *gpu, int nr)
	38	{
	39	struct msm_gem_submit *submit;
	40	int sz = sizeof(submit) + (nr sizeof(submit->bos[0]));
	41
	42	submit = kmalloc(sz, GFP_TEMPORARY \| __GFP_NOWARN \| __GFP_NORETRY);
	43	if (submit) {
	44	submit->dev = dev;
	45	submit->gpu = gpu;
	46
	47	/* initially, until copy_from_user() and bo lookup succeeds: */
	48	submit->nr_bos = 0;
	49	submit->nr_cmds = 0;
	50
	51	INIT_LIST_HEAD(&submit->bo_list);
	52	ww_acquire_init(&submit->ticket, &reservation_ww_class);
	53	}
	54
	55	return submit;
	56	}
	57
	58	static int submit_lookup_objects(struct msm_gem_submit *submit,
	59	struct drm_msm_gem_submit args, struct drm_file file)
	60	{
	61	unsigned i;
	62	int ret = 0;
	63
	64	spin_lock(&file->table_lock);
	65
	66	for (i = 0; i < args->nr_bos; i++) {
	67	struct drm_msm_gem_submit_bo submit_bo;
	68	struct drm_gem_object *obj;
	69	struct msm_gem_object *msm_obj;
	70	void __user *userptr =
	71	to_user_ptr(args->bos + (i * sizeof(submit_bo)));
	72
	73	ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
	74	if (ret) {
	75	ret = -EFAULT;
	76	goto out_unlock;
	77	}
	78
93ddb0d3	79	if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
19872533	80	DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
7198e6b0 RC	81	ret = -EINVAL;
	82	goto out_unlock;
	83	}
	84
	85	submit->bos[i].flags = submit_bo.flags;
	86	/* in validate_objects() we figure out if this is true: */
	87	submit->bos[i].iova = submit_bo.presumed;
	88
	89	/* normally use drm_gem_object_lookup(), but for bulk lookup
	90	* all under single table_lock just hit object_idr directly:
	91	*/
	92	obj = idr_find(&file->object_idr, submit_bo.handle);
	93	if (!obj) {
19872533	94	DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
7198e6b0 RC	95	ret = -EINVAL;
	96	goto out_unlock;
	97	}
	98
	99	msm_obj = to_msm_bo(obj);
	100
	101	if (!list_empty(&msm_obj->submit_entry)) {
19872533	102	DRM_ERROR("handle %u at index %u already on submit list\n",
7198e6b0 RC	103	submit_bo.handle, i);
	104	ret = -EINVAL;
	105	goto out_unlock;
	106	}
	107
	108	drm_gem_object_reference(obj);
	109
	110	submit->bos[i].obj = msm_obj;
	111
	112	list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
	113	}
	114
	115	out_unlock:
	116	submit->nr_bos = i;
	117	spin_unlock(&file->table_lock);
	118
	119	return ret;
	120	}
	121
	122	static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
	123	{
	124	struct msm_gem_object *msm_obj = submit->bos[i].obj;
	125
	126	if (submit->bos[i].flags & BO_PINNED)
	127	msm_gem_put_iova(&msm_obj->base, submit->gpu->id);
	128
	129	if (submit->bos[i].flags & BO_LOCKED)
	130	ww_mutex_unlock(&msm_obj->resv->lock);
	131
	132	if (!(submit->bos[i].flags & BO_VALID))
	133	submit->bos[i].iova = 0;
	134
	135	submit->bos[i].flags &= ~(BO_LOCKED \| BO_PINNED);
	136	}
	137
	138	/* This is where we make sure all the bo's are reserved and pin'd: */
	139	static int submit_validate_objects(struct msm_gem_submit *submit)
	140	{
	141	int contended, slow_locked = -1, i, ret = 0;
	142
	143	retry:
	144	submit->valid = true;
	145
	146	for (i = 0; i < submit->nr_bos; i++) {
	147	struct msm_gem_object *msm_obj = submit->bos[i].obj;
	148	uint32_t iova;
	149
	150	if (slow_locked == i)
	151	slow_locked = -1;
	152
	153	contended = i;
	154
	155	if (!(submit->bos[i].flags & BO_LOCKED)) {
	156	ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
	157	&submit->ticket);
	158	if (ret)
	159	goto fail;
	160	submit->bos[i].flags \|= BO_LOCKED;
	161	}
	162
	163
	164	/* if locking succeeded, pin bo: */
c2703b13	165	ret = msm_gem_get_iova_locked(&msm_obj->base,
7198e6b0 RC	166	submit->gpu->id, &iova);
	167
	168	/* this would break the logic in the fail path.. there is no
	169	* reason for this to happen, but just to be on the safe side
	170	* let's notice if this starts happening in the future:
	171	*/
	172	WARN_ON(ret == -EDEADLK);
	173
	174	if (ret)
	175	goto fail;
	176
	177	submit->bos[i].flags \|= BO_PINNED;
	178
	179	if (iova == submit->bos[i].iova) {
	180	submit->bos[i].flags \|= BO_VALID;
	181	} else {
	182	submit->bos[i].iova = iova;
	183	submit->bos[i].flags &= ~BO_VALID;
	184	submit->valid = false;
	185	}
	186	}
	187
	188	ww_acquire_done(&submit->ticket);
	189
	190	return 0;
	191
	192	fail:
	193	for (; i >= 0; i--)
	194	submit_unlock_unpin_bo(submit, i);
	195
	196	if (slow_locked > 0)
	197	submit_unlock_unpin_bo(submit, slow_locked);
	198
	199	if (ret == -EDEADLK) {
	200	struct msm_gem_object *msm_obj = submit->bos[contended].obj;
	201	/* we lost out in a seqno race, lock and retry.. */
	202	ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
	203	&submit->ticket);
	204	if (!ret) {
	205	submit->bos[contended].flags \|= BO_LOCKED;
	206	slow_locked = contended;
	207	goto retry;
	208	}
	209	}
	210
	211	return ret;
	212	}
	213
	214	static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
	215	struct msm_gem_object *obj, uint32_t iova, bool *valid)
	216	{
	217	if (idx >= submit->nr_bos) {
19872533 RC	218	DRM_ERROR("invalid buffer index: %u (out of %u)\n",
	219	idx, submit->nr_bos);
	220	return -EINVAL;
7198e6b0 RC	221	}
	222
	223	if (obj)
	224	*obj = submit->bos[idx].obj;
	225	if (iova)
	226	*iova = submit->bos[idx].iova;
	227	if (valid)
	228	*valid = !!(submit->bos[idx].flags & BO_VALID);
	229
	230	return 0;
	231	}
	232
	233	/* process the reloc's and patch up the cmdstream as needed: */
	234	static int submit_reloc(struct msm_gem_submit submit, struct msm_gem_object obj,
	235	uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
	236	{
	237	uint32_t i, last_offset = 0;
	238	uint32_t *ptr;
	239	int ret;
	240
	241	if (offset % 4) {
19872533	242	DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
7198e6b0 RC	243	return -EINVAL;
	244	}
	245
	246	/* For now, just map the entire thing. Eventually we probably
	247	* to do it page-by-page, w/ kmap() if not vmap()d..
	248	*/
c2703b13	249	ptr = msm_gem_vaddr_locked(&obj->base);
7198e6b0 RC	250
	251	if (IS_ERR(ptr)) {
	252	ret = PTR_ERR(ptr);
	253	DBG("failed to map: %d", ret);
	254	return ret;
	255	}
	256
	257	for (i = 0; i < nr_relocs; i++) {
	258	struct drm_msm_gem_submit_reloc submit_reloc;
	259	void __user *userptr =
	260	to_user_ptr(relocs + (i * sizeof(submit_reloc)));
	261	uint32_t iova, off;
	262	bool valid;
	263
	264	ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
	265	if (ret)
	266	return -EFAULT;
	267
	268	if (submit_reloc.submit_offset % 4) {
19872533	269	DRM_ERROR("non-aligned reloc offset: %u\n",
7198e6b0 RC	270	submit_reloc.submit_offset);
	271	return -EINVAL;
	272	}
	273
	274	/* offset in dwords: */
	275	off = submit_reloc.submit_offset / 4;
	276
	277	if ((off >= (obj->base.size / 4)) \|\|
	278	(off < last_offset)) {
19872533	279	DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
7198e6b0 RC	280	return -EINVAL;
	281	}
	282
	283	ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
	284	if (ret)
	285	return ret;
	286
	287	if (valid)
	288	continue;
	289
	290	iova += submit_reloc.reloc_offset;
	291
	292	if (submit_reloc.shift < 0)
	293	iova >>= -submit_reloc.shift;
	294	else
	295	iova <<= submit_reloc.shift;
	296
	297	ptr[off] = iova \| submit_reloc.or;
	298
	299	last_offset = off;
	300	}
	301
	302	return 0;
	303	}
	304
	305	static void submit_cleanup(struct msm_gem_submit *submit, bool fail)
	306	{
	307	unsigned i;
	308
7198e6b0 RC	309	for (i = 0; i < submit->nr_bos; i++) {
	310	struct msm_gem_object *msm_obj = submit->bos[i].obj;
	311	submit_unlock_unpin_bo(submit, i);
	312	list_del_init(&msm_obj->submit_entry);
	313	drm_gem_object_unreference(&msm_obj->base);
	314	}
7198e6b0 RC	315
	316	ww_acquire_fini(&submit->ticket);
	317	kfree(submit);
	318	}
	319
	320	int msm_ioctl_gem_submit(struct drm_device dev, void data,
	321	struct drm_file *file)
	322	{
	323	struct msm_drm_private *priv = dev->dev_private;
	324	struct drm_msm_gem_submit *args = data;
	325	struct msm_file_private *ctx = file->driver_priv;
	326	struct msm_gem_submit *submit;
	327	struct msm_gpu *gpu;
	328	unsigned i;
	329	int ret;
	330
	331	/* for now, we just have 3d pipe.. eventually this would need to
	332	* be more clever to dispatch to appropriate gpu module:
	333	*/
	334	if (args->pipe != MSM_PIPE_3D0)
	335	return -EINVAL;
	336
	337	gpu = priv->gpu;
	338
	339	if (args->nr_cmds > MAX_CMDS)
	340	return -EINVAL;
	341
c2703b13 RC	342	mutex_lock(&dev->struct_mutex);
c2703b13 RC	343
7198e6b0 RC	344	submit = submit_create(dev, gpu, args->nr_bos);
	345	if (!submit) {
	346	ret = -ENOMEM;
	347	goto out;
	348	}
	349
	350	ret = submit_lookup_objects(submit, args, file);
	351	if (ret)
	352	goto out;
	353
	354	ret = submit_validate_objects(submit);
	355	if (ret)
	356	goto out;
	357
	358	for (i = 0; i < args->nr_cmds; i++) {
	359	struct drm_msm_gem_submit_cmd submit_cmd;
	360	void __user *userptr =
	361	to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
	362	struct msm_gem_object *msm_obj;
	363	uint32_t iova;
	364
	365	ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
	366	if (ret) {
	367	ret = -EFAULT;
	368	goto out;
	369	}
	370
93ddb0d3 RC	371	/* validate input from userspace: */
	372	switch (submit_cmd.type) {
	373	case MSM_SUBMIT_CMD_BUF:
	374	case MSM_SUBMIT_CMD_IB_TARGET_BUF:
	375	case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
	376	break;
	377	default:
	378	DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
	379	ret = -EINVAL;
	380	goto out;
	381	}
	382
7198e6b0 RC	383	ret = submit_bo(submit, submit_cmd.submit_idx,
	384	&msm_obj, &iova, NULL);
	385	if (ret)
	386	goto out;
	387
	388	if (submit_cmd.size % 4) {
19872533	389	DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
7198e6b0 RC	390	submit_cmd.size);
	391	ret = -EINVAL;
	392	goto out;
	393	}
	394
19872533 RC	395	if ((submit_cmd.size + submit_cmd.submit_offset) >=
	396	msm_obj->base.size) {
	397	DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
7198e6b0 RC	398	ret = -EINVAL;
	399	goto out;
	400	}
	401
	402	submit->cmd[i].type = submit_cmd.type;
	403	submit->cmd[i].size = submit_cmd.size / 4;
	404	submit->cmd[i].iova = iova + submit_cmd.submit_offset;
a7d3c950	405	submit->cmd[i].idx = submit_cmd.submit_idx;
7198e6b0 RC	406
	407	if (submit->valid)
	408	continue;
	409
	410	ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
	411	submit_cmd.nr_relocs, submit_cmd.relocs);
	412	if (ret)
	413	goto out;
	414	}
	415
	416	submit->nr_cmds = i;
	417
	418	ret = msm_gpu_submit(gpu, submit, ctx);
	419
	420	args->fence = submit->fence;
	421
	422	out:
	423	if (submit)
	424	submit_cleanup(submit, !!ret);
c2703b13	425	mutex_unlock(&dev->struct_mutex);
7198e6b0 RC	426	return ret;
7198e6b0 RC	427	}