[linux.git] / drivers / base / dma-mapping.c

/*
 * drivers/base/dma-mapping.c - arch-independent dma-mapping routines
 *
 * Copyright (c) 2006  SUSE Linux Products GmbH
 * Copyright (c) 2006  Tejun Heo <[email protected]>
 *
 * This file is released under the GPLv2.
 */

#include <linux/dma-mapping.h>
#include <linux/export.h>
#include <linux/gfp.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>

/*
 * Managed DMA API
 */
struct dma_devres {
	size_t		size;
	void		*vaddr;
	dma_addr_t	dma_handle;
};

static void dmam_coherent_release(struct device *dev, void *res)
{
	struct dma_devres *this = res;

	dma_free_coherent(dev, this->size, this->vaddr, this->dma_handle);
}

static void dmam_noncoherent_release(struct device *dev, void *res)
{
	struct dma_devres *this = res;

	dma_free_noncoherent(dev, this->size, this->vaddr, this->dma_handle);
}

static int dmam_match(struct device *dev, void *res, void *match_data)
{
	struct dma_devres *this = res, *match = match_data;

	if (this->vaddr == match->vaddr) {
		WARN_ON(this->size != match->size ||
			this->dma_handle != match->dma_handle);
		return 1;
	}
	return 0;
}

/**
 * dmam_alloc_coherent - Managed dma_alloc_coherent()
 * @dev: Device to allocate coherent memory for
 * @size: Size of allocation
 * @dma_handle: Out argument for allocated DMA handle
 * @gfp: Allocation flags
 *
 * Managed dma_alloc_coherent().  Memory allocated using this function
 * will be automatically released on driver detach.
 *
 * RETURNS:
 * Pointer to allocated memory on success, NULL on failure.
 */
void *dmam_alloc_coherent(struct device *dev, size_t size,
			   dma_addr_t *dma_handle, gfp_t gfp)
{
	struct dma_devres *dr;
	void *vaddr;

	dr = devres_alloc(dmam_coherent_release, sizeof(*dr), gfp);
	if (!dr)
		return NULL;

	vaddr = dma_alloc_coherent(dev, size, dma_handle, gfp);
	if (!vaddr) {
		devres_free(dr);
		return NULL;
	}

	dr->vaddr = vaddr;
	dr->dma_handle = *dma_handle;
	dr->size = size;

	devres_add(dev, dr);

	return vaddr;
}
EXPORT_SYMBOL(dmam_alloc_coherent);

/**
 * dmam_free_coherent - Managed dma_free_coherent()
 * @dev: Device to free coherent memory for
 * @size: Size of allocation
 * @vaddr: Virtual address of the memory to free
 * @dma_handle: DMA handle of the memory to free
 *
 * Managed dma_free_coherent().
 */
void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
			dma_addr_t dma_handle)
{
	struct dma_devres match_data = { size, vaddr, dma_handle };

	dma_free_coherent(dev, size, vaddr, dma_handle);
	WARN_ON(devres_destroy(dev, dmam_coherent_release, dmam_match,
			       &match_data));
}
EXPORT_SYMBOL(dmam_free_coherent);

/**
 * dmam_alloc_non_coherent - Managed dma_alloc_noncoherent()
 * @dev: Device to allocate non_coherent memory for
 * @size: Size of allocation
 * @dma_handle: Out argument for allocated DMA handle
 * @gfp: Allocation flags
 *
 * Managed dma_alloc_noncoherent().  Memory allocated using this
 * function will be automatically released on driver detach.
 *
 * RETURNS:
 * Pointer to allocated memory on success, NULL on failure.
 */
void *dmam_alloc_noncoherent(struct device *dev, size_t size,
			     dma_addr_t *dma_handle, gfp_t gfp)
{
	struct dma_devres *dr;
	void *vaddr;

	dr = devres_alloc(dmam_noncoherent_release, sizeof(*dr), gfp);
	if (!dr)
		return NULL;

	vaddr = dma_alloc_noncoherent(dev, size, dma_handle, gfp);
	if (!vaddr) {
		devres_free(dr);
		return NULL;
	}

	dr->vaddr = vaddr;
	dr->dma_handle = *dma_handle;
	dr->size = size;

	devres_add(dev, dr);

	return vaddr;
}
EXPORT_SYMBOL(dmam_alloc_noncoherent);

/**
 * dmam_free_coherent - Managed dma_free_noncoherent()
 * @dev: Device to free noncoherent memory for
 * @size: Size of allocation
 * @vaddr: Virtual address of the memory to free
 * @dma_handle: DMA handle of the memory to free
 *
 * Managed dma_free_noncoherent().
 */
void dmam_free_noncoherent(struct device *dev, size_t size, void *vaddr,
			   dma_addr_t dma_handle)
{
	struct dma_devres match_data = { size, vaddr, dma_handle };

	dma_free_noncoherent(dev, size, vaddr, dma_handle);
	WARN_ON(!devres_destroy(dev, dmam_noncoherent_release, dmam_match,
				&match_data));
}
EXPORT_SYMBOL(dmam_free_noncoherent);

#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT

static void dmam_coherent_decl_release(struct device *dev, void *res)
{
	dma_release_declared_memory(dev);
}

/**
 * dmam_declare_coherent_memory - Managed dma_declare_coherent_memory()
 * @dev: Device to declare coherent memory for
 * @phys_addr: Physical address of coherent memory to be declared
 * @device_addr: Device address of coherent memory to be declared
 * @size: Size of coherent memory to be declared
 * @flags: Flags
 *
 * Managed dma_declare_coherent_memory().
 *
 * RETURNS:
 * 0 on success, -errno on failure.
 */
int dmam_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
				 dma_addr_t device_addr, size_t size, int flags)
{
	void *res;
	int rc;

	res = devres_alloc(dmam_coherent_decl_release, 0, GFP_KERNEL);
	if (!res)
		return -ENOMEM;

	rc = dma_declare_coherent_memory(dev, phys_addr, device_addr, size,
					 flags);
	if (rc) {
		devres_add(dev, res);
		rc = 0;
	} else {
		devres_free(res);
		rc = -ENOMEM;
	}

	return rc;
}
EXPORT_SYMBOL(dmam_declare_coherent_memory);

/**
 * dmam_release_declared_memory - Managed dma_release_declared_memory().
 * @dev: Device to release declared coherent memory for
 *
 * Managed dmam_release_declared_memory().
 */
void dmam_release_declared_memory(struct device *dev)
{
	WARN_ON(devres_destroy(dev, dmam_coherent_decl_release, NULL, NULL));
}
EXPORT_SYMBOL(dmam_release_declared_memory);

#endif

/*
 * Create scatter-list for the already allocated DMA buffer.
 */
int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
		 void *cpu_addr, dma_addr_t handle, size_t size)
{
	struct page *page = virt_to_page(cpu_addr);
	int ret;

	ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
	if (unlikely(ret))
		return ret;

	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
	return 0;
}
EXPORT_SYMBOL(dma_common_get_sgtable);

/*
 * Create userspace mapping for the DMA-coherent memory.
 */
int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
		    void *cpu_addr, dma_addr_t dma_addr, size_t size)
{
	int ret = -ENXIO;
#if defined(CONFIG_MMU) && !defined(CONFIG_ARCH_NO_COHERENT_DMA_MMAP)
	unsigned long user_count = vma_pages(vma);
	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
	unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr));
	unsigned long off = vma->vm_pgoff;

	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);

	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
		return ret;

	if (off < count && user_count <= (count - off)) {
		ret = remap_pfn_range(vma, vma->vm_start,
				      pfn + off,
				      user_count << PAGE_SHIFT,
				      vma->vm_page_prot);
	}
#endif	/* CONFIG_MMU && !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */

	return ret;
}
EXPORT_SYMBOL(dma_common_mmap);

#ifdef CONFIG_MMU
/*
 * remaps an array of PAGE_SIZE pages into another vm_area
 * Cannot be used in non-sleeping contexts
 */
void *dma_common_pages_remap(struct page **pages, size_t size,
			unsigned long vm_flags, pgprot_t prot,
			const void *caller)
{
	struct vm_struct *area;

	area = get_vm_area_caller(size, vm_flags, caller);
	if (!area)
		return NULL;

	area->pages = pages;

	if (map_vm_area(area, prot, pages)) {
		vunmap(area->addr);
		return NULL;
	}

	return area->addr;
}

/*
 * remaps an allocated contiguous region into another vm_area.
 * Cannot be used in non-sleeping contexts
 */

void *dma_common_contiguous_remap(struct page *page, size_t size,
			unsigned long vm_flags,
			pgprot_t prot, const void *caller)
{
	int i;
	struct page **pages;
	void *ptr;
	unsigned long pfn;

	pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL);
	if (!pages)
		return NULL;

	for (i = 0, pfn = page_to_pfn(page); i < (size >> PAGE_SHIFT); i++)
		pages[i] = pfn_to_page(pfn + i);

	ptr = dma_common_pages_remap(pages, size, vm_flags, prot, caller);

	kfree(pages);

	return ptr;
}

/*
 * unmaps a range previously mapped by dma_common_*_remap
 */
void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
{
	struct vm_struct *area = find_vm_area(cpu_addr);

	if (!area || (area->flags & vm_flags) != vm_flags) {
		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
		return;
	}

	unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
	vunmap(cpu_addr);
}
#endif
Commit	Line	Data
9ac7849e TH	1	/*
	2	* drivers/base/dma-mapping.c - arch-independent dma-mapping routines
	3	*
	4	* Copyright (c) 2006 SUSE Linux Products GmbH
	5	* Copyright (c) 2006 Tejun Heo <[email protected]>
	6	*
	7	* This file is released under the GPLv2.
	8	*/
	9
	10	#include <linux/dma-mapping.h>
1b6bc32f	11	#include <linux/export.h>
5a0e3ad6	12	#include <linux/gfp.h>
513510dd LA	13	#include <linux/slab.h>
513510dd LA	14	#include <linux/vmalloc.h>
9ac7849e TH	15
	16	/*
	17	* Managed DMA API
	18	*/
	19	struct dma_devres {
	20	size_t size;
	21	void *vaddr;
	22	dma_addr_t dma_handle;
	23	};
	24
	25	static void dmam_coherent_release(struct device dev, void res)
	26	{
	27	struct dma_devres *this = res;
	28
	29	dma_free_coherent(dev, this->size, this->vaddr, this->dma_handle);
	30	}
	31
	32	static void dmam_noncoherent_release(struct device dev, void res)
	33	{
	34	struct dma_devres *this = res;
	35
	36	dma_free_noncoherent(dev, this->size, this->vaddr, this->dma_handle);
	37	}
	38
	39	static int dmam_match(struct device dev, void res, void *match_data)
	40	{
	41	struct dma_devres this = res, match = match_data;
	42
	43	if (this->vaddr == match->vaddr) {
	44	WARN_ON(this->size != match->size \|\|
	45	this->dma_handle != match->dma_handle);
	46	return 1;
	47	}
	48	return 0;
	49	}
	50
	51	/**
	52	* dmam_alloc_coherent - Managed dma_alloc_coherent()
	53	* @dev: Device to allocate coherent memory for
	54	* @size: Size of allocation
	55	* @dma_handle: Out argument for allocated DMA handle
	56	* @gfp: Allocation flags
	57	*
	58	* Managed dma_alloc_coherent(). Memory allocated using this function
	59	* will be automatically released on driver detach.
	60	*
	61	* RETURNS:
	62	* Pointer to allocated memory on success, NULL on failure.
	63	*/
6d42d79e	64	void dmam_alloc_coherent(struct device dev, size_t size,
9ac7849e TH	65	dma_addr_t *dma_handle, gfp_t gfp)
	66	{
	67	struct dma_devres *dr;
	68	void *vaddr;
	69
	70	dr = devres_alloc(dmam_coherent_release, sizeof(*dr), gfp);
	71	if (!dr)
	72	return NULL;
	73
	74	vaddr = dma_alloc_coherent(dev, size, dma_handle, gfp);
	75	if (!vaddr) {
	76	devres_free(dr);
	77	return NULL;
	78	}
	79
	80	dr->vaddr = vaddr;
	81	dr->dma_handle = *dma_handle;
	82	dr->size = size;
	83
	84	devres_add(dev, dr);
	85
	86	return vaddr;
	87	}
	88	EXPORT_SYMBOL(dmam_alloc_coherent);
	89
	90	/**
	91	* dmam_free_coherent - Managed dma_free_coherent()
	92	* @dev: Device to free coherent memory for
	93	* @size: Size of allocation
	94	* @vaddr: Virtual address of the memory to free
	95	* @dma_handle: DMA handle of the memory to free
	96	*
	97	* Managed dma_free_coherent().
	98	*/
	99	void dmam_free_coherent(struct device dev, size_t size, void vaddr,
	100	dma_addr_t dma_handle)
	101	{
	102	struct dma_devres match_data = { size, vaddr, dma_handle };
	103
	104	dma_free_coherent(dev, size, vaddr, dma_handle);
	105	WARN_ON(devres_destroy(dev, dmam_coherent_release, dmam_match,
	106	&match_data));
	107	}
	108	EXPORT_SYMBOL(dmam_free_coherent);
	109
	110	/**
cd74da95	111	* dmam_alloc_non_coherent - Managed dma_alloc_noncoherent()
9ac7849e TH	112	* @dev: Device to allocate non_coherent memory for
	113	* @size: Size of allocation
	114	* @dma_handle: Out argument for allocated DMA handle
	115	* @gfp: Allocation flags
	116	*
cd74da95	117	* Managed dma_alloc_noncoherent(). Memory allocated using this
9ac7849e TH	118	* function will be automatically released on driver detach.
	119	*
	120	* RETURNS:
	121	* Pointer to allocated memory on success, NULL on failure.
	122	*/
	123	void dmam_alloc_noncoherent(struct device dev, size_t size,
	124	dma_addr_t *dma_handle, gfp_t gfp)
	125	{
	126	struct dma_devres *dr;
	127	void *vaddr;
	128
	129	dr = devres_alloc(dmam_noncoherent_release, sizeof(*dr), gfp);
	130	if (!dr)
	131	return NULL;
	132
	133	vaddr = dma_alloc_noncoherent(dev, size, dma_handle, gfp);
	134	if (!vaddr) {
	135	devres_free(dr);
	136	return NULL;
	137	}
	138
	139	dr->vaddr = vaddr;
	140	dr->dma_handle = *dma_handle;
	141	dr->size = size;
	142
	143	devres_add(dev, dr);
	144
	145	return vaddr;
	146	}
	147	EXPORT_SYMBOL(dmam_alloc_noncoherent);
	148
	149	/**
	150	* dmam_free_coherent - Managed dma_free_noncoherent()
	151	* @dev: Device to free noncoherent memory for
	152	* @size: Size of allocation
	153	* @vaddr: Virtual address of the memory to free
	154	* @dma_handle: DMA handle of the memory to free
	155	*
	156	* Managed dma_free_noncoherent().
	157	*/
	158	void dmam_free_noncoherent(struct device dev, size_t size, void vaddr,
	159	dma_addr_t dma_handle)
	160	{
	161	struct dma_devres match_data = { size, vaddr, dma_handle };
	162
	163	dma_free_noncoherent(dev, size, vaddr, dma_handle);
	164	WARN_ON(!devres_destroy(dev, dmam_noncoherent_release, dmam_match,
	165	&match_data));
	166	}
	167	EXPORT_SYMBOL(dmam_free_noncoherent);
	168
20d666e4	169	#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
9ac7849e TH	170
	171	static void dmam_coherent_decl_release(struct device dev, void res)
	172	{
	173	dma_release_declared_memory(dev);
	174	}
	175
	176	/**
	177	* dmam_declare_coherent_memory - Managed dma_declare_coherent_memory()
	178	* @dev: Device to declare coherent memory for
88a984ba	179	* @phys_addr: Physical address of coherent memory to be declared
9ac7849e TH	180	* @device_addr: Device address of coherent memory to be declared
	181	* @size: Size of coherent memory to be declared
	182	* @flags: Flags
	183	*
	184	* Managed dma_declare_coherent_memory().
	185	*
	186	* RETURNS:
	187	* 0 on success, -errno on failure.
	188	*/
88a984ba	189	int dmam_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
9ac7849e TH	190	dma_addr_t device_addr, size_t size, int flags)
	191	{
	192	void *res;
	193	int rc;
	194
	195	res = devres_alloc(dmam_coherent_decl_release, 0, GFP_KERNEL);
	196	if (!res)
	197	return -ENOMEM;
	198
88a984ba	199	rc = dma_declare_coherent_memory(dev, phys_addr, device_addr, size,
9ac7849e	200	flags);
775115c0	201	if (rc) {
9ac7849e	202	devres_add(dev, res);
775115c0 VY	203	rc = 0;
775115c0 VY	204	} else {
9ac7849e	205	devres_free(res);
775115c0 VY	206	rc = -ENOMEM;
775115c0 VY	207	}
9ac7849e TH	208
	209	return rc;
	210	}
	211	EXPORT_SYMBOL(dmam_declare_coherent_memory);
	212
	213	/**
	214	* dmam_release_declared_memory - Managed dma_release_declared_memory().
	215	* @dev: Device to release declared coherent memory for
	216	*
	217	* Managed dmam_release_declared_memory().
	218	*/
	219	void dmam_release_declared_memory(struct device *dev)
	220	{
	221	WARN_ON(devres_destroy(dev, dmam_coherent_decl_release, NULL, NULL));
	222	}
	223	EXPORT_SYMBOL(dmam_release_declared_memory);
	224
c6c22955 MS	225	#endif
c6c22955 MS	226
d2b7428e MS	227	/*
	228	* Create scatter-list for the already allocated DMA buffer.
	229	*/
	230	int dma_common_get_sgtable(struct device dev, struct sg_table sgt,
	231	void *cpu_addr, dma_addr_t handle, size_t size)
	232	{
	233	struct page *page = virt_to_page(cpu_addr);
	234	int ret;
	235
	236	ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
	237	if (unlikely(ret))
	238	return ret;
	239
	240	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
	241	return 0;
	242	}
	243	EXPORT_SYMBOL(dma_common_get_sgtable);
	244
64ccc9c0 MS	245	/*
	246	* Create userspace mapping for the DMA-coherent memory.
	247	*/
	248	int dma_common_mmap(struct device dev, struct vm_area_struct vma,
	249	void *cpu_addr, dma_addr_t dma_addr, size_t size)
	250	{
	251	int ret = -ENXIO;
0d4a619b	252	#if defined(CONFIG_MMU) && !defined(CONFIG_ARCH_NO_COHERENT_DMA_MMAP)
95da00e3	253	unsigned long user_count = vma_pages(vma);
64ccc9c0 MS	254	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
	255	unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr));
	256	unsigned long off = vma->vm_pgoff;
	257
	258	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	259
	260	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
	261	return ret;
	262
	263	if (off < count && user_count <= (count - off)) {
	264	ret = remap_pfn_range(vma, vma->vm_start,
	265	pfn + off,
	266	user_count << PAGE_SHIFT,
	267	vma->vm_page_prot);
	268	}
0d4a619b	269	#endif /* CONFIG_MMU && !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */
64ccc9c0 MS	270
	271	return ret;
	272	}
	273	EXPORT_SYMBOL(dma_common_mmap);
513510dd LA	274
	275	#ifdef CONFIG_MMU
	276	/*
	277	* remaps an array of PAGE_SIZE pages into another vm_area
	278	* Cannot be used in non-sleeping contexts
	279	*/
	280	void dma_common_pages_remap(struct page *pages, size_t size,
	281	unsigned long vm_flags, pgprot_t prot,
	282	const void *caller)
	283	{
	284	struct vm_struct *area;
	285
	286	area = get_vm_area_caller(size, vm_flags, caller);
	287	if (!area)
	288	return NULL;
	289
	290	area->pages = pages;
	291
	292	if (map_vm_area(area, prot, pages)) {
	293	vunmap(area->addr);
	294	return NULL;
	295	}
	296
	297	return area->addr;
	298	}
	299
	300	/*
	301	* remaps an allocated contiguous region into another vm_area.
	302	* Cannot be used in non-sleeping contexts
	303	*/
	304
	305	void dma_common_contiguous_remap(struct page page, size_t size,
	306	unsigned long vm_flags,
	307	pgprot_t prot, const void *caller)
	308	{
	309	int i;
	310	struct page **pages;
	311	void *ptr;
	312	unsigned long pfn;
	313
	314	pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL);
	315	if (!pages)
	316	return NULL;
	317
	318	for (i = 0, pfn = page_to_pfn(page); i < (size >> PAGE_SHIFT); i++)
	319	pages[i] = pfn_to_page(pfn + i);
	320
	321	ptr = dma_common_pages_remap(pages, size, vm_flags, prot, caller);
	322
	323	kfree(pages);
	324
	325	return ptr;
	326	}
	327
	328	/*
	329	* unmaps a range previously mapped by dma_common_*_remap
	330	*/
	331	void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
	332	{
	333	struct vm_struct *area = find_vm_area(cpu_addr);
	334
	335	if (!area \|\| (area->flags & vm_flags) != vm_flags) {
	336	WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
	337	return;
338	}
339
85714108	340	unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
513510dd LA	341	vunmap(cpu_addr);
	342	}
	343	#endif