[linux.git] / arch / powerpc / mm / dma-noncoherent.c

/*
 *  PowerPC version derived from arch/arm/mm/consistent.c
 *    Copyright (C) 2001 Dan Malek ([email protected])
 *
 *  Copyright (C) 2000 Russell King
 *
 * Consistent memory allocators.  Used for DMA devices that want to
 * share uncached memory with the processor core.  The function return
 * is the virtual address and 'dma_handle' is the physical address.
 * Mostly stolen from the ARM port, with some changes for PowerPC.
 *						-- Dan
 *
 * Reorganized to get rid of the arch-specific consistent_* functions
 * and provide non-coherent implementations for the DMA API. -Matt
 *
 * Added in_interrupt() safe dma_alloc_coherent()/dma_free_coherent()
 * implementation. This is pulled straight from ARM and barely
 * modified. -Matt
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/highmem.h>
#include <linux/dma-direct.h>
#include <linux/dma-noncoherent.h>
#include <linux/export.h>

#include <asm/tlbflush.h>
#include <asm/dma.h>

#include <mm/mmu_decl.h>

/*
 * This address range defaults to a value that is safe for all
 * platforms which currently set CONFIG_NOT_COHERENT_CACHE. It
 * can be further configured for specific applications under
 * the "Advanced Setup" menu. -Matt
 */
#define CONSISTENT_BASE		(IOREMAP_TOP)
#define CONSISTENT_END 		(CONSISTENT_BASE + CONFIG_CONSISTENT_SIZE)
#define CONSISTENT_OFFSET(x)	(((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)

/*
 * This is the page table (2MB) covering uncached, DMA consistent allocations
 */
static DEFINE_SPINLOCK(consistent_lock);

/*
 * VM region handling support.
 *
 * This should become something generic, handling VM region allocations for
 * vmalloc and similar (ioremap, module space, etc).
 *
 * I envisage vmalloc()'s supporting vm_struct becoming:
 *
 *  struct vm_struct {
 *    struct vm_region	region;
 *    unsigned long	flags;
 *    struct page	**pages;
 *    unsigned int	nr_pages;
 *    unsigned long	phys_addr;
 *  };
 *
 * get_vm_area() would then call vm_region_alloc with an appropriate
 * struct vm_region head (eg):
 *
 *  struct vm_region vmalloc_head = {
 *	.vm_list	= LIST_HEAD_INIT(vmalloc_head.vm_list),
 *	.vm_start	= VMALLOC_START,
 *	.vm_end		= VMALLOC_END,
 *  };
 *
 * However, vmalloc_head.vm_start is variable (typically, it is dependent on
 * the amount of RAM found at boot time.)  I would imagine that get_vm_area()
 * would have to initialise this each time prior to calling vm_region_alloc().
 */
struct ppc_vm_region {
	struct list_head	vm_list;
	unsigned long		vm_start;
	unsigned long		vm_end;
};

static struct ppc_vm_region consistent_head = {
	.vm_list	= LIST_HEAD_INIT(consistent_head.vm_list),
	.vm_start	= CONSISTENT_BASE,
	.vm_end		= CONSISTENT_END,
};

static struct ppc_vm_region *
ppc_vm_region_alloc(struct ppc_vm_region *head, size_t size, gfp_t gfp)
{
	unsigned long addr = head->vm_start, end = head->vm_end - size;
	unsigned long flags;
	struct ppc_vm_region *c, *new;

	new = kmalloc(sizeof(struct ppc_vm_region), gfp);
	if (!new)
		goto out;

	spin_lock_irqsave(&consistent_lock, flags);

	list_for_each_entry(c, &head->vm_list, vm_list) {
		if ((addr + size) < addr)
			goto nospc;
		if ((addr + size) <= c->vm_start)
			goto found;
		addr = c->vm_end;
		if (addr > end)
			goto nospc;
	}

 found:
	/*
	 * Insert this entry _before_ the one we found.
	 */
	list_add_tail(&new->vm_list, &c->vm_list);
	new->vm_start = addr;
	new->vm_end = addr + size;

	spin_unlock_irqrestore(&consistent_lock, flags);
	return new;

 nospc:
	spin_unlock_irqrestore(&consistent_lock, flags);
	kfree(new);
 out:
	return NULL;
}

static struct ppc_vm_region *ppc_vm_region_find(struct ppc_vm_region *head, unsigned long addr)
{
	struct ppc_vm_region *c;

	list_for_each_entry(c, &head->vm_list, vm_list) {
		if (c->vm_start == addr)
			goto out;
	}
	c = NULL;
 out:
	return c;
}

/*
 * Allocate DMA-coherent memory space and return both the kernel remapped
 * virtual and bus address for that space.
 */
void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
		gfp_t gfp, unsigned long attrs)
{
	struct page *page;
	struct ppc_vm_region *c;
	unsigned long order;
	u64 mask = ISA_DMA_THRESHOLD, limit;

	if (dev) {
		mask = dev->coherent_dma_mask;

		/*
		 * Sanity check the DMA mask - it must be non-zero, and
		 * must be able to be satisfied by a DMA allocation.
		 */
		if (mask == 0) {
			dev_warn(dev, "coherent DMA mask is unset\n");
			goto no_page;
		}

		if ((~mask) & ISA_DMA_THRESHOLD) {
			dev_warn(dev, "coherent DMA mask %#llx is smaller "
				 "than system GFP_DMA mask %#llx\n",
				 mask, (unsigned long long)ISA_DMA_THRESHOLD);
			goto no_page;
		}
	}


	size = PAGE_ALIGN(size);
	limit = (mask + 1) & ~mask;
	if ((limit && size >= limit) ||
	    size >= (CONSISTENT_END - CONSISTENT_BASE)) {
		printk(KERN_WARNING "coherent allocation too big (requested %#x mask %#Lx)\n",
		       size, mask);
		return NULL;
	}

	order = get_order(size);

	/* Might be useful if we ever have a real legacy DMA zone... */
	if (mask != 0xffffffff)
		gfp |= GFP_DMA;

	page = alloc_pages(gfp, order);
	if (!page)
		goto no_page;

	/*
	 * Invalidate any data that might be lurking in the
	 * kernel direct-mapped region for device DMA.
	 */
	{
		unsigned long kaddr = (unsigned long)page_address(page);
		memset(page_address(page), 0, size);
		flush_dcache_range(kaddr, kaddr + size);
	}

	/*
	 * Allocate a virtual address in the consistent mapping region.
	 */
	c = ppc_vm_region_alloc(&consistent_head, size,
			    gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
	if (c) {
		unsigned long vaddr = c->vm_start;
		struct page *end = page + (1 << order);

		split_page(page, order);

		/*
		 * Set the "dma handle"
		 */
		*dma_handle = phys_to_dma(dev, page_to_phys(page));

		do {
			SetPageReserved(page);
			map_kernel_page(vaddr, page_to_phys(page),
					pgprot_noncached(PAGE_KERNEL));
			page++;
			vaddr += PAGE_SIZE;
		} while (size -= PAGE_SIZE);

		/*
		 * Free the otherwise unused pages.
		 */
		while (page < end) {
			__free_page(page);
			page++;
		}

		return (void *)c->vm_start;
	}

	if (page)
		__free_pages(page, order);
 no_page:
	return NULL;
}

/*
 * free a page as defined by the above mapping.
 */
void arch_dma_free(struct device *dev, size_t size, void *vaddr,
		dma_addr_t dma_handle, unsigned long attrs)
{
	struct ppc_vm_region *c;
	unsigned long flags, addr;
	
	size = PAGE_ALIGN(size);

	spin_lock_irqsave(&consistent_lock, flags);

	c = ppc_vm_region_find(&consistent_head, (unsigned long)vaddr);
	if (!c)
		goto no_area;

	if ((c->vm_end - c->vm_start) != size) {
		printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
		       __func__, c->vm_end - c->vm_start, size);
		dump_stack();
		size = c->vm_end - c->vm_start;
	}

	addr = c->vm_start;
	do {
		pte_t *ptep;
		unsigned long pfn;

		ptep = pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(addr),
							       addr),
						    addr),
					 addr);
		if (!pte_none(*ptep) && pte_present(*ptep)) {
			pfn = pte_pfn(*ptep);
			pte_clear(&init_mm, addr, ptep);
			if (pfn_valid(pfn)) {
				struct page *page = pfn_to_page(pfn);
				__free_reserved_page(page);
			}
		}
		addr += PAGE_SIZE;
	} while (size -= PAGE_SIZE);

	flush_tlb_kernel_range(c->vm_start, c->vm_end);

	list_del(&c->vm_list);

	spin_unlock_irqrestore(&consistent_lock, flags);

	kfree(c);
	return;

 no_area:
	spin_unlock_irqrestore(&consistent_lock, flags);
	printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n",
	       __func__, vaddr);
	dump_stack();
}

/*
 * make an area consistent.
 */
static void __dma_sync(void *vaddr, size_t size, int direction)
{
	unsigned long start = (unsigned long)vaddr;
	unsigned long end   = start + size;

	switch (direction) {
	case DMA_NONE:
		BUG();
	case DMA_FROM_DEVICE:
		/*
		 * invalidate only when cache-line aligned otherwise there is
		 * the potential for discarding uncommitted data from the cache
		 */
		if ((start | end) & (L1_CACHE_BYTES - 1))
			flush_dcache_range(start, end);
		else
			invalidate_dcache_range(start, end);
		break;
	case DMA_TO_DEVICE:		/* writeback only */
		clean_dcache_range(start, end);
		break;
	case DMA_BIDIRECTIONAL:	/* writeback and invalidate */
		flush_dcache_range(start, end);
		break;
	}
}

#ifdef CONFIG_HIGHMEM
/*
 * __dma_sync_page() implementation for systems using highmem.
 * In this case, each page of a buffer must be kmapped/kunmapped
 * in order to have a virtual address for __dma_sync(). This must
 * not sleep so kmap_atomic()/kunmap_atomic() are used.
 *
 * Note: yes, it is possible and correct to have a buffer extend
 * beyond the first page.
 */
static inline void __dma_sync_page_highmem(struct page *page,
		unsigned long offset, size_t size, int direction)
{
	size_t seg_size = min((size_t)(PAGE_SIZE - offset), size);
	size_t cur_size = seg_size;
	unsigned long flags, start, seg_offset = offset;
	int nr_segs = 1 + ((size - seg_size) + PAGE_SIZE - 1)/PAGE_SIZE;
	int seg_nr = 0;

	local_irq_save(flags);

	do {
		start = (unsigned long)kmap_atomic(page + seg_nr) + seg_offset;

		/* Sync this buffer segment */
		__dma_sync((void *)start, seg_size, direction);
		kunmap_atomic((void *)start);
		seg_nr++;

		/* Calculate next buffer segment size */
		seg_size = min((size_t)PAGE_SIZE, size - cur_size);

		/* Add the segment size to our running total */
		cur_size += seg_size;
		seg_offset = 0;
	} while (seg_nr < nr_segs);

	local_irq_restore(flags);
}
#endif /* CONFIG_HIGHMEM */

/*
 * __dma_sync_page makes memory consistent. identical to __dma_sync, but
 * takes a struct page instead of a virtual address
 */
static void __dma_sync_page(phys_addr_t paddr, size_t size, int dir)
{
	struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
	unsigned offset = paddr & ~PAGE_MASK;

#ifdef CONFIG_HIGHMEM
	__dma_sync_page_highmem(page, offset, size, dir);
#else
	unsigned long start = (unsigned long)page_address(page) + offset;
	__dma_sync((void *)start, size, dir);
#endif
}

void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
		size_t size, enum dma_data_direction dir)
{
	__dma_sync_page(paddr, size, dir);
}

void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
		size_t size, enum dma_data_direction dir)
{
	__dma_sync_page(paddr, size, dir);
}

/*
 * Return the PFN for a given cpu virtual address returned by arch_dma_alloc.
 */
long arch_dma_coherent_to_pfn(struct device *dev, void *vaddr,
		dma_addr_t dma_addr)
{
	/* This should always be populated, so we don't test every
	 * level. If that fails, we'll have a nice crash which
	 * will be as good as a BUG_ON()
	 */
	unsigned long cpu_addr = (unsigned long)vaddr;
	pgd_t *pgd = pgd_offset_k(cpu_addr);
	pud_t *pud = pud_offset(pgd, cpu_addr);
	pmd_t *pmd = pmd_offset(pud, cpu_addr);
	pte_t *ptep = pte_offset_kernel(pmd, cpu_addr);

	if (pte_none(*ptep) || !pte_present(*ptep))
		return 0;
	return pte_pfn(*ptep);
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* PowerPC version derived from arch/arm/mm/consistent.c
	3	* Copyright (C) 2001 Dan Malek ([email protected])
	4	*
	5	* Copyright (C) 2000 Russell King
	6	*
	7	* Consistent memory allocators. Used for DMA devices that want to
	8	* share uncached memory with the processor core. The function return
	9	* is the virtual address and 'dma_handle' is the physical address.
	10	* Mostly stolen from the ARM port, with some changes for PowerPC.
	11	* -- Dan
	12	*
	13	* Reorganized to get rid of the arch-specific consistent_* functions
	14	* and provide non-coherent implementations for the DMA API. -Matt
	15	*
	16	* Added in_interrupt() safe dma_alloc_coherent()/dma_free_coherent()
	17	* implementation. This is pulled straight from ARM and barely
	18	* modified. -Matt
	19	*
	20	* This program is free software; you can redistribute it and/or modify
	21	* it under the terms of the GNU General Public License version 2 as
	22	* published by the Free Software Foundation.
	23	*/
	24
1da177e4	25	#include <linux/sched.h>
5a0e3ad6	26	#include <linux/slab.h>
1da177e4 LT	27	#include <linux/kernel.h>
	28	#include <linux/errno.h>
	29	#include <linux/string.h>
	30	#include <linux/types.h>
1da177e4	31	#include <linux/highmem.h>
44a0337b	32	#include <linux/dma-direct.h>
6666cc17	33	#include <linux/dma-noncoherent.h>
93087948	34	#include <linux/export.h>
1da177e4 LT	35
1da177e4 LT	36	#include <asm/tlbflush.h>
308c09f1	37	#include <asm/dma.h>
1da177e4	38
9d9f2ccc	39	#include <mm/mmu_decl.h>
8b31e49d	40
84532a0f BH	41	/*
	42	* This address range defaults to a value that is safe for all
	43	* platforms which currently set CONFIG_NOT_COHERENT_CACHE. It
	44	* can be further configured for specific applications under
	45	* the "Advanced Setup" menu. -Matt
	46	*/
8b31e49d BH	47	#define CONSISTENT_BASE (IOREMAP_TOP)
8b31e49d BH	48	#define CONSISTENT_END (CONSISTENT_BASE + CONFIG_CONSISTENT_SIZE)
84532a0f BH	49	#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
	50
	51	/*
	52	* This is the page table (2MB) covering uncached, DMA consistent allocations
	53	*/
84532a0f BH	54	static DEFINE_SPINLOCK(consistent_lock);
	55
	56	/*
	57	* VM region handling support.
	58	*
	59	* This should become something generic, handling VM region allocations for
	60	* vmalloc and similar (ioremap, module space, etc).
	61	*
	62	* I envisage vmalloc()'s supporting vm_struct becoming:
	63	*
	64	* struct vm_struct {
	65	* struct vm_region region;
	66	* unsigned long flags;
	67	* struct page **pages;
	68	* unsigned int nr_pages;
	69	* unsigned long phys_addr;
	70	* };
	71	*
	72	* get_vm_area() would then call vm_region_alloc with an appropriate
	73	* struct vm_region head (eg):
	74	*
	75	* struct vm_region vmalloc_head = {
	76	* .vm_list = LIST_HEAD_INIT(vmalloc_head.vm_list),
	77	* .vm_start = VMALLOC_START,
	78	* .vm_end = VMALLOC_END,
	79	* };
	80	*
	81	* However, vmalloc_head.vm_start is variable (typically, it is dependent on
	82	* the amount of RAM found at boot time.) I would imagine that get_vm_area()
	83	* would have to initialise this each time prior to calling vm_region_alloc().
	84	*/
	85	struct ppc_vm_region {
	86	struct list_head vm_list;
	87	unsigned long vm_start;
	88	unsigned long vm_end;
	89	};
	90
	91	static struct ppc_vm_region consistent_head = {
	92	.vm_list = LIST_HEAD_INIT(consistent_head.vm_list),
	93	.vm_start = CONSISTENT_BASE,
	94	.vm_end = CONSISTENT_END,
	95	};
	96
	97	static struct ppc_vm_region *
	98	ppc_vm_region_alloc(struct ppc_vm_region *head, size_t size, gfp_t gfp)
	99	{
	100	unsigned long addr = head->vm_start, end = head->vm_end - size;
	101	unsigned long flags;
	102	struct ppc_vm_region c, new;
	103
	104	new = kmalloc(sizeof(struct ppc_vm_region), gfp);
	105	if (!new)
	106	goto out;
	107
	108	spin_lock_irqsave(&consistent_lock, flags);
	109
	110	list_for_each_entry(c, &head->vm_list, vm_list) {
	111	if ((addr + size) < addr)
	112	goto nospc;
	113	if ((addr + size) <= c->vm_start)
	114	goto found;
	115	addr = c->vm_end;
	116	if (addr > end)
	117	goto nospc;
118	}
119
120	found:
121	/*
122	* Insert this entry _before_ the one we found.
123	*/
124	list_add_tail(&new->vm_list, &c->vm_list);
125	new->vm_start = addr;
126	new->vm_end = addr + size;
127
128	spin_unlock_irqrestore(&consistent_lock, flags);
129	return new;
130
131	nospc:
132	spin_unlock_irqrestore(&consistent_lock, flags);
133	kfree(new);
134	out:
135	return NULL;
136	}
137
138	static struct ppc_vm_region ppc_vm_region_find(struct ppc_vm_region head, unsigned long addr)
139	{
140	struct ppc_vm_region *c;
141
142	list_for_each_entry(c, &head->vm_list, vm_list) {
143	if (c->vm_start == addr)
144	goto out;
145	}
146	c = NULL;
147	out:
148	return c;
149	}
150
1da177e4 LT	151	/*
	152	* Allocate DMA-coherent memory space and return both the kernel remapped
	153	* virtual and bus address for that space.
	154	*/
68005b67 CH	155	void arch_dma_alloc(struct device dev, size_t size, dma_addr_t *dma_handle,
68005b67 CH	156	gfp_t gfp, unsigned long attrs)
1da177e4 LT	157	{
1da177e4 LT	158	struct page *page;
84532a0f	159	struct ppc_vm_region *c;
1da177e4	160	unsigned long order;
8b31e49d	161	u64 mask = ISA_DMA_THRESHOLD, limit;
1da177e4	162
8b31e49d BH	163	if (dev) {
	164	mask = dev->coherent_dma_mask;
	165
	166	/*
	167	* Sanity check the DMA mask - it must be non-zero, and
	168	* must be able to be satisfied by a DMA allocation.
	169	*/
	170	if (mask == 0) {
	171	dev_warn(dev, "coherent DMA mask is unset\n");
	172	goto no_page;
	173	}
	174
	175	if ((~mask) & ISA_DMA_THRESHOLD) {
	176	dev_warn(dev, "coherent DMA mask %#llx is smaller "
	177	"than system GFP_DMA mask %#llx\n",
	178	mask, (unsigned long long)ISA_DMA_THRESHOLD);
	179	goto no_page;
	180	}
84532a0f BH	181	}
84532a0f BH	182
8b31e49d	183
1da177e4 LT	184	size = PAGE_ALIGN(size);
1da177e4 LT	185	limit = (mask + 1) & ~mask;
8b31e49d BH	186	if ((limit && size >= limit) \|\|
8b31e49d BH	187	size >= (CONSISTENT_END - CONSISTENT_BASE)) {
84532a0f BH	188	printk(KERN_WARNING "coherent allocation too big (requested %#x mask %#Lx)\n",
84532a0f BH	189	size, mask);
1da177e4 LT	190	return NULL;
	191	}
	192
	193	order = get_order(size);
	194
8b31e49d	195	/* Might be useful if we ever have a real legacy DMA zone... */
1da177e4 LT	196	if (mask != 0xffffffff)
	197	gfp \|= GFP_DMA;
	198
	199	page = alloc_pages(gfp, order);
	200	if (!page)
	201	goto no_page;
	202
	203	/*
	204	* Invalidate any data that might be lurking in the
	205	* kernel direct-mapped region for device DMA.
	206	*/
	207	{
	208	unsigned long kaddr = (unsigned long)page_address(page);
	209	memset(page_address(page), 0, size);
	210	flush_dcache_range(kaddr, kaddr + size);
	211	}
	212
	213	/*
84532a0f	214	* Allocate a virtual address in the consistent mapping region.
1da177e4	215	*/
84532a0f BH	216	c = ppc_vm_region_alloc(&consistent_head, size,
	217	gfp & ~(__GFP_DMA \| __GFP_HIGHMEM));
	218	if (c) {
	219	unsigned long vaddr = c->vm_start;
84532a0f	220	struct page *end = page + (1 << order);
1da177e4	221
84532a0f BH	222	split_page(page, order);
	223
	224	/*
	225	* Set the "dma handle"
	226	*/
44a0337b	227	*dma_handle = phys_to_dma(dev, page_to_phys(page));
1da177e4	228
84532a0f	229	do {
84532a0f	230	SetPageReserved(page);
4386c096	231	map_kernel_page(vaddr, page_to_phys(page),
c766ee72	232	pgprot_noncached(PAGE_KERNEL));
84532a0f	233	page++;
84532a0f BH	234	vaddr += PAGE_SIZE;
84532a0f BH	235	} while (size -= PAGE_SIZE);
1da177e4	236
84532a0f BH	237	/*
	238	* Free the otherwise unused pages.
	239	*/
	240	while (page < end) {
	241	__free_page(page);
	242	page++;
	243	}
	244
	245	return (void *)c->vm_start;
1da177e4 LT	246	}
	247
	248	if (page)
	249	__free_pages(page, order);
84532a0f	250	no_page:
1da177e4 LT	251	return NULL;
1da177e4 LT	252	}
1da177e4 LT	253
	254	/*
	255	* free a page as defined by the above mapping.
	256	*/
68005b67	257	void arch_dma_free(struct device dev, size_t size, void vaddr,
44a0337b	258	dma_addr_t dma_handle, unsigned long attrs)
1da177e4	259	{
84532a0f BH	260	struct ppc_vm_region *c;
84532a0f BH	261	unsigned long flags, addr;
8b31e49d	262
84532a0f BH	263	size = PAGE_ALIGN(size);
	264
	265	spin_lock_irqsave(&consistent_lock, flags);
	266
	267	c = ppc_vm_region_find(&consistent_head, (unsigned long)vaddr);
	268	if (!c)
	269	goto no_area;
	270
	271	if ((c->vm_end - c->vm_start) != size) {
	272	printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
	273	__func__, c->vm_end - c->vm_start, size);
	274	dump_stack();
	275	size = c->vm_end - c->vm_start;
	276	}
	277
84532a0f BH	278	addr = c->vm_start;
84532a0f BH	279	do {
8b31e49d	280	pte_t *ptep;
84532a0f BH	281	unsigned long pfn;
84532a0f BH	282
8b31e49d BH	283	ptep = pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(addr),
	284	addr),
	285	addr),
	286	addr);
	287	if (!pte_none(ptep) && pte_present(ptep)) {
	288	pfn = pte_pfn(*ptep);
	289	pte_clear(&init_mm, addr, ptep);
84532a0f BH	290	if (pfn_valid(pfn)) {
84532a0f BH	291	struct page *page = pfn_to_page(pfn);
c1ce4b37	292	__free_reserved_page(page);
84532a0f BH	293	}
84532a0f BH	294	}
8b31e49d	295	addr += PAGE_SIZE;
84532a0f BH	296	} while (size -= PAGE_SIZE);
	297
	298	flush_tlb_kernel_range(c->vm_start, c->vm_end);
	299
	300	list_del(&c->vm_list);
	301
	302	spin_unlock_irqrestore(&consistent_lock, flags);
	303
	304	kfree(c);
	305	return;
	306
	307	no_area:
	308	spin_unlock_irqrestore(&consistent_lock, flags);
	309	printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n",
	310	__func__, vaddr);
	311	dump_stack();
1da177e4	312	}
1da177e4	313
1da177e4 LT	314	/*
	315	* make an area consistent.
	316	*/
461db2bd	317	static void __dma_sync(void *vaddr, size_t size, int direction)
1da177e4 LT	318	{
	319	unsigned long start = (unsigned long)vaddr;
	320	unsigned long end = start + size;
	321
	322	switch (direction) {
	323	case DMA_NONE:
	324	BUG();
03d70617 AL	325	case DMA_FROM_DEVICE:
	326	/*
	327	* invalidate only when cache-line aligned otherwise there is
	328	* the potential for discarding uncommitted data from the cache
	329	*/
8478d7f0	330	if ((start \| end) & (L1_CACHE_BYTES - 1))
03d70617 AL	331	flush_dcache_range(start, end);
	332	else
	333	invalidate_dcache_range(start, end);
1da177e4 LT	334	break;
	335	case DMA_TO_DEVICE: /* writeback only */
	336	clean_dcache_range(start, end);
	337	break;
	338	case DMA_BIDIRECTIONAL: /* writeback and invalidate */
	339	flush_dcache_range(start, end);
	340	break;
	341	}
	342	}
1da177e4 LT	343
	344	#ifdef CONFIG_HIGHMEM
	345	/*
	346	* __dma_sync_page() implementation for systems using highmem.
	347	* In this case, each page of a buffer must be kmapped/kunmapped
	348	* in order to have a virtual address for __dma_sync(). This must
338cec32	349	* not sleep so kmap_atomic()/kunmap_atomic() are used.
1da177e4 LT	350	*
	351	* Note: yes, it is possible and correct to have a buffer extend
	352	* beyond the first page.
	353	*/
	354	static inline void __dma_sync_page_highmem(struct page *page,
	355	unsigned long offset, size_t size, int direction)
	356	{
a0c111c6	357	size_t seg_size = min((size_t)(PAGE_SIZE - offset), size);
1da177e4 LT	358	size_t cur_size = seg_size;
1da177e4 LT	359	unsigned long flags, start, seg_offset = offset;
a0c111c6	360	int nr_segs = 1 + ((size - seg_size) + PAGE_SIZE - 1)/PAGE_SIZE;
1da177e4 LT	361	int seg_nr = 0;
	362
	363	local_irq_save(flags);
	364
	365	do {
2480b208	366	start = (unsigned long)kmap_atomic(page + seg_nr) + seg_offset;
1da177e4 LT	367
	368	/* Sync this buffer segment */
	369	__dma_sync((void *)start, seg_size, direction);
2480b208	370	kunmap_atomic((void *)start);
1da177e4 LT	371	seg_nr++;
	372
	373	/* Calculate next buffer segment size */
	374	seg_size = min((size_t)PAGE_SIZE, size - cur_size);
	375
	376	/* Add the segment size to our running total */
	377	cur_size += seg_size;
	378	seg_offset = 0;
	379	} while (seg_nr < nr_segs);
	380
	381	local_irq_restore(flags);
	382	}
	383	#endif /* CONFIG_HIGHMEM */
	384
	385	/*
	386	* __dma_sync_page makes memory consistent. identical to __dma_sync, but
	387	* takes a struct page instead of a virtual address
	388	*/
461db2bd	389	static void __dma_sync_page(phys_addr_t paddr, size_t size, int dir)
1da177e4	390	{
461db2bd CH	391	struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
	392	unsigned offset = paddr & ~PAGE_MASK;
	393
1da177e4	394	#ifdef CONFIG_HIGHMEM
461db2bd	395	__dma_sync_page_highmem(page, offset, size, dir);
1da177e4 LT	396	#else
1da177e4 LT	397	unsigned long start = (unsigned long)page_address(page) + offset;
461db2bd	398	__dma_sync((void *)start, size, dir);
1da177e4 LT	399	#endif
1da177e4 LT	400	}
461db2bd CH	401
	402	void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
	403	size_t size, enum dma_data_direction dir)
	404	{
	405	__dma_sync_page(paddr, size, dir);
	406	}
	407
	408	void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
	409	size_t size, enum dma_data_direction dir)
	410	{
	411	__dma_sync_page(paddr, size, dir);
	412	}
6090912c BH	413
6090912c BH	414	/*
461db2bd	415	* Return the PFN for a given cpu virtual address returned by arch_dma_alloc.
6090912c	416	*/
6666cc17 CH	417	long arch_dma_coherent_to_pfn(struct device dev, void vaddr,
6666cc17 CH	418	dma_addr_t dma_addr)
6090912c BH	419	{
	420	/* This should always be populated, so we don't test every
	421	* level. If that fails, we'll have a nice crash which
	422	* will be as good as a BUG_ON()
	423	*/
6666cc17	424	unsigned long cpu_addr = (unsigned long)vaddr;
6090912c BH	425	pgd_t *pgd = pgd_offset_k(cpu_addr);
	426	pud_t *pud = pud_offset(pgd, cpu_addr);
	427	pmd_t *pmd = pmd_offset(pud, cpu_addr);
	428	pte_t *ptep = pte_offset_kernel(pmd, cpu_addr);
	429
	430	if (pte_none(ptep) \|\| !pte_present(ptep))
	431	return 0;
	432	return pte_pfn(*ptep);
	433	}