[linux.git] / arch / sparc / mm / iommu.c

/*
 * iommu.c:  IOMMU specific routines for memory management.
 *
 * Copyright (C) 1995 David S. Miller  ([email protected])
 * Copyright (C) 1995,2002 Pete Zaitcev     ([email protected])
 * Copyright (C) 1996 Eddie C. Dost    ([email protected])
 * Copyright (C) 1997,1998 Jakub Jelinek    ([email protected])
 */
 
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/highmem.h>	/* pte_offset_map => kmap_atomic */

#include <asm/scatterlist.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/sbus.h>
#include <asm/io.h>
#include <asm/mxcc.h>
#include <asm/mbus.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#include <asm/bitext.h>
#include <asm/iommu.h>
#include <asm/dma.h>

/*
 * This can be sized dynamically, but we will do this
 * only when we have a guidance about actual I/O pressures.
 */
#define IOMMU_RNGE	IOMMU_RNGE_256MB
#define IOMMU_START	0xF0000000
#define IOMMU_WINSIZE	(256*1024*1024U)
#define IOMMU_NPTES	(IOMMU_WINSIZE/PAGE_SIZE)	/* 64K PTEs, 265KB */
#define IOMMU_ORDER	6				/* 4096 * (1<<6) */

/* srmmu.c */
extern int viking_mxcc_present;
BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long)
#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page)
extern int flush_page_for_dma_global;
static int viking_flush;
/* viking.S */
extern void viking_flush_page(unsigned long page);
extern void viking_mxcc_flush_page(unsigned long page);

/*
 * Values precomputed according to CPU type.
 */
static unsigned int ioperm_noc;		/* Consistent mapping iopte flags */
static pgprot_t dvma_prot;		/* Consistent mapping pte flags */

#define IOPERM        (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)

void __init
iommu_init(int iommund, struct sbus_bus *sbus)
{
	unsigned int impl, vers;
	unsigned long tmp;
	struct iommu_struct *iommu;
	struct linux_prom_registers iommu_promregs[PROMREG_MAX];
	struct resource r;
	unsigned long *bitmap;

	iommu = kmalloc(sizeof(struct iommu_struct), GFP_ATOMIC);
	if (!iommu) {
		prom_printf("Unable to allocate iommu structure\n");
		prom_halt();
	}
	iommu->regs = NULL;
	if (prom_getproperty(iommund, "reg", (void *) iommu_promregs,
			 sizeof(iommu_promregs)) != -1) {
		memset(&r, 0, sizeof(r));
		r.flags = iommu_promregs[0].which_io;
		r.start = iommu_promregs[0].phys_addr;
		iommu->regs = (struct iommu_regs *)
			sbus_ioremap(&r, 0, PAGE_SIZE * 3, "iommu_regs");
	}
	if (!iommu->regs) {
		prom_printf("Cannot map IOMMU registers\n");
		prom_halt();
	}
	impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28;
	vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24;
	tmp = iommu->regs->control;
	tmp &= ~(IOMMU_CTRL_RNGE);
	tmp |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
	iommu->regs->control = tmp;
	iommu_invalidate(iommu->regs);
	iommu->start = IOMMU_START;
	iommu->end = 0xffffffff;

	/* Allocate IOMMU page table */
	/* Stupid alignment constraints give me a headache. 
	   We need 256K or 512K or 1M or 2M area aligned to
           its size and current gfp will fortunately give
           it to us. */
        tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
	if (!tmp) {
		prom_printf("Unable to allocate iommu table [0x%08x]\n",
			    IOMMU_NPTES*sizeof(iopte_t));
		prom_halt();
	}
	iommu->page_table = (iopte_t *)tmp;

	/* Initialize new table. */
	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
	flush_cache_all();
	flush_tlb_all();
	iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4;
	iommu_invalidate(iommu->regs);

	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
	if (!bitmap) {
		prom_printf("Unable to allocate iommu bitmap [%d]\n",
			    (int)(IOMMU_NPTES>>3));
		prom_halt();
	}
	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
	/* To be coherent on HyperSparc, the page color of DVMA
	 * and physical addresses must match.
	 */
	if (srmmu_modtype == HyperSparc)
		iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
	else
		iommu->usemap.num_colors = 1;

	printk("IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
	    impl, vers, iommu->page_table,
	    (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);

	sbus->ofdev.dev.archdata.iommu = iommu;
}

/* This begs to be btfixup-ed by srmmu. */
/* Flush the iotlb entries to ram. */
/* This could be better if we didn't have to flush whole pages. */
static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
{
	unsigned long start;
	unsigned long end;

	start = (unsigned long)iopte;
	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
	start &= PAGE_MASK;
	if (viking_mxcc_present) {
		while(start < end) {
			viking_mxcc_flush_page(start);
			start += PAGE_SIZE;
		}
	} else if (viking_flush) {
		while(start < end) {
			viking_flush_page(start);
			start += PAGE_SIZE;
		}
	} else {
		while(start < end) {
			__flush_page_to_ram(start);
			start += PAGE_SIZE;
		}
	}
}

static u32 iommu_get_one(struct page *page, int npages, struct sbus_bus *sbus)
{
	struct iommu_struct *iommu = sbus->ofdev.dev.archdata.iommu;
	int ioptex;
	iopte_t *iopte, *iopte0;
	unsigned int busa, busa0;
	int i;

	/* page color = pfn of page */
	ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page));
	if (ioptex < 0)
		panic("iommu out");
	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
	iopte0 = &iommu->page_table[ioptex];

	busa = busa0;
	iopte = iopte0;
	for (i = 0; i < npages; i++) {
		iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM);
		iommu_invalidate_page(iommu->regs, busa);
		busa += PAGE_SIZE;
		iopte++;
		page++;
	}

	iommu_flush_iotlb(iopte0, npages);

	return busa0;
}

static u32 iommu_get_scsi_one(char *vaddr, unsigned int len,
    struct sbus_bus *sbus)
{
	unsigned long off;
	int npages;
	struct page *page;
	u32 busa;

	off = (unsigned long)vaddr & ~PAGE_MASK;
	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
	page = virt_to_page((unsigned long)vaddr & PAGE_MASK);
	busa = iommu_get_one(page, npages, sbus);
	return busa + off;
}

static __u32 iommu_get_scsi_one_noflush(char *vaddr, unsigned long len, struct sbus_bus *sbus)
{
	return iommu_get_scsi_one(vaddr, len, sbus);
}

static __u32 iommu_get_scsi_one_gflush(char *vaddr, unsigned long len, struct sbus_bus *sbus)
{
	flush_page_for_dma(0);
	return iommu_get_scsi_one(vaddr, len, sbus);
}

static __u32 iommu_get_scsi_one_pflush(char *vaddr, unsigned long len, struct sbus_bus *sbus)
{
	unsigned long page = ((unsigned long) vaddr) & PAGE_MASK;

	while(page < ((unsigned long)(vaddr + len))) {
		flush_page_for_dma(page);
		page += PAGE_SIZE;
	}
	return iommu_get_scsi_one(vaddr, len, sbus);
}

static void iommu_get_scsi_sgl_noflush(struct scatterlist *sg, int sz, struct sbus_bus *sbus)
{
	int n;

	while (sz != 0) {
		--sz;
		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
		sg->dvma_address = iommu_get_one(sg->page, n, sbus) + sg->offset;
		sg->dvma_length = (__u32) sg->length;
		sg++;
	}
}

static void iommu_get_scsi_sgl_gflush(struct scatterlist *sg, int sz, struct sbus_bus *sbus)
{
	int n;

	flush_page_for_dma(0);
	while (sz != 0) {
		--sz;
		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
		sg->dvma_address = iommu_get_one(sg->page, n, sbus) + sg->offset;
		sg->dvma_length = (__u32) sg->length;
		sg++;
	}
}

static void iommu_get_scsi_sgl_pflush(struct scatterlist *sg, int sz, struct sbus_bus *sbus)
{
	unsigned long page, oldpage = 0;
	int n, i;

	while(sz != 0) {
		--sz;

		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;

		/*
		 * We expect unmapped highmem pages to be not in the cache.
		 * XXX Is this a good assumption?
		 * XXX What if someone else unmaps it here and races us?
		 */
		if ((page = (unsigned long) page_address(sg->page)) != 0) {
			for (i = 0; i < n; i++) {
				if (page != oldpage) {	/* Already flushed? */
					flush_page_for_dma(page);
					oldpage = page;
				}
				page += PAGE_SIZE;
			}
		}

		sg->dvma_address = iommu_get_one(sg->page, n, sbus) + sg->offset;
		sg->dvma_length = (__u32) sg->length;
		sg++;
	}
}

static void iommu_release_one(u32 busa, int npages, struct sbus_bus *sbus)
{
	struct iommu_struct *iommu = sbus->ofdev.dev.archdata.iommu;
	int ioptex;
	int i;

	BUG_ON(busa < iommu->start);
	ioptex = (busa - iommu->start) >> PAGE_SHIFT;
	for (i = 0; i < npages; i++) {
		iopte_val(iommu->page_table[ioptex + i]) = 0;
		iommu_invalidate_page(iommu->regs, busa);
		busa += PAGE_SIZE;
	}
	bit_map_clear(&iommu->usemap, ioptex, npages);
}

static void iommu_release_scsi_one(__u32 vaddr, unsigned long len, struct sbus_bus *sbus)
{
	unsigned long off;
	int npages;

	off = vaddr & ~PAGE_MASK;
	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
	iommu_release_one(vaddr & PAGE_MASK, npages, sbus);
}

static void iommu_release_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_bus *sbus)
{
	int n;

	while(sz != 0) {
		--sz;

		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
		iommu_release_one(sg->dvma_address & PAGE_MASK, n, sbus);
		sg->dvma_address = 0x21212121;
		sg++;
	}
}

#ifdef CONFIG_SBUS
static int iommu_map_dma_area(dma_addr_t *pba, unsigned long va,
    unsigned long addr, int len)
{
	unsigned long page, end;
	struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
	iopte_t *iopte = iommu->page_table;
	iopte_t *first;
	int ioptex;

	BUG_ON((va & ~PAGE_MASK) != 0);
	BUG_ON((addr & ~PAGE_MASK) != 0);
	BUG_ON((len & ~PAGE_MASK) != 0);

	/* page color = physical address */
	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
		addr >> PAGE_SHIFT);
	if (ioptex < 0)
		panic("iommu out");

	iopte += ioptex;
	first = iopte;
	end = addr + len;
	while(addr < end) {
		page = va;
		{
			pgd_t *pgdp;
			pmd_t *pmdp;
			pte_t *ptep;

			if (viking_mxcc_present)
				viking_mxcc_flush_page(page);
			else if (viking_flush)
				viking_flush_page(page);
			else
				__flush_page_to_ram(page);

			pgdp = pgd_offset(&init_mm, addr);
			pmdp = pmd_offset(pgdp, addr);
			ptep = pte_offset_map(pmdp, addr);

			set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
		}
		iopte_val(*iopte++) =
		    MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
		addr += PAGE_SIZE;
		va += PAGE_SIZE;
	}
	/* P3: why do we need this?
	 *
	 * DAVEM: Because there are several aspects, none of which
	 *        are handled by a single interface.  Some cpus are
	 *        completely not I/O DMA coherent, and some have
	 *        virtually indexed caches.  The driver DMA flushing
	 *        methods handle the former case, but here during
	 *        IOMMU page table modifications, and usage of non-cacheable
	 *        cpu mappings of pages potentially in the cpu caches, we have
	 *        to handle the latter case as well.
	 */
	flush_cache_all();
	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
	flush_tlb_all();
	iommu_invalidate(iommu->regs);

	*pba = iommu->start + (ioptex << PAGE_SHIFT);
	return 0;
}

static void iommu_unmap_dma_area(unsigned long busa, int len)
{
	struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
	iopte_t *iopte = iommu->page_table;
	unsigned long end;
	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;

	BUG_ON((busa & ~PAGE_MASK) != 0);
	BUG_ON((len & ~PAGE_MASK) != 0);

	iopte += ioptex;
	end = busa + len;
	while (busa < end) {
		iopte_val(*iopte++) = 0;
		busa += PAGE_SIZE;
	}
	flush_tlb_all();
	iommu_invalidate(iommu->regs);
	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
}

static struct page *iommu_translate_dvma(unsigned long busa)
{
	struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
	iopte_t *iopte = iommu->page_table;

	iopte += ((busa - iommu->start) >> PAGE_SHIFT);
	return pfn_to_page((iopte_val(*iopte) & IOPTE_PAGE) >> (PAGE_SHIFT-4));
}
#endif

static char *iommu_lockarea(char *vaddr, unsigned long len)
{
	return vaddr;
}

static void iommu_unlockarea(char *vaddr, unsigned long len)
{
}

void __init ld_mmu_iommu(void)
{
	viking_flush = (BTFIXUPVAL_CALL(flush_page_for_dma) == (unsigned long)viking_flush_page);
	BTFIXUPSET_CALL(mmu_lockarea, iommu_lockarea, BTFIXUPCALL_RETO0);
	BTFIXUPSET_CALL(mmu_unlockarea, iommu_unlockarea, BTFIXUPCALL_NOP);

	if (!BTFIXUPVAL_CALL(flush_page_for_dma)) {
		/* IO coherent chip */
		BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_noflush, BTFIXUPCALL_RETO0);
		BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_noflush, BTFIXUPCALL_NORM);
	} else if (flush_page_for_dma_global) {
		/* flush_page_for_dma flushes everything, no matter of what page is it */
		BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_gflush, BTFIXUPCALL_NORM);
		BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_gflush, BTFIXUPCALL_NORM);
	} else {
		BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_pflush, BTFIXUPCALL_NORM);
		BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_pflush, BTFIXUPCALL_NORM);
	}
	BTFIXUPSET_CALL(mmu_release_scsi_one, iommu_release_scsi_one, BTFIXUPCALL_NORM);
	BTFIXUPSET_CALL(mmu_release_scsi_sgl, iommu_release_scsi_sgl, BTFIXUPCALL_NORM);

#ifdef CONFIG_SBUS
	BTFIXUPSET_CALL(mmu_map_dma_area, iommu_map_dma_area, BTFIXUPCALL_NORM);
	BTFIXUPSET_CALL(mmu_unmap_dma_area, iommu_unmap_dma_area, BTFIXUPCALL_NORM);
	BTFIXUPSET_CALL(mmu_translate_dvma, iommu_translate_dvma, BTFIXUPCALL_NORM);
#endif

	if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
		dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
		ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
	} else {
		dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
		ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
	}
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* iommu.c: IOMMU specific routines for memory management.
	3	*
	4	* Copyright (C) 1995 David S. Miller ([email protected])
	5	* Copyright (C) 1995,2002 Pete Zaitcev ([email protected])
	6	* Copyright (C) 1996 Eddie C. Dost ([email protected])
	7	* Copyright (C) 1997,1998 Jakub Jelinek ([email protected])
	8	*/
	9
1da177e4 LT	10	#include <linux/kernel.h>
	11	#include <linux/init.h>
	12	#include <linux/mm.h>
	13	#include <linux/slab.h>
	14	#include <linux/highmem.h> /* pte_offset_map => kmap_atomic */
	15
	16	#include <asm/scatterlist.h>
	17	#include <asm/pgalloc.h>
	18	#include <asm/pgtable.h>
	19	#include <asm/sbus.h>
	20	#include <asm/io.h>
	21	#include <asm/mxcc.h>
	22	#include <asm/mbus.h>
	23	#include <asm/cacheflush.h>
	24	#include <asm/tlbflush.h>
	25	#include <asm/bitext.h>
	26	#include <asm/iommu.h>
	27	#include <asm/dma.h>
	28
	29	/*
	30	* This can be sized dynamically, but we will do this
	31	* only when we have a guidance about actual I/O pressures.
	32	*/
	33	#define IOMMU_RNGE IOMMU_RNGE_256MB
	34	#define IOMMU_START 0xF0000000
	35	#define IOMMU_WINSIZE (25610241024U)
	36	#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 265KB */
	37	#define IOMMU_ORDER 6 /* 4096 * (1<<6) */
	38
	39	/* srmmu.c */
	40	extern int viking_mxcc_present;
	41	BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long)
	42	#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page)
	43	extern int flush_page_for_dma_global;
	44	static int viking_flush;
	45	/* viking.S */
	46	extern void viking_flush_page(unsigned long page);
	47	extern void viking_mxcc_flush_page(unsigned long page);
	48
	49	/*
	50	* Values precomputed according to CPU type.
	51	*/
	52	static unsigned int ioperm_noc; /* Consistent mapping iopte flags */
	53	static pgprot_t dvma_prot; /* Consistent mapping pte flags */
	54
	55	#define IOPERM (IOPTE_CACHE \| IOPTE_WRITE \| IOPTE_VALID)
	56	#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) \| (perm)) & ~IOPTE_WAZ)
	57
	58	void __init
	59	iommu_init(int iommund, struct sbus_bus *sbus)
	60	{
	61	unsigned int impl, vers;
	62	unsigned long tmp;
	63	struct iommu_struct *iommu;
	64	struct linux_prom_registers iommu_promregs[PROMREG_MAX];
	65	struct resource r;
	66	unsigned long *bitmap;
	67
	68	iommu = kmalloc(sizeof(struct iommu_struct), GFP_ATOMIC);
	69	if (!iommu) {
	70	prom_printf("Unable to allocate iommu structure\n");
	71	prom_halt();
	72	}
	73	iommu->regs = NULL;
74	if (prom_getproperty(iommund, "reg", (void *) iommu_promregs,
75	sizeof(iommu_promregs)) != -1) {
76	memset(&r, 0, sizeof(r));
77	r.flags = iommu_promregs[0].which_io;
78	r.start = iommu_promregs[0].phys_addr;
79	iommu->regs = (struct iommu_regs *)
80	sbus_ioremap(&r, 0, PAGE_SIZE * 3, "iommu_regs");
81	}
82	if (!iommu->regs) {
83	prom_printf("Cannot map IOMMU registers\n");
84	prom_halt();
85	}
86	impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28;
87	vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24;
88	tmp = iommu->regs->control;
89	tmp &= ~(IOMMU_CTRL_RNGE);
90	tmp \|= (IOMMU_RNGE_256MB \| IOMMU_CTRL_ENAB);
91	iommu->regs->control = tmp;
92	iommu_invalidate(iommu->regs);
93	iommu->start = IOMMU_START;
94	iommu->end = 0xffffffff;
95
96	/* Allocate IOMMU page table */
97	/* Stupid alignment constraints give me a headache.
98	We need 256K or 512K or 1M or 2M area aligned to
99	its size and current gfp will fortunately give
100	it to us. */
101	tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
102	if (!tmp) {
103	prom_printf("Unable to allocate iommu table [0x%08x]\n",
104	IOMMU_NPTES*sizeof(iopte_t));
105	prom_halt();
106	}
107	iommu->page_table = (iopte_t *)tmp;
108
109	/* Initialize new table. */
110	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
111	flush_cache_all();
112	flush_tlb_all();
113	iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4;
114	iommu_invalidate(iommu->regs);
115
116	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
117	if (!bitmap) {
118	prom_printf("Unable to allocate iommu bitmap [%d]\n",
119	(int)(IOMMU_NPTES>>3));
120	prom_halt();
121	}
122	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
123	/* To be coherent on HyperSparc, the page color of DVMA
124	* and physical addresses must match.
125	*/
126	if (srmmu_modtype == HyperSparc)
127	iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
128	else
129	iommu->usemap.num_colors = 1;
130
131	printk("IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
132	impl, vers, iommu->page_table,
133	(int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
134
3ac4c949	135	sbus->ofdev.dev.archdata.iommu = iommu;
1da177e4 LT	136	}
	137
	138	/* This begs to be btfixup-ed by srmmu. */
	139	/* Flush the iotlb entries to ram. */
	140	/* This could be better if we didn't have to flush whole pages. */
	141	static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
	142	{
	143	unsigned long start;
	144	unsigned long end;
	145
3185d4d2	146	start = (unsigned long)iopte;
1da177e4	147	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
3185d4d2	148	start &= PAGE_MASK;
1da177e4 LT	149	if (viking_mxcc_present) {
	150	while(start < end) {
	151	viking_mxcc_flush_page(start);
	152	start += PAGE_SIZE;
	153	}
	154	} else if (viking_flush) {
	155	while(start < end) {
	156	viking_flush_page(start);
	157	start += PAGE_SIZE;
	158	}
	159	} else {
	160	while(start < end) {
	161	__flush_page_to_ram(start);
	162	start += PAGE_SIZE;
	163	}
	164	}
	165	}
	166
	167	static u32 iommu_get_one(struct page page, int npages, struct sbus_bus sbus)
	168	{
3ac4c949	169	struct iommu_struct *iommu = sbus->ofdev.dev.archdata.iommu;
1da177e4 LT	170	int ioptex;
	171	iopte_t iopte, iopte0;
	172	unsigned int busa, busa0;
	173	int i;
	174
	175	/* page color = pfn of page */
	176	ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page));
	177	if (ioptex < 0)
	178	panic("iommu out");
	179	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
	180	iopte0 = &iommu->page_table[ioptex];
	181
	182	busa = busa0;
	183	iopte = iopte0;
	184	for (i = 0; i < npages; i++) {
	185	iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM);
	186	iommu_invalidate_page(iommu->regs, busa);
	187	busa += PAGE_SIZE;
	188	iopte++;
	189	page++;
	190	}
	191
	192	iommu_flush_iotlb(iopte0, npages);
	193
	194	return busa0;
	195	}
	196
	197	static u32 iommu_get_scsi_one(char *vaddr, unsigned int len,
	198	struct sbus_bus *sbus)
	199	{
	200	unsigned long off;
	201	int npages;
	202	struct page *page;
	203	u32 busa;
	204
	205	off = (unsigned long)vaddr & ~PAGE_MASK;
	206	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
	207	page = virt_to_page((unsigned long)vaddr & PAGE_MASK);
	208	busa = iommu_get_one(page, npages, sbus);
	209	return busa + off;
	210	}
	211
	212	static __u32 iommu_get_scsi_one_noflush(char vaddr, unsigned long len, struct sbus_bus sbus)
	213	{
	214	return iommu_get_scsi_one(vaddr, len, sbus);
	215	}
	216
	217	static __u32 iommu_get_scsi_one_gflush(char vaddr, unsigned long len, struct sbus_bus sbus)
	218	{
	219	flush_page_for_dma(0);
	220	return iommu_get_scsi_one(vaddr, len, sbus);
	221	}
	222
	223	static __u32 iommu_get_scsi_one_pflush(char vaddr, unsigned long len, struct sbus_bus sbus)
	224	{
	225	unsigned long page = ((unsigned long) vaddr) & PAGE_MASK;
	226
	227	while(page < ((unsigned long)(vaddr + len))) {
	228	flush_page_for_dma(page);
	229	page += PAGE_SIZE;
	230	}
	231	return iommu_get_scsi_one(vaddr, len, sbus);
	232	}
	233
234	static void iommu_get_scsi_sgl_noflush(struct scatterlist sg, int sz, struct sbus_bus sbus)
235	{
236	int n;
237
238	while (sz != 0) {
239	--sz;
240	n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
241	sg->dvma_address = iommu_get_one(sg->page, n, sbus) + sg->offset;
242	sg->dvma_length = (__u32) sg->length;
243	sg++;
244	}
245	}
246
247	static void iommu_get_scsi_sgl_gflush(struct scatterlist sg, int sz, struct sbus_bus sbus)
248	{
249	int n;
250
251	flush_page_for_dma(0);
252	while (sz != 0) {
253	--sz;
254	n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
255	sg->dvma_address = iommu_get_one(sg->page, n, sbus) + sg->offset;
256	sg->dvma_length = (__u32) sg->length;
257	sg++;
258	}
259	}
260
261	static void iommu_get_scsi_sgl_pflush(struct scatterlist sg, int sz, struct sbus_bus sbus)
262	{
263	unsigned long page, oldpage = 0;
264	int n, i;
265
266	while(sz != 0) {
267	--sz;
268
269	n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
270
271	/*
272	* We expect unmapped highmem pages to be not in the cache.
273	* XXX Is this a good assumption?
274	* XXX What if someone else unmaps it here and races us?
275	*/
276	if ((page = (unsigned long) page_address(sg->page)) != 0) {
277	for (i = 0; i < n; i++) {
278	if (page != oldpage) { /* Already flushed? */
279	flush_page_for_dma(page);
280	oldpage = page;
281	}
282	page += PAGE_SIZE;
283	}
284	}
285
286	sg->dvma_address = iommu_get_one(sg->page, n, sbus) + sg->offset;
287	sg->dvma_length = (__u32) sg->length;
288	sg++;
289	}
290	}
291
292	static void iommu_release_one(u32 busa, int npages, struct sbus_bus *sbus)
293	{
3ac4c949	294	struct iommu_struct *iommu = sbus->ofdev.dev.archdata.iommu;
1da177e4 LT	295	int ioptex;
	296	int i;
	297
1ae61388	298	BUG_ON(busa < iommu->start);
1da177e4 LT	299	ioptex = (busa - iommu->start) >> PAGE_SHIFT;
	300	for (i = 0; i < npages; i++) {
	301	iopte_val(iommu->page_table[ioptex + i]) = 0;
	302	iommu_invalidate_page(iommu->regs, busa);
	303	busa += PAGE_SIZE;
	304	}
	305	bit_map_clear(&iommu->usemap, ioptex, npages);
	306	}
	307
	308	static void iommu_release_scsi_one(__u32 vaddr, unsigned long len, struct sbus_bus *sbus)
	309	{
	310	unsigned long off;
	311	int npages;
	312
	313	off = vaddr & ~PAGE_MASK;
	314	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
	315	iommu_release_one(vaddr & PAGE_MASK, npages, sbus);
	316	}
	317
	318	static void iommu_release_scsi_sgl(struct scatterlist sg, int sz, struct sbus_bus sbus)
	319	{
	320	int n;
	321
	322	while(sz != 0) {
	323	--sz;
	324
	325	n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
	326	iommu_release_one(sg->dvma_address & PAGE_MASK, n, sbus);
	327	sg->dvma_address = 0x21212121;
	328	sg++;
	329	}
	330	}
	331
	332	#ifdef CONFIG_SBUS
	333	static int iommu_map_dma_area(dma_addr_t *pba, unsigned long va,
	334	unsigned long addr, int len)
	335	{
	336	unsigned long page, end;
3ac4c949	337	struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
1da177e4 LT	338	iopte_t *iopte = iommu->page_table;
	339	iopte_t *first;
	340	int ioptex;
	341
1ae61388 ES	342	BUG_ON((va & ~PAGE_MASK) != 0);
	343	BUG_ON((addr & ~PAGE_MASK) != 0);
	344	BUG_ON((len & ~PAGE_MASK) != 0);
1da177e4 LT	345
	346	/* page color = physical address */
	347	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
	348	addr >> PAGE_SHIFT);
	349	if (ioptex < 0)
	350	panic("iommu out");
	351
	352	iopte += ioptex;
	353	first = iopte;
	354	end = addr + len;
	355	while(addr < end) {
	356	page = va;
	357	{
	358	pgd_t *pgdp;
	359	pmd_t *pmdp;
	360	pte_t *ptep;
	361
	362	if (viking_mxcc_present)
	363	viking_mxcc_flush_page(page);
	364	else if (viking_flush)
	365	viking_flush_page(page);
	366	else
	367	__flush_page_to_ram(page);
	368
	369	pgdp = pgd_offset(&init_mm, addr);
	370	pmdp = pmd_offset(pgdp, addr);
	371	ptep = pte_offset_map(pmdp, addr);
	372
	373	set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
	374	}
	375	iopte_val(*iopte++) =
	376	MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
	377	addr += PAGE_SIZE;
	378	va += PAGE_SIZE;
	379	}
	380	/* P3: why do we need this?
	381	*
	382	* DAVEM: Because there are several aspects, none of which
	383	* are handled by a single interface. Some cpus are
	384	* completely not I/O DMA coherent, and some have
	385	* virtually indexed caches. The driver DMA flushing
	386	* methods handle the former case, but here during
	387	* IOMMU page table modifications, and usage of non-cacheable
	388	* cpu mappings of pages potentially in the cpu caches, we have
	389	* to handle the latter case as well.
	390	*/
	391	flush_cache_all();
	392	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
	393	flush_tlb_all();
	394	iommu_invalidate(iommu->regs);
	395
	396	*pba = iommu->start + (ioptex << PAGE_SHIFT);
	397	return 0;
	398	}
	399
	400	static void iommu_unmap_dma_area(unsigned long busa, int len)
	401	{
3ac4c949	402	struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
1da177e4 LT	403	iopte_t *iopte = iommu->page_table;
	404	unsigned long end;
	405	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
	406
1ae61388 ES	407	BUG_ON((busa & ~PAGE_MASK) != 0);
1ae61388 ES	408	BUG_ON((len & ~PAGE_MASK) != 0);
1da177e4 LT	409
	410	iopte += ioptex;
	411	end = busa + len;
	412	while (busa < end) {
	413	iopte_val(*iopte++) = 0;
	414	busa += PAGE_SIZE;
	415	}
	416	flush_tlb_all();
	417	iommu_invalidate(iommu->regs);
	418	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
	419	}
	420
	421	static struct page *iommu_translate_dvma(unsigned long busa)
	422	{
3ac4c949	423	struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
1da177e4 LT	424	iopte_t *iopte = iommu->page_table;
	425
	426	iopte += ((busa - iommu->start) >> PAGE_SHIFT);
	427	return pfn_to_page((iopte_val(*iopte) & IOPTE_PAGE) >> (PAGE_SHIFT-4));
	428	}
	429	#endif
	430
	431	static char iommu_lockarea(char vaddr, unsigned long len)
	432	{
	433	return vaddr;
	434	}
	435
	436	static void iommu_unlockarea(char *vaddr, unsigned long len)
	437	{
	438	}
	439
	440	void __init ld_mmu_iommu(void)
	441	{
	442	viking_flush = (BTFIXUPVAL_CALL(flush_page_for_dma) == (unsigned long)viking_flush_page);
	443	BTFIXUPSET_CALL(mmu_lockarea, iommu_lockarea, BTFIXUPCALL_RETO0);
	444	BTFIXUPSET_CALL(mmu_unlockarea, iommu_unlockarea, BTFIXUPCALL_NOP);
	445
	446	if (!BTFIXUPVAL_CALL(flush_page_for_dma)) {
	447	/* IO coherent chip */
	448	BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_noflush, BTFIXUPCALL_RETO0);
	449	BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_noflush, BTFIXUPCALL_NORM);
	450	} else if (flush_page_for_dma_global) {
	451	/* flush_page_for_dma flushes everything, no matter of what page is it */
	452	BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_gflush, BTFIXUPCALL_NORM);
	453	BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_gflush, BTFIXUPCALL_NORM);
	454	} else {
	455	BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_pflush, BTFIXUPCALL_NORM);
	456	BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_pflush, BTFIXUPCALL_NORM);
	457	}
	458	BTFIXUPSET_CALL(mmu_release_scsi_one, iommu_release_scsi_one, BTFIXUPCALL_NORM);
	459	BTFIXUPSET_CALL(mmu_release_scsi_sgl, iommu_release_scsi_sgl, BTFIXUPCALL_NORM);
	460
	461	#ifdef CONFIG_SBUS
	462	BTFIXUPSET_CALL(mmu_map_dma_area, iommu_map_dma_area, BTFIXUPCALL_NORM);
	463	BTFIXUPSET_CALL(mmu_unmap_dma_area, iommu_unmap_dma_area, BTFIXUPCALL_NORM);
	464	BTFIXUPSET_CALL(mmu_translate_dvma, iommu_translate_dvma, BTFIXUPCALL_NORM);
	465	#endif
	466
	467	if (viking_mxcc_present \|\| srmmu_modtype == HyperSparc) {
	468	dvma_prot = __pgprot(SRMMU_CACHE \| SRMMU_ET_PTE \| SRMMU_PRIV);
	469	ioperm_noc = IOPTE_CACHE \| IOPTE_WRITE \| IOPTE_VALID;
	470	} else {
	471	dvma_prot = __pgprot(SRMMU_ET_PTE \| SRMMU_PRIV);
	472	ioperm_noc = IOPTE_WRITE \| IOPTE_VALID;
	473	}
	474	}