[linux.git] / arch / sh / mm / tlbflush_64.c

/*
 * arch/sh/mm/tlb-flush_64.c
 *
 * Copyright (C) 2000, 2001  Paolo Alberelli
 * Copyright (C) 2003  Richard Curnow (/proc/tlb, bug fixes)
 * Copyright (C) 2003 - 2009 Paul Mundt
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 */
#include <linux/signal.h>
#include <linux/rwsem.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/perf_event.h>
#include <linux/interrupt.h>
#include <asm/system.h>
#include <asm/io.h>
#include <asm/tlb.h>
#include <asm/uaccess.h>
#include <asm/pgalloc.h>
#include <asm/mmu_context.h>

extern void die(const char *,struct pt_regs *,long);

#define PFLAG(val,flag)   (( (val) & (flag) ) ? #flag : "" )
#define PPROT(flag) PFLAG(pgprot_val(prot),flag)

static inline void print_prots(pgprot_t prot)
{
	printk("prot is 0x%016llx\n",pgprot_val(prot));

	printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
	       PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
}

static inline void print_vma(struct vm_area_struct *vma)
{
	printk("vma start 0x%08lx\n", vma->vm_start);
	printk("vma end   0x%08lx\n", vma->vm_end);

	print_prots(vma->vm_page_prot);
	printk("vm_flags 0x%08lx\n", vma->vm_flags);
}

static inline void print_task(struct task_struct *tsk)
{
	printk("Task pid %d\n", task_pid_nr(tsk));
}

static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
{
	pgd_t *dir;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
	pte_t entry;

	dir = pgd_offset(mm, address);
	if (pgd_none(*dir))
		return NULL;

	pud = pud_offset(dir, address);
	if (pud_none(*pud))
		return NULL;

	pmd = pmd_offset(pud, address);
	if (pmd_none(*pmd))
		return NULL;

	pte = pte_offset_kernel(pmd, address);
	entry = *pte;
	if (pte_none(entry) || !pte_present(entry))
		return NULL;

	return pte;
}

/*
 * This routine handles page faults.  It determines the address,
 * and the problem, and then passes it off to one of the appropriate
 * routines.
 */
asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
			      unsigned long textaccess, unsigned long address)
{
	struct task_struct *tsk;
	struct mm_struct *mm;
	struct vm_area_struct * vma;
	const struct exception_table_entry *fixup;
	pte_t *pte;
	int fault;

	/* SIM
	 * Note this is now called with interrupts still disabled
	 * This is to cope with being called for a missing IO port
	 * address with interrupts disabled. This should be fixed as
	 * soon as we have a better 'fast path' miss handler.
	 *
	 * Plus take care how you try and debug this stuff.
	 * For example, writing debug data to a port which you
	 * have just faulted on is not going to work.
	 */

	tsk = current;
	mm = tsk->mm;

	/* Not an IO address, so reenable interrupts */
	local_irq_enable();

	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);

	/*
	 * If we're in an interrupt or have no user
	 * context, we must not take the fault..
	 */
	if (in_atomic() || !mm)
		goto no_context;

	/* TLB misses upon some cache flushes get done under cli() */
	down_read(&mm->mmap_sem);

	vma = find_vma(mm, address);

	if (!vma) {
#ifdef DEBUG_FAULT
		print_task(tsk);
		printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
		       __func__, __LINE__,
		       address,regs->pc,textaccess,writeaccess);
		show_regs(regs);
#endif
		goto bad_area;
	}
	if (vma->vm_start <= address) {
		goto good_area;
	}

	if (!(vma->vm_flags & VM_GROWSDOWN)) {
#ifdef DEBUG_FAULT
		print_task(tsk);
		printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
		       __func__, __LINE__,
		       address,regs->pc,textaccess,writeaccess);
		show_regs(regs);

		print_vma(vma);
#endif
		goto bad_area;
	}
	if (expand_stack(vma, address)) {
#ifdef DEBUG_FAULT
		print_task(tsk);
		printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
		       __func__, __LINE__,
		       address,regs->pc,textaccess,writeaccess);
		show_regs(regs);
#endif
		goto bad_area;
	}
/*
 * Ok, we have a good vm_area for this memory access, so
 * we can handle it..
 */
good_area:
	if (textaccess) {
		if (!(vma->vm_flags & VM_EXEC))
			goto bad_area;
	} else {
		if (writeaccess) {
			if (!(vma->vm_flags & VM_WRITE))
				goto bad_area;
		} else {
			if (!(vma->vm_flags & VM_READ))
				goto bad_area;
		}
	}

	/*
	 * If for any reason at all we couldn't handle the fault,
	 * make sure we exit gracefully rather than endlessly redo
	 * the fault.
	 */
	fault = handle_mm_fault(mm, vma, address, writeaccess ? FAULT_FLAG_WRITE : 0);
	if (unlikely(fault & VM_FAULT_ERROR)) {
		if (fault & VM_FAULT_OOM)
			goto out_of_memory;
		else if (fault & VM_FAULT_SIGBUS)
			goto do_sigbus;
		BUG();
	}

	if (fault & VM_FAULT_MAJOR) {
		tsk->maj_flt++;
		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
				     regs, address);
	} else {
		tsk->min_flt++;
		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
				     regs, address);
	}

	/* If we get here, the page fault has been handled.  Do the TLB refill
	   now from the newly-setup PTE, to avoid having to fault again right
	   away on the same instruction. */
	pte = lookup_pte (mm, address);
	if (!pte) {
		/* From empirical evidence, we can get here, due to
		   !pte_present(pte).  (e.g. if a swap-in occurs, and the page
		   is swapped back out again before the process that wanted it
		   gets rescheduled?) */
		goto no_pte;
	}

	__do_tlb_refill(address, textaccess, pte);

no_pte:

	up_read(&mm->mmap_sem);
	return;

/*
 * Something tried to access memory that isn't in our memory map..
 * Fix it, but check if it's kernel or user first..
 */
bad_area:
#ifdef DEBUG_FAULT
	printk("fault:bad area\n");
#endif
	up_read(&mm->mmap_sem);

	if (user_mode(regs)) {
		static int count=0;
		siginfo_t info;
		if (count < 4) {
			/* This is really to help debug faults when starting
			 * usermode, so only need a few */
			count++;
			printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
				address, task_pid_nr(current), current->comm,
				(unsigned long) regs->pc);
#if 0
			show_regs(regs);
#endif
		}
		if (is_global_init(tsk)) {
			panic("INIT had user mode bad_area\n");
		}
		tsk->thread.address = address;
		tsk->thread.error_code = writeaccess;
		info.si_signo = SIGSEGV;
		info.si_errno = 0;
		info.si_addr = (void *) address;
		force_sig_info(SIGSEGV, &info, tsk);
		return;
	}

no_context:
#ifdef DEBUG_FAULT
	printk("fault:No context\n");
#endif
	/* Are we prepared to handle this kernel fault?  */
	fixup = search_exception_tables(regs->pc);
	if (fixup) {
		regs->pc = fixup->fixup;
		return;
	}

/*
 * Oops. The kernel tried to access some bad page. We'll have to
 * terminate things with extreme prejudice.
 *
 */
	if (address < PAGE_SIZE)
		printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
	else
		printk(KERN_ALERT "Unable to handle kernel paging request");
	printk(" at virtual address %08lx\n", address);
	printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
	die("Oops", regs, writeaccess);
	do_exit(SIGKILL);

/*
 * We ran out of memory, or some other thing happened to us that made
 * us unable to handle the page fault gracefully.
 */
out_of_memory:
	up_read(&mm->mmap_sem);
	if (!user_mode(regs))
		goto no_context;
	pagefault_out_of_memory();
	return;

do_sigbus:
	printk("fault:Do sigbus\n");
	up_read(&mm->mmap_sem);

	/*
	 * Send a sigbus, regardless of whether we were in kernel
	 * or user mode.
	 */
	tsk->thread.address = address;
	tsk->thread.error_code = writeaccess;
	tsk->thread.trap_no = 14;
	force_sig(SIGBUS, tsk);

	/* Kernel mode? Handle exceptions or die */
	if (!user_mode(regs))
		goto no_context;
}

void local_flush_tlb_one(unsigned long asid, unsigned long page)
{
	unsigned long long match, pteh=0, lpage;
	unsigned long tlb;

	/*
	 * Sign-extend based on neff.
	 */
	lpage = neff_sign_extend(page);
	match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
	match |= lpage;

	for_each_itlb_entry(tlb) {
		asm volatile ("getcfg	%1, 0, %0"
			      : "=r" (pteh)
			      : "r" (tlb) );

		if (pteh == match) {
			__flush_tlb_slot(tlb);
			break;
		}
	}

	for_each_dtlb_entry(tlb) {
		asm volatile ("getcfg	%1, 0, %0"
			      : "=r" (pteh)
			      : "r" (tlb) );

		if (pteh == match) {
			__flush_tlb_slot(tlb);
			break;
		}

	}
}

void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
	unsigned long flags;

	if (vma->vm_mm) {
		page &= PAGE_MASK;
		local_irq_save(flags);
		local_flush_tlb_one(get_asid(), page);
		local_irq_restore(flags);
	}
}

void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
			   unsigned long end)
{
	unsigned long flags;
	unsigned long long match, pteh=0, pteh_epn, pteh_low;
	unsigned long tlb;
	unsigned int cpu = smp_processor_id();
	struct mm_struct *mm;

	mm = vma->vm_mm;
	if (cpu_context(cpu, mm) == NO_CONTEXT)
		return;

	local_irq_save(flags);

	start &= PAGE_MASK;
	end &= PAGE_MASK;

	match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID;

	/* Flush ITLB */
	for_each_itlb_entry(tlb) {
		asm volatile ("getcfg	%1, 0, %0"
			      : "=r" (pteh)
			      : "r" (tlb) );

		pteh_epn = pteh & PAGE_MASK;
		pteh_low = pteh & ~PAGE_MASK;

		if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
			__flush_tlb_slot(tlb);
	}

	/* Flush DTLB */
	for_each_dtlb_entry(tlb) {
		asm volatile ("getcfg	%1, 0, %0"
			      : "=r" (pteh)
			      : "r" (tlb) );

		pteh_epn = pteh & PAGE_MASK;
		pteh_low = pteh & ~PAGE_MASK;

		if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
			__flush_tlb_slot(tlb);
	}

	local_irq_restore(flags);
}

void local_flush_tlb_mm(struct mm_struct *mm)
{
	unsigned long flags;
	unsigned int cpu = smp_processor_id();

	if (cpu_context(cpu, mm) == NO_CONTEXT)
		return;

	local_irq_save(flags);

	cpu_context(cpu, mm) = NO_CONTEXT;
	if (mm == current->mm)
		activate_context(mm, cpu);

	local_irq_restore(flags);
}

void local_flush_tlb_all(void)
{
	/* Invalidate all, including shared pages, excluding fixed TLBs */
	unsigned long flags, tlb;

	local_irq_save(flags);

	/* Flush each ITLB entry */
	for_each_itlb_entry(tlb)
		__flush_tlb_slot(tlb);

	/* Flush each DTLB entry */
	for_each_dtlb_entry(tlb)
		__flush_tlb_slot(tlb);

	local_irq_restore(flags);
}

void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
        /* FIXME: Optimize this later.. */
        flush_tlb_all();
}

void __flush_tlb_global(void)
{
	flush_tlb_all();
}

void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
{
}
Commit	Line	Data
1da177e4	1	/*
3eeffb32	2	* arch/sh/mm/tlb-flush_64.c
1da177e4 LT	3	*
	4	* Copyright (C) 2000, 2001 Paolo Alberelli
	5	* Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes)
163b2f0b	6	* Copyright (C) 2003 - 2009 Paul Mundt
1da177e4	7	*
3eeffb32 PM	8	* This file is subject to the terms and conditions of the GNU General Public
	9	* License. See the file "COPYING" in the main directory of this archive
	10	* for more details.
1da177e4	11	*/
1da177e4 LT	12	#include <linux/signal.h>
	13	#include <linux/rwsem.h>
	14	#include <linux/sched.h>
	15	#include <linux/kernel.h>
	16	#include <linux/errno.h>
	17	#include <linux/string.h>
	18	#include <linux/types.h>
	19	#include <linux/ptrace.h>
	20	#include <linux/mman.h>
	21	#include <linux/mm.h>
	22	#include <linux/smp.h>
cdd6c482	23	#include <linux/perf_event.h>
1da177e4	24	#include <linux/interrupt.h>
1da177e4 LT	25	#include <asm/system.h>
	26	#include <asm/io.h>
	27	#include <asm/tlb.h>
	28	#include <asm/uaccess.h>
	29	#include <asm/pgalloc.h>
	30	#include <asm/mmu_context.h>
1da177e4 LT	31
	32	extern void die(const char ,struct pt_regs ,long);
	33
	34	#define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" )
	35	#define PPROT(flag) PFLAG(pgprot_val(prot),flag)
	36
	37	static inline void print_prots(pgprot_t prot)
	38	{
24ef7fc4	39	printk("prot is 0x%016llx\n",pgprot_val(prot));
1da177e4 LT	40
	41	printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
	42	PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
	43	}
	44
	45	static inline void print_vma(struct vm_area_struct *vma)
	46	{
	47	printk("vma start 0x%08lx\n", vma->vm_start);
	48	printk("vma end 0x%08lx\n", vma->vm_end);
	49
	50	print_prots(vma->vm_page_prot);
	51	printk("vm_flags 0x%08lx\n", vma->vm_flags);
	52	}
	53
	54	static inline void print_task(struct task_struct *tsk)
	55	{
19c5870c	56	printk("Task pid %d\n", task_pid_nr(tsk));
1da177e4 LT	57	}
	58
	59	static pte_t lookup_pte(struct mm_struct mm, unsigned long address)
	60	{
	61	pgd_t *dir;
3eeffb32	62	pud_t *pud;
1da177e4 LT	63	pmd_t *pmd;
	64	pte_t *pte;
	65	pte_t entry;
	66
	67	dir = pgd_offset(mm, address);
3eeffb32	68	if (pgd_none(*dir))
1da177e4	69	return NULL;
1da177e4	70
3eeffb32 PM	71	pud = pud_offset(dir, address);
	72	if (pud_none(*pud))
	73	return NULL;
	74
	75	pmd = pmd_offset(pud, address);
	76	if (pmd_none(*pmd))
1da177e4	77	return NULL;
1da177e4 LT	78
	79	pte = pte_offset_kernel(pmd, address);
	80	entry = *pte;
3eeffb32	81	if (pte_none(entry) \|\| !pte_present(entry))
1da177e4	82	return NULL;
1da177e4 LT	83
	84	return pte;
	85	}
	86
	87	/*
	88	* This routine handles page faults. It determines the address,
	89	* and the problem, and then passes it off to one of the appropriate
	90	* routines.
	91	*/
	92	asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
	93	unsigned long textaccess, unsigned long address)
	94	{
	95	struct task_struct *tsk;
	96	struct mm_struct *mm;
	97	struct vm_area_struct * vma;
	98	const struct exception_table_entry *fixup;
	99	pte_t *pte;
83c54070	100	int fault;
1da177e4	101
1da177e4 LT	102	/* SIM
	103	* Note this is now called with interrupts still disabled
	104	* This is to cope with being called for a missing IO port
0a354775	105	* address with interrupts disabled. This should be fixed as
1da177e4 LT	106	* soon as we have a better 'fast path' miss handler.
	107	*
	108	* Plus take care how you try and debug this stuff.
	109	* For example, writing debug data to a port which you
	110	* have just faulted on is not going to work.
	111	*/
	112
	113	tsk = current;
	114	mm = tsk->mm;
	115
	116	/* Not an IO address, so reenable interrupts */
	117	local_irq_enable();
	118
a8b0ca17	119	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
163b2f0b	120
1da177e4 LT	121	/*
	122	* If we're in an interrupt or have no user
	123	* context, we must not take the fault..
	124	*/
6edaf68a	125	if (in_atomic() \|\| !mm)
1da177e4 LT	126	goto no_context;
	127
	128	/* TLB misses upon some cache flushes get done under cli() */
	129	down_read(&mm->mmap_sem);
	130
	131	vma = find_vma(mm, address);
	132
	133	if (!vma) {
	134	#ifdef DEBUG_FAULT
	135	print_task(tsk);
	136	printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
866e6b9e	137	__func__, __LINE__,
1da177e4 LT	138	address,regs->pc,textaccess,writeaccess);
	139	show_regs(regs);
	140	#endif
	141	goto bad_area;
	142	}
	143	if (vma->vm_start <= address) {
	144	goto good_area;
	145	}
	146
	147	if (!(vma->vm_flags & VM_GROWSDOWN)) {
	148	#ifdef DEBUG_FAULT
	149	print_task(tsk);
	150	printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
866e6b9e	151	__func__, __LINE__,
1da177e4 LT	152	address,regs->pc,textaccess,writeaccess);
	153	show_regs(regs);
	154
	155	print_vma(vma);
	156	#endif
	157	goto bad_area;
	158	}
	159	if (expand_stack(vma, address)) {
	160	#ifdef DEBUG_FAULT
	161	print_task(tsk);
	162	printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
866e6b9e	163	__func__, __LINE__,
1da177e4 LT	164	address,regs->pc,textaccess,writeaccess);
	165	show_regs(regs);
	166	#endif
	167	goto bad_area;
	168	}
	169	/*
	170	* Ok, we have a good vm_area for this memory access, so
	171	* we can handle it..
	172	*/
	173	good_area:
	174	if (textaccess) {
	175	if (!(vma->vm_flags & VM_EXEC))
	176	goto bad_area;
	177	} else {
	178	if (writeaccess) {
	179	if (!(vma->vm_flags & VM_WRITE))
	180	goto bad_area;
	181	} else {
	182	if (!(vma->vm_flags & VM_READ))
	183	goto bad_area;
	184	}
	185	}
	186
	187	/*
	188	* If for any reason at all we couldn't handle the fault,
	189	* make sure we exit gracefully rather than endlessly redo
	190	* the fault.
	191	*/
d06063cc	192	fault = handle_mm_fault(mm, vma, address, writeaccess ? FAULT_FLAG_WRITE : 0);
83c54070 NP	193	if (unlikely(fault & VM_FAULT_ERROR)) {
	194	if (fault & VM_FAULT_OOM)
	195	goto out_of_memory;
	196	else if (fault & VM_FAULT_SIGBUS)
	197	goto do_sigbus;
	198	BUG();
1da177e4	199	}
163b2f0b PM	200
163b2f0b PM	201	if (fault & VM_FAULT_MAJOR) {
83c54070	202	tsk->maj_flt++;
a8b0ca17	203	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
163b2f0b PM	204	regs, address);
163b2f0b PM	205	} else {
83c54070	206	tsk->min_flt++;
a8b0ca17	207	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
163b2f0b PM	208	regs, address);
163b2f0b PM	209	}
83c54070	210
1da177e4 LT	211	/* If we get here, the page fault has been handled. Do the TLB refill
	212	now from the newly-setup PTE, to avoid having to fault again right
	213	away on the same instruction. */
	214	pte = lookup_pte (mm, address);
	215	if (!pte) {
	216	/* From empirical evidence, we can get here, due to
	217	!pte_present(pte). (e.g. if a swap-in occurs, and the page
	218	is swapped back out again before the process that wanted it
	219	gets rescheduled?) */
	220	goto no_pte;
	221	}
	222
	223	__do_tlb_refill(address, textaccess, pte);
	224
	225	no_pte:
	226
	227	up_read(&mm->mmap_sem);
	228	return;
	229
	230	/*
	231	* Something tried to access memory that isn't in our memory map..
	232	* Fix it, but check if it's kernel or user first..
	233	*/
	234	bad_area:
	235	#ifdef DEBUG_FAULT
	236	printk("fault:bad area\n");
	237	#endif
	238	up_read(&mm->mmap_sem);
	239
	240	if (user_mode(regs)) {
	241	static int count=0;
	242	siginfo_t info;
	243	if (count < 4) {
	244	/* This is really to help debug faults when starting
	245	* usermode, so only need a few */
	246	count++;
	247	printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
19c5870c	248	address, task_pid_nr(current), current->comm,
1da177e4 LT	249	(unsigned long) regs->pc);
	250	#if 0
	251	show_regs(regs);
	252	#endif
	253	}
b460cbc5	254	if (is_global_init(tsk)) {
1da177e4 LT	255	panic("INIT had user mode bad_area\n");
	256	}
	257	tsk->thread.address = address;
	258	tsk->thread.error_code = writeaccess;
	259	info.si_signo = SIGSEGV;
	260	info.si_errno = 0;
	261	info.si_addr = (void *) address;
	262	force_sig_info(SIGSEGV, &info, tsk);
	263	return;
	264	}
	265
	266	no_context:
	267	#ifdef DEBUG_FAULT
	268	printk("fault:No context\n");
	269	#endif
	270	/* Are we prepared to handle this kernel fault? */
	271	fixup = search_exception_tables(regs->pc);
	272	if (fixup) {
	273	regs->pc = fixup->fixup;
	274	return;
	275	}
	276
	277	/*
	278	* Oops. The kernel tried to access some bad page. We'll have to
	279	* terminate things with extreme prejudice.
	280	*
	281	*/
	282	if (address < PAGE_SIZE)
	283	printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
	284	else
	285	printk(KERN_ALERT "Unable to handle kernel paging request");
	286	printk(" at virtual address %08lx\n", address);
	287	printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
	288	die("Oops", regs, writeaccess);
	289	do_exit(SIGKILL);
	290
	291	/*
	292	* We ran out of memory, or some other thing happened to us that made
	293	* us unable to handle the page fault gracefully.
	294	*/
	295	out_of_memory:
1da177e4	296	up_read(&mm->mmap_sem);
6b6b18e6 NP	297	if (!user_mode(regs))
	298	goto no_context;
	299	pagefault_out_of_memory();
	300	return;
1da177e4 LT	301
	302	do_sigbus:
	303	printk("fault:Do sigbus\n");
	304	up_read(&mm->mmap_sem);
	305
	306	/*
	307	* Send a sigbus, regardless of whether we were in kernel
	308	* or user mode.
	309	*/
	310	tsk->thread.address = address;
	311	tsk->thread.error_code = writeaccess;
	312	tsk->thread.trap_no = 14;
	313	force_sig(SIGBUS, tsk);
	314
	315	/* Kernel mode? Handle exceptions or die */
	316	if (!user_mode(regs))
	317	goto no_context;
	318	}
	319
3eeffb32	320	void local_flush_tlb_one(unsigned long asid, unsigned long page)
1da177e4 LT	321	{
	322	unsigned long long match, pteh=0, lpage;
	323	unsigned long tlb;
1da177e4 LT	324
	325	/*
	326	* Sign-extend based on neff.
	327	*/
c7914834	328	lpage = neff_sign_extend(page);
3eeffb32	329	match = (asid << PTEH_ASID_SHIFT) \| PTEH_VALID;
1da177e4 LT	330	match \|= lpage;
1da177e4 LT	331
3eeffb32 PM	332	for_each_itlb_entry(tlb) {
	333	asm volatile ("getcfg %1, 0, %0"
	334	: "=r" (pteh)
	335	: "r" (tlb) );
1da177e4	336
3eeffb32 PM	337	if (pteh == match) {
	338	__flush_tlb_slot(tlb);
	339	break;
1da177e4 LT	340	}
	341	}
	342
1da177e4 LT	343	for_each_dtlb_entry(tlb) {
	344	asm volatile ("getcfg %1, 0, %0"
	345	: "=r" (pteh)
	346	: "r" (tlb) );
	347
	348	if (pteh == match) {
	349	__flush_tlb_slot(tlb);
	350	break;
	351	}
	352
	353	}
	354	}
	355
3eeffb32	356	void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1da177e4 LT	357	{
	358	unsigned long flags;
	359
1da177e4 LT	360	if (vma->vm_mm) {
	361	page &= PAGE_MASK;
	362	local_irq_save(flags);
3eeffb32	363	local_flush_tlb_one(get_asid(), page);
1da177e4 LT	364	local_irq_restore(flags);
	365	}
	366	}
	367
3eeffb32 PM	368	void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
3eeffb32 PM	369	unsigned long end)
1da177e4 LT	370	{
	371	unsigned long flags;
	372	unsigned long long match, pteh=0, pteh_epn, pteh_low;
	373	unsigned long tlb;
3eeffb32	374	unsigned int cpu = smp_processor_id();
1da177e4 LT	375	struct mm_struct *mm;
	376
	377	mm = vma->vm_mm;
3eeffb32	378	if (cpu_context(cpu, mm) == NO_CONTEXT)
1da177e4 LT	379	return;
	380
	381	local_irq_save(flags);
	382
	383	start &= PAGE_MASK;
	384	end &= PAGE_MASK;
	385
3eeffb32	386	match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) \| PTEH_VALID;
1da177e4 LT	387
	388	/* Flush ITLB */
	389	for_each_itlb_entry(tlb) {
	390	asm volatile ("getcfg %1, 0, %0"
	391	: "=r" (pteh)
	392	: "r" (tlb) );
	393
	394	pteh_epn = pteh & PAGE_MASK;
	395	pteh_low = pteh & ~PAGE_MASK;
	396
	397	if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
	398	__flush_tlb_slot(tlb);
	399	}
	400
	401	/* Flush DTLB */
	402	for_each_dtlb_entry(tlb) {
	403	asm volatile ("getcfg %1, 0, %0"
	404	: "=r" (pteh)
	405	: "r" (tlb) );
	406
	407	pteh_epn = pteh & PAGE_MASK;
	408	pteh_low = pteh & ~PAGE_MASK;
	409
	410	if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
	411	__flush_tlb_slot(tlb);
	412	}
	413
	414	local_irq_restore(flags);
	415	}
	416
3eeffb32	417	void local_flush_tlb_mm(struct mm_struct *mm)
1da177e4 LT	418	{
1da177e4 LT	419	unsigned long flags;
3eeffb32	420	unsigned int cpu = smp_processor_id();
1da177e4	421
3eeffb32	422	if (cpu_context(cpu, mm) == NO_CONTEXT)
1da177e4 LT	423	return;
	424
	425	local_irq_save(flags);
	426
3eeffb32 PM	427	cpu_context(cpu, mm) = NO_CONTEXT;
	428	if (mm == current->mm)
	429	activate_context(mm, cpu);
1da177e4 LT	430
1da177e4 LT	431	local_irq_restore(flags);
1da177e4 LT	432	}
1da177e4 LT	433
3eeffb32	434	void local_flush_tlb_all(void)
1da177e4 LT	435	{
1da177e4 LT	436	/* Invalidate all, including shared pages, excluding fixed TLBs */
1da177e4 LT	437	unsigned long flags, tlb;
1da177e4 LT	438
1da177e4 LT	439	local_irq_save(flags);
	440
	441	/* Flush each ITLB entry */
3eeffb32	442	for_each_itlb_entry(tlb)
1da177e4	443	__flush_tlb_slot(tlb);
1da177e4 LT	444
1da177e4 LT	445	/* Flush each DTLB entry */
3eeffb32	446	for_each_dtlb_entry(tlb)
1da177e4	447	__flush_tlb_slot(tlb);
1da177e4 LT	448
	449	local_irq_restore(flags);
	450	}
	451
3eeffb32	452	void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
1da177e4 LT	453	{
	454	/* FIXME: Optimize this later.. */
	455	flush_tlb_all();
	456	}
9cef7492	457
59615ecd PM	458	void __flush_tlb_global(void)
	459	{
	460	flush_tlb_all();
	461	}
	462
9cef7492 PM	463	void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
	464	{
	465	}