powerpc/mm/radix: Invalidate ERAT on tlbiel for POWER9 DD1

[linux.git] / arch / powerpc / mm / tlb-radix.c

diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 48df05ef523100e9aa67eb29fadf8b22cd116daf..3493cf4e045258df20f5cf47990e991b6af265b7 100644 (file)
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -50,6 +50,8 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
        for (set = 0; set < POWER9_TLB_SETS_RADIX ; set++) {
                __tlbiel_pid(pid, set, ric);
        }
+       if (cpu_has_feature(CPU_FTR_POWER9_DD1))
+               asm volatile(PPC_INVALIDATE_ERAT : : :"memory");
        return;
 }
 
@@ -83,6 +85,8 @@ static inline void _tlbiel_va(unsigned long va, unsigned long pid,
        asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
                     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
        asm volatile("ptesync": : :"memory");
+       if (cpu_has_feature(CPU_FTR_POWER9_DD1))
+               asm volatile(PPC_INVALIDATE_ERAT : : :"memory");
 }
 
 static inline void _tlbie_va(unsigned long va, unsigned long pid,
@@ -175,7 +179,7 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
        if (unlikely(pid == MMU_NO_CONTEXT))
                goto no_context;
 
-       if (!mm_is_core_local(mm)) {
+       if (!mm_is_thread_local(mm)) {
                int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
 
                if (lock_tlbie)
@@ -201,7 +205,7 @@ void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
        if (unlikely(pid == MMU_NO_CONTEXT))
                goto no_context;
 
-       if (!mm_is_core_local(mm)) {
+       if (!mm_is_thread_local(mm)) {
                int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
 
                if (lock_tlbie)
@@ -226,7 +230,7 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
        pid = mm ? mm->context.id : 0;
        if (unlikely(pid == MMU_NO_CONTEXT))
                goto bail;
-       if (!mm_is_core_local(mm)) {
+       if (!mm_is_thread_local(mm)) {
                int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
 
                if (lock_tlbie)
@@ -321,7 +325,7 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
 {
        unsigned long pid;
        unsigned long addr;
-       int local = mm_is_core_local(mm);
+       int local = mm_is_thread_local(mm);
        unsigned long ap = mmu_get_ap(psize);
        int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
        unsigned long page_size = 1UL << mmu_psize_defs[psize].shift;
@@ -400,3 +404,27 @@ void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
        radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
 }
 EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
+
+void radix__flush_tlb_all(void)
+{
+       unsigned long rb,prs,r,rs;
+       unsigned long ric = RIC_FLUSH_ALL;
+
+       rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */
+       prs = 0; /* partition scoped */
+       r = 1;   /* raidx format */
+       rs = 1 & ((1UL << 32) - 1); /* any LPID value to flush guest mappings */
+
+       asm volatile("ptesync": : :"memory");
+       /*
+        * now flush guest entries by passing PRS = 1 and LPID != 0
+        */
+       asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
+                    : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory");
+       /*
+        * now flush host entires by passing PRS = 0 and LPID == 0
+        */
+       asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
+                    : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory");
+       asm volatile("eieio; tlbsync; ptesync": : :"memory");
+}