arch/powerpc/mm/nohash/tlb.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * This file contains the routines for TLB flushing.
   4  * On machines where the MMU does not use a hash table to store virtual to
   5  * physical translations (ie, SW loaded TLBs or Book3E compilant processors,
   6  * this does -not- include 603 however which shares the implementation with
   7  * hash based processors)
   8  *
   9  *  -- BenH
  10  *
  11  * Copyright 2008,2009 Ben Herrenschmidt <[email protected]>
  12  *                     IBM Corp.
  13  *
  14  *  Derived from arch/ppc/mm/init.c:
  15  *    Copyright (C) 1995-1996 Gary Thomas ([email protected])
  16  *
  17  *  Modifications by Paul Mackerras (PowerMac) ([email protected])
  18  *  and Cort Dougan (PReP) ([email protected])
  19  *    Copyright (C) 1996 Paul Mackerras
  20  *
  21  *  Derived from "arch/i386/mm/init.c"
  22  *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
  23  */
  24
  25 #include <linux/kernel.h>
  26 #include <linux/export.h>
  27 #include <linux/mm.h>
  28 #include <linux/init.h>
  29 #include <linux/highmem.h>
  30 #include <linux/pagemap.h>
  31 #include <linux/preempt.h>
  32 #include <linux/spinlock.h>
  33 #include <linux/memblock.h>
  34 #include <linux/of_fdt.h>
  35 #include <linux/hugetlb.h>
  36
  37 #include <asm/pgalloc.h>
  38 #include <asm/tlbflush.h>
  39 #include <asm/tlb.h>
  40 #include <asm/text-patching.h>
  41 #include <asm/cputhreads.h>
  42 #include <asm/hugetlb.h>
  43 #include <asm/paca.h>
  44
  45 #include <mm/mmu_decl.h>
  46
  47 /*
  48  * This struct lists the sw-supported page sizes.  The hardawre MMU may support
  49  * other sizes not listed here.   The .ind field is only used on MMUs that have
  50  * indirect page table entries.
  51  */
  52 #ifdef CONFIG_PPC_E500
  53 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
  54         [MMU_PAGE_4K] = {
  55                 .shift  = 12,
  56         },
  57         [MMU_PAGE_2M] = {
  58                 .shift  = 21,
  59         },
  60         [MMU_PAGE_4M] = {
  61                 .shift  = 22,
  62         },
  63         [MMU_PAGE_16M] = {
  64                 .shift  = 24,
  65         },
  66         [MMU_PAGE_64M] = {
  67                 .shift  = 26,
  68         },
  69         [MMU_PAGE_256M] = {
  70                 .shift  = 28,
  71         },
  72         [MMU_PAGE_1G] = {
  73                 .shift  = 30,
  74         },
  75 };
  76
  77 static inline int mmu_get_tsize(int psize)
  78 {
  79         return mmu_psize_defs[psize].shift - 10;
  80 }
  81 #else
  82 static inline int mmu_get_tsize(int psize)
  83 {
  84         /* This isn't used on !Book3E for now */
  85         return 0;
  86 }
  87 #endif
  88
  89 #ifdef CONFIG_PPC_8xx
  90 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
  91         [MMU_PAGE_4K] = {
  92                 .shift  = 12,
  93         },
  94         [MMU_PAGE_16K] = {
  95                 .shift  = 14,
  96         },
  97         [MMU_PAGE_512K] = {
  98                 .shift  = 19,
  99         },
 100         [MMU_PAGE_8M] = {
 101                 .shift  = 23,
 102         },
 103 };
 104 #endif
 105
 106 #ifdef CONFIG_PPC_E500
 107 /* next_tlbcam_idx is used to round-robin tlbcam entry assignment */
 108 DEFINE_PER_CPU(int, next_tlbcam_idx);
 109 EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
 110 #endif
 111
 112 /*
 113  * Base TLB flushing operations:
 114  *
 115  *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
 116  *  - flush_tlb_page(vma, vmaddr) flushes one page
 117  *  - flush_tlb_range(vma, start, end) flushes a range of pages
 118  *  - flush_tlb_kernel_range(start, end) flushes kernel pages
 119  *
 120  *  - local_* variants of page and mm only apply to the current
 121  *    processor
 122  */
 123
 124 #ifndef CONFIG_PPC_8xx
 125 /*
 126  * These are the base non-SMP variants of page and mm flushing
 127  */
 128 void local_flush_tlb_mm(struct mm_struct *mm)
 129 {
 130         unsigned int pid;
 131
 132         preempt_disable();
 133         pid = mm->context.id;
 134         if (pid != MMU_NO_CONTEXT)
 135                 _tlbil_pid(pid);
 136         preempt_enable();
 137 }
 138 EXPORT_SYMBOL(local_flush_tlb_mm);
 139
 140 void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
 141                             int tsize, int ind)
 142 {
 143         unsigned int pid;
 144
 145         preempt_disable();
 146         pid = mm ? mm->context.id : 0;
 147         if (pid != MMU_NO_CONTEXT)
 148                 _tlbil_va(vmaddr, pid, tsize, ind);
 149         preempt_enable();
 150 }
 151
 152 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 153 {
 154         __local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
 155                                mmu_get_tsize(mmu_virtual_psize), 0);
 156 }
 157 EXPORT_SYMBOL(local_flush_tlb_page);
 158
 159 void local_flush_tlb_page_psize(struct mm_struct *mm,
 160                                 unsigned long vmaddr, int psize)
 161 {
 162         __local_flush_tlb_page(mm, vmaddr, mmu_get_tsize(psize), 0);
 163 }
 164 EXPORT_SYMBOL(local_flush_tlb_page_psize);
 165
 166 #endif
 167
 168 /*
 169  * And here are the SMP non-local implementations
 170  */
 171 #ifdef CONFIG_SMP
 172
 173 static DEFINE_RAW_SPINLOCK(tlbivax_lock);
 174
 175 struct tlb_flush_param {
 176         unsigned long addr;
 177         unsigned int pid;
 178         unsigned int tsize;
 179         unsigned int ind;
 180 };
 181
 182 static void do_flush_tlb_mm_ipi(void *param)
 183 {
 184         struct tlb_flush_param *p = param;
 185
 186         _tlbil_pid(p ? p->pid : 0);
 187 }
 188
 189 static void do_flush_tlb_page_ipi(void *param)
 190 {
 191         struct tlb_flush_param *p = param;
 192
 193         _tlbil_va(p->addr, p->pid, p->tsize, p->ind);
 194 }
 195
 196
 197 /* Note on invalidations and PID:
 198  *
 199  * We snapshot the PID with preempt disabled. At this point, it can still
 200  * change either because:
 201  * - our context is being stolen (PID -> NO_CONTEXT) on another CPU
 202  * - we are invaliating some target that isn't currently running here
 203  *   and is concurrently acquiring a new PID on another CPU
 204  * - some other CPU is re-acquiring a lost PID for this mm
 205  * etc...
 206  *
 207  * However, this shouldn't be a problem as we only guarantee
 208  * invalidation of TLB entries present prior to this call, so we
 209  * don't care about the PID changing, and invalidating a stale PID
 210  * is generally harmless.
 211  */
 212
 213 void flush_tlb_mm(struct mm_struct *mm)
 214 {
 215         unsigned int pid;
 216
 217         preempt_disable();
 218         pid = mm->context.id;
 219         if (unlikely(pid == MMU_NO_CONTEXT))
 220                 goto no_context;
 221         if (!mm_is_core_local(mm)) {
 222                 struct tlb_flush_param p = { .pid = pid };
 223                 /* Ignores smp_processor_id() even if set. */
 224                 smp_call_function_many(mm_cpumask(mm),
 225                                        do_flush_tlb_mm_ipi, &p, 1);
 226         }
 227         _tlbil_pid(pid);
 228  no_context:
 229         preempt_enable();
 230 }
 231 EXPORT_SYMBOL(flush_tlb_mm);
 232
 233 void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
 234                       int tsize, int ind)
 235 {
 236         struct cpumask *cpu_mask;
 237         unsigned int pid;
 238
 239         /*
 240          * This function as well as __local_flush_tlb_page() must only be called
 241          * for user contexts.
 242          */
 243         if (WARN_ON(!mm))
 244                 return;
 245
 246         preempt_disable();
 247         pid = mm->context.id;
 248         if (unlikely(pid == MMU_NO_CONTEXT))
 249                 goto bail;
 250         cpu_mask = mm_cpumask(mm);
 251         if (!mm_is_core_local(mm)) {
 252                 /* If broadcast tlbivax is supported, use it */
 253                 if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
 254                         int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
 255                         if (lock)
 256                                 raw_spin_lock(&tlbivax_lock);
 257                         _tlbivax_bcast(vmaddr, pid, tsize, ind);
 258                         if (lock)
 259                                 raw_spin_unlock(&tlbivax_lock);
 260                         goto bail;
 261                 } else {
 262                         struct tlb_flush_param p = {
 263                                 .pid = pid,
 264                                 .addr = vmaddr,
 265                                 .tsize = tsize,
 266                                 .ind = ind,
 267                         };
 268                         /* Ignores smp_processor_id() even if set in cpu_mask */
 269                         smp_call_function_many(cpu_mask,
 270                                                do_flush_tlb_page_ipi, &p, 1);
 271                 }
 272         }
 273         _tlbil_va(vmaddr, pid, tsize, ind);
 274  bail:
 275         preempt_enable();
 276 }
 277
 278 void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 279 {
 280 #ifdef CONFIG_HUGETLB_PAGE
 281         if (vma && is_vm_hugetlb_page(vma))
 282                 flush_hugetlb_page(vma, vmaddr);
 283 #endif
 284
 285         __flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
 286                          mmu_get_tsize(mmu_virtual_psize), 0);
 287 }
 288 EXPORT_SYMBOL(flush_tlb_page);
 289
 290 #endif /* CONFIG_SMP */
 291
 292 /*
 293  * Flush kernel TLB entries in the given range
 294  */
 295 #ifndef CONFIG_PPC_8xx
 296 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 297 {
 298 #ifdef CONFIG_SMP
 299         preempt_disable();
 300         smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
 301         _tlbil_pid(0);
 302         preempt_enable();
 303 #else
 304         _tlbil_pid(0);
 305 #endif
 306 }
 307 EXPORT_SYMBOL(flush_tlb_kernel_range);
 308 #endif
 309
 310 /*
 311  * Currently, for range flushing, we just do a full mm flush. This should
 312  * be optimized based on a threshold on the size of the range, since
 313  * some implementation can stack multiple tlbivax before a tlbsync but
 314  * for now, we keep it that way
 315  */
 316 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 317                      unsigned long end)
 318
 319 {
 320         if (end - start == PAGE_SIZE && !(start & ~PAGE_MASK))
 321                 flush_tlb_page(vma, start);
 322         else
 323                 flush_tlb_mm(vma->vm_mm);
 324 }
 325 EXPORT_SYMBOL(flush_tlb_range);
 326
 327 void tlb_flush(struct mmu_gather *tlb)
 328 {
 329         flush_tlb_mm(tlb->mm);
 330 }
 331
 332 #ifndef CONFIG_PPC64
 333 void __init early_init_mmu(void)
 334 {
 335         unsigned long root = of_get_flat_dt_root();
 336
 337         if (IS_ENABLED(CONFIG_PPC_47x) && IS_ENABLED(CONFIG_SMP) &&
 338             of_get_flat_dt_prop(root, "cooperative-partition", NULL))
 339                 mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
 340 }
 341 #endif /* CONFIG_PPC64 */