arch/arm/mm/tlb.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Copyright 2024 Google LLC
   3 // Author: Ard Biesheuvel <[email protected]>
   4
   5 #include <linux/types.h>
   6 #include <asm/tlbflush.h>
   7
   8 #ifdef CONFIG_CPU_TLB_V4WT
   9 void v4_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
  10 void v4_flush_kern_tlb_range(unsigned long, unsigned long);
  11
  12 struct cpu_tlb_fns v4_tlb_fns __initconst = {
  13         .flush_user_range       = v4_flush_user_tlb_range,
  14         .flush_kern_range       = v4_flush_kern_tlb_range,
  15         .tlb_flags              = v4_tlb_flags,
  16 };
  17 #endif
  18
  19 #ifdef CONFIG_CPU_TLB_V4WB
  20 void v4wb_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
  21 void v4wb_flush_kern_tlb_range(unsigned long, unsigned long);
  22
  23 struct cpu_tlb_fns v4wb_tlb_fns __initconst = {
  24         .flush_user_range       = v4wb_flush_user_tlb_range,
  25         .flush_kern_range       = v4wb_flush_kern_tlb_range,
  26         .tlb_flags              = v4wb_tlb_flags,
  27 };
  28 #endif
  29
  30 #if defined(CONFIG_CPU_TLB_V4WBI) || defined(CONFIG_CPU_TLB_FEROCEON)
  31 void v4wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
  32 void v4wbi_flush_kern_tlb_range(unsigned long, unsigned long);
  33
  34 struct cpu_tlb_fns v4wbi_tlb_fns __initconst = {
  35         .flush_user_range       = v4wbi_flush_user_tlb_range,
  36         .flush_kern_range       = v4wbi_flush_kern_tlb_range,
  37         .tlb_flags              = v4wbi_tlb_flags,
  38 };
  39 #endif
  40
  41 #ifdef CONFIG_CPU_TLB_V6
  42 void v6wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
  43 void v6wbi_flush_kern_tlb_range(unsigned long, unsigned long);
  44
  45 struct cpu_tlb_fns v6wbi_tlb_fns __initconst = {
  46         .flush_user_range       = v6wbi_flush_user_tlb_range,
  47         .flush_kern_range       = v6wbi_flush_kern_tlb_range,
  48         .tlb_flags              = v6wbi_tlb_flags,
  49 };
  50 #endif
  51
  52 #ifdef CONFIG_CPU_TLB_V7
  53 void v7wbi_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
  54 void v7wbi_flush_kern_tlb_range(unsigned long, unsigned long);
  55
  56 struct cpu_tlb_fns v7wbi_tlb_fns __initconst = {
  57         .flush_user_range       = v7wbi_flush_user_tlb_range,
  58         .flush_kern_range       = v7wbi_flush_kern_tlb_range,
  59         .tlb_flags              = IS_ENABLED(CONFIG_SMP) ? v7wbi_tlb_flags_smp
  60                                                          : v7wbi_tlb_flags_up,
  61 };
  62
  63 #ifdef CONFIG_SMP_ON_UP
  64 /* This will be run-time patched so the offset better be right */
  65 static_assert(offsetof(struct cpu_tlb_fns, tlb_flags) == 8);
  66
  67 asm("   .pushsection    \".alt.smp.init\", \"a\"                \n" \
  68     "   .align          2                                       \n" \
  69     "   .long           v7wbi_tlb_fns + 8 - .                   \n" \
  70     "   .long "         __stringify(v7wbi_tlb_flags_up) "       \n" \
  71     "   .popsection                                             \n");
  72 #endif
  73 #endif
  74
  75 #ifdef CONFIG_CPU_TLB_FA
  76 void fa_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
  77 void fa_flush_kern_tlb_range(unsigned long, unsigned long);
  78
  79 struct cpu_tlb_fns fa_tlb_fns __initconst = {
  80         .flush_user_range       = fa_flush_user_tlb_range,
  81         .flush_kern_range       = fa_flush_kern_tlb_range,
  82         .tlb_flags              = fa_tlb_flags,
  83 };
  84 #endif