riscv: enable MMU_GATHER_RCU_TABLE_FREE for SMP && MMU

In order to implement fast gup we need to ensure that the page
table walker is protected from page table pages being freed from
under it.

riscv situation is more complicated than other architectures: some
riscv platforms may use IPI to perform TLB shootdown, for example,
those platforms which support AIA, usually the riscv_ipi_for_rfence is
true on these platforms; some riscv platforms may rely on the SBI to
perform TLB shootdown, usually the riscv_ipi_for_rfence is false on
these platforms. To keep software pagetable walkers safe in this case
we switch to RCU based table free (MMU_GATHER_RCU_TABLE_FREE). See the
comment below 'ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE' in
include/asm-generic/tlb.h for more details.

This patch enables MMU_GATHER_RCU_TABLE_FREE, then use

*tlb_remove_page_ptdesc() for those platforms which use IPI to perform
TLB shootdown;

*tlb_remove_ptdesc() for those platforms which use SBI to perform TLB
shootdown;

Both case mean that disabling interrupts will block the free and
protect the fast gup page walker.

Signed-off-by: Jisheng Zhang <[email protected]>
Reviewed-by: Alexandre Ghiti <[email protected]>
Link: https://lore.kernel.org/r/[email protected]
Signed-off-by: Palmer Dabbelt <[email protected]>

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 95a2a06acc6a62412894e491c3bfd5d4a161d15b..a0d9b8f5371d9f9d762fb223a928ce82df5314ea 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -147,6 +147,7 @@ config RISCV
        select IRQ_FORCED_THREADING
        select KASAN_VMALLOC if KASAN
        select LOCK_MM_AND_FIND_VMA
+       select MMU_GATHER_RCU_TABLE_FREE if SMP && MMU
        select MODULES_USE_ELF_RELA if MODULES
        select MODULE_SECTIONS if MODULES
        select OF

diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h
index 3c5e3bd15f46e336a923e317d7db64fb528443d8..deaf971253a20102685aff042b10f8edbf7e614f 100644 (file)
--- a/arch/riscv/include/asm/pgalloc.h
+++ b/arch/riscv/include/asm/pgalloc.h
@@ -102,7 +102,10 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
                struct ptdesc *ptdesc = virt_to_ptdesc(pud);
 
                pagetable_pud_dtor(ptdesc);
-               tlb_remove_page_ptdesc(tlb, ptdesc);
+               if (riscv_use_ipi_for_rfence())
+                       tlb_remove_page_ptdesc(tlb, ptdesc);
+               else
+                       tlb_remove_ptdesc(tlb, ptdesc);
        }
 }
 
@@ -136,8 +139,12 @@ static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
                                  unsigned long addr)
 {
-       if (pgtable_l5_enabled)
-               tlb_remove_page_ptdesc(tlb, virt_to_ptdesc(p4d));
+       if (pgtable_l5_enabled) {
+               if (riscv_use_ipi_for_rfence())
+                       tlb_remove_page_ptdesc(tlb, virt_to_ptdesc(p4d));
+               else
+                       tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d));
+       }
 }
 #endif /* __PAGETABLE_PMD_FOLDED */
 
@@ -169,7 +176,10 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
        struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
 
        pagetable_pmd_dtor(ptdesc);
-       tlb_remove_page_ptdesc(tlb, ptdesc);
+       if (riscv_use_ipi_for_rfence())
+               tlb_remove_page_ptdesc(tlb, ptdesc);
+       else
+               tlb_remove_ptdesc(tlb, ptdesc);
 }
 
 #endif /* __PAGETABLE_PMD_FOLDED */
@@ -180,7 +190,10 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
        struct ptdesc *ptdesc = page_ptdesc(pte);
 
        pagetable_pte_dtor(ptdesc);
-       tlb_remove_page_ptdesc(tlb, ptdesc);
+       if (riscv_use_ipi_for_rfence())
+               tlb_remove_page_ptdesc(tlb, ptdesc);
+       else
+               tlb_remove_ptdesc(tlb, ptdesc);
 }
 #endif /* CONFIG_MMU */
 

diff --git a/arch/riscv/include/asm/tlb.h b/arch/riscv/include/asm/tlb.h
index 1eb5682b2af6065c9019e398df729f5b97a573c6..a0b8b853503fe7994e1c95410f5613a9b40fefb4 100644 (file)
--- a/arch/riscv/include/asm/tlb.h
+++ b/arch/riscv/include/asm/tlb.h
@@ -10,6 +10,24 @@ struct mmu_gather;
 
 static void tlb_flush(struct mmu_gather *tlb);
 
+#ifdef CONFIG_MMU
+#include <linux/swap.h>
+
+/*
+ * While riscv platforms with riscv_ipi_for_rfence as true require an IPI to
+ * perform TLB shootdown, some platforms with riscv_ipi_for_rfence as false use
+ * SBI to perform TLB shootdown. To keep software pagetable walkers safe in this
+ * case we switch to RCU based table free (MMU_GATHER_RCU_TABLE_FREE). See the
+ * comment below 'ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE' in include/asm-generic/tlb.h
+ * for more details.
+ */
+static inline void __tlb_remove_table(void *table)
+{
+       free_page_and_swap_cache(table);
+}
+
+#endif /* CONFIG_MMU */
+
 #define tlb_flush tlb_flush
 #include <asm-generic/tlb.h>