arch/powerpc/mm/book3s64/radix_pgtable.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Page table handling routines for radix page table.
   4  *
   5  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
   6  */
   7
   8 #define pr_fmt(fmt) "radix-mmu: " fmt
   9
  10 #include <linux/io.h>
  11 #include <linux/kernel.h>
  12 #include <linux/sched/mm.h>
  13 #include <linux/memblock.h>
  14 #include <linux/of.h>
  15 #include <linux/of_fdt.h>
  16 #include <linux/mm.h>
  17 #include <linux/hugetlb.h>
  18 #include <linux/string_helpers.h>
  19 #include <linux/memory.h>
  20
  21 #include <asm/pgalloc.h>
  22 #include <asm/mmu_context.h>
  23 #include <asm/dma.h>
  24 #include <asm/machdep.h>
  25 #include <asm/mmu.h>
  26 #include <asm/firmware.h>
  27 #include <asm/powernv.h>
  28 #include <asm/sections.h>
  29 #include <asm/smp.h>
  30 #include <asm/trace.h>
  31 #include <asm/uaccess.h>
  32 #include <asm/ultravisor.h>
  33 #include <asm/set_memory.h>
  34
  35 #include <trace/events/thp.h>
  36
  37 #include <mm/mmu_decl.h>
  38
  39 unsigned int mmu_base_pid;
  40 unsigned long radix_mem_block_size __ro_after_init;
  41
  42 static __ref void *early_alloc_pgtable(unsigned long size, int nid,
  43                         unsigned long region_start, unsigned long region_end)
  44 {
  45         phys_addr_t min_addr = MEMBLOCK_LOW_LIMIT;
  46         phys_addr_t max_addr = MEMBLOCK_ALLOC_ANYWHERE;
  47         void *ptr;
  48
  49         if (region_start)
  50                 min_addr = region_start;
  51         if (region_end)
  52                 max_addr = region_end;
  53
  54         ptr = memblock_alloc_try_nid(size, size, min_addr, max_addr, nid);
  55
  56         if (!ptr)
  57                 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa max_addr=%pa\n",
  58                       __func__, size, size, nid, &min_addr, &max_addr);
  59
  60         return ptr;
  61 }
  62
  63 /*
  64  * When allocating pud or pmd pointers, we allocate a complete page
  65  * of PAGE_SIZE rather than PUD_TABLE_SIZE or PMD_TABLE_SIZE. This
  66  * is to ensure that the page obtained from the memblock allocator
  67  * can be completely used as page table page and can be freed
  68  * correctly when the page table entries are removed.
  69  */
  70 static int early_map_kernel_page(unsigned long ea, unsigned long pa,
  71                           pgprot_t flags,
  72                           unsigned int map_page_size,
  73                           int nid,
  74                           unsigned long region_start, unsigned long region_end)
  75 {
  76         unsigned long pfn = pa >> PAGE_SHIFT;
  77         pgd_t *pgdp;
  78         p4d_t *p4dp;
  79         pud_t *pudp;
  80         pmd_t *pmdp;
  81         pte_t *ptep;
  82
  83         pgdp = pgd_offset_k(ea);
  84         p4dp = p4d_offset(pgdp, ea);
  85         if (p4d_none(*p4dp)) {
  86                 pudp = early_alloc_pgtable(PAGE_SIZE, nid,
  87                                            region_start, region_end);
  88                 p4d_populate(&init_mm, p4dp, pudp);
  89         }
  90         pudp = pud_offset(p4dp, ea);
  91         if (map_page_size == PUD_SIZE) {
  92                 ptep = (pte_t *)pudp;
  93                 goto set_the_pte;
  94         }
  95         if (pud_none(*pudp)) {
  96                 pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start,
  97                                            region_end);
  98                 pud_populate(&init_mm, pudp, pmdp);
  99         }
 100         pmdp = pmd_offset(pudp, ea);
 101         if (map_page_size == PMD_SIZE) {
 102                 ptep = pmdp_ptep(pmdp);
 103                 goto set_the_pte;
 104         }
 105         if (!pmd_present(*pmdp)) {
 106                 ptep = early_alloc_pgtable(PAGE_SIZE, nid,
 107                                                 region_start, region_end);
 108                 pmd_populate_kernel(&init_mm, pmdp, ptep);
 109         }
 110         ptep = pte_offset_kernel(pmdp, ea);
 111
 112 set_the_pte:
 113         set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
 114         asm volatile("ptesync": : :"memory");
 115         return 0;
 116 }
 117
 118 /*
 119  * nid, region_start, and region_end are hints to try to place the page
 120  * table memory in the same node or region.
 121  */
 122 static int __map_kernel_page(unsigned long ea, unsigned long pa,
 123                           pgprot_t flags,
 124                           unsigned int map_page_size,
 125                           int nid,
 126                           unsigned long region_start, unsigned long region_end)
 127 {
 128         unsigned long pfn = pa >> PAGE_SHIFT;
 129         pgd_t *pgdp;
 130         p4d_t *p4dp;
 131         pud_t *pudp;
 132         pmd_t *pmdp;
 133         pte_t *ptep;
 134         /*
 135          * Make sure task size is correct as per the max adddr
 136          */
 137         BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
 138
 139 #ifdef CONFIG_PPC_64K_PAGES
 140         BUILD_BUG_ON(RADIX_KERN_MAP_SIZE != (1UL << MAX_EA_BITS_PER_CONTEXT));
 141 #endif
 142
 143         if (unlikely(!slab_is_available()))
 144                 return early_map_kernel_page(ea, pa, flags, map_page_size,
 145                                                 nid, region_start, region_end);
 146
 147         /*
 148          * Should make page table allocation functions be able to take a
 149          * node, so we can place kernel page tables on the right nodes after
 150          * boot.
 151          */
 152         pgdp = pgd_offset_k(ea);
 153         p4dp = p4d_offset(pgdp, ea);
 154         pudp = pud_alloc(&init_mm, p4dp, ea);
 155         if (!pudp)
 156                 return -ENOMEM;
 157         if (map_page_size == PUD_SIZE) {
 158                 ptep = (pte_t *)pudp;
 159                 goto set_the_pte;
 160         }
 161         pmdp = pmd_alloc(&init_mm, pudp, ea);
 162         if (!pmdp)
 163                 return -ENOMEM;
 164         if (map_page_size == PMD_SIZE) {
 165                 ptep = pmdp_ptep(pmdp);
 166                 goto set_the_pte;
 167         }
 168         ptep = pte_alloc_kernel(pmdp, ea);
 169         if (!ptep)
 170                 return -ENOMEM;
 171
 172 set_the_pte:
 173         set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
 174         asm volatile("ptesync": : :"memory");
 175         return 0;
 176 }
 177
 178 int radix__map_kernel_page(unsigned long ea, unsigned long pa,
 179                           pgprot_t flags,
 180                           unsigned int map_page_size)
 181 {
 182         return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
 183 }
 184
 185 #ifdef CONFIG_STRICT_KERNEL_RWX
 186 static void radix__change_memory_range(unsigned long start, unsigned long end,
 187                                        unsigned long clear)
 188 {
 189         unsigned long idx;
 190         pgd_t *pgdp;
 191         p4d_t *p4dp;
 192         pud_t *pudp;
 193         pmd_t *pmdp;
 194         pte_t *ptep;
 195
 196         start = ALIGN_DOWN(start, PAGE_SIZE);
 197         end = PAGE_ALIGN(end); // aligns up
 198
 199         pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
 200                  start, end, clear);
 201
 202         for (idx = start; idx < end; idx += PAGE_SIZE) {
 203                 pgdp = pgd_offset_k(idx);
 204                 p4dp = p4d_offset(pgdp, idx);
 205                 pudp = pud_alloc(&init_mm, p4dp, idx);
 206                 if (!pudp)
 207                         continue;
 208                 if (pud_is_leaf(*pudp)) {
 209                         ptep = (pte_t *)pudp;
 210                         goto update_the_pte;
 211                 }
 212                 pmdp = pmd_alloc(&init_mm, pudp, idx);
 213                 if (!pmdp)
 214                         continue;
 215                 if (pmd_is_leaf(*pmdp)) {
 216                         ptep = pmdp_ptep(pmdp);
 217                         goto update_the_pte;
 218                 }
 219                 ptep = pte_alloc_kernel(pmdp, idx);
 220                 if (!ptep)
 221                         continue;
 222 update_the_pte:
 223                 radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
 224         }
 225
 226         radix__flush_tlb_kernel_range(start, end);
 227 }
 228
 229 void radix__mark_rodata_ro(void)
 230 {
 231         unsigned long start, end;
 232
 233         start = (unsigned long)_stext;
 234         end = (unsigned long)__end_rodata;
 235
 236         radix__change_memory_range(start, end, _PAGE_WRITE);
 237 }
 238
 239 void radix__mark_initmem_nx(void)
 240 {
 241         unsigned long start = (unsigned long)__init_begin;
 242         unsigned long end = (unsigned long)__init_end;
 243
 244         radix__change_memory_range(start, end, _PAGE_EXEC);
 245 }
 246 #endif /* CONFIG_STRICT_KERNEL_RWX */
 247
 248 static inline void __meminit
 249 print_mapping(unsigned long start, unsigned long end, unsigned long size, bool exec)
 250 {
 251         char buf[10];
 252
 253         if (end <= start)
 254                 return;
 255
 256         string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
 257
 258         pr_info("Mapped 0x%016lx-0x%016lx with %s pages%s\n", start, end, buf,
 259                 exec ? " (exec)" : "");
 260 }
 261
 262 static unsigned long next_boundary(unsigned long addr, unsigned long end)
 263 {
 264 #ifdef CONFIG_STRICT_KERNEL_RWX
 265         if (addr < __pa_symbol(__srwx_boundary))
 266                 return __pa_symbol(__srwx_boundary);
 267 #endif
 268         return end;
 269 }
 270
 271 static int __meminit create_physical_mapping(unsigned long start,
 272                                              unsigned long end,
 273                                              int nid, pgprot_t _prot)
 274 {
 275         unsigned long vaddr, addr, mapping_size = 0;
 276         bool prev_exec, exec = false;
 277         pgprot_t prot;
 278         int psize;
 279         unsigned long max_mapping_size = radix_mem_block_size;
 280
 281         if (debug_pagealloc_enabled_or_kfence())
 282                 max_mapping_size = PAGE_SIZE;
 283
 284         start = ALIGN(start, PAGE_SIZE);
 285         end   = ALIGN_DOWN(end, PAGE_SIZE);
 286         for (addr = start; addr < end; addr += mapping_size) {
 287                 unsigned long gap, previous_size;
 288                 int rc;
 289
 290                 gap = next_boundary(addr, end) - addr;
 291                 if (gap > max_mapping_size)
 292                         gap = max_mapping_size;
 293                 previous_size = mapping_size;
 294                 prev_exec = exec;
 295
 296                 if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
 297                     mmu_psize_defs[MMU_PAGE_1G].shift) {
 298                         mapping_size = PUD_SIZE;
 299                         psize = MMU_PAGE_1G;
 300                 } else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
 301                            mmu_psize_defs[MMU_PAGE_2M].shift) {
 302                         mapping_size = PMD_SIZE;
 303                         psize = MMU_PAGE_2M;
 304                 } else {
 305                         mapping_size = PAGE_SIZE;
 306                         psize = mmu_virtual_psize;
 307                 }
 308
 309                 vaddr = (unsigned long)__va(addr);
 310
 311                 if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
 312                     overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size)) {
 313                         prot = PAGE_KERNEL_X;
 314                         exec = true;
 315                 } else {
 316                         prot = _prot;
 317                         exec = false;
 318                 }
 319
 320                 if (mapping_size != previous_size || exec != prev_exec) {
 321                         print_mapping(start, addr, previous_size, prev_exec);
 322                         start = addr;
 323                 }
 324
 325                 rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
 326                 if (rc)
 327                         return rc;
 328
 329                 update_page_count(psize, 1);
 330         }
 331
 332         print_mapping(start, addr, mapping_size, exec);
 333         return 0;
 334 }
 335
 336 static void __init radix_init_pgtable(void)
 337 {
 338         unsigned long rts_field;
 339         phys_addr_t start, end;
 340         u64 i;
 341
 342         /* We don't support slb for radix */
 343         slb_set_size(0);
 344
 345         /*
 346          * Create the linear mapping
 347          */
 348         for_each_mem_range(i, &start, &end) {
 349                 /*
 350                  * The memblock allocator  is up at this point, so the
 351                  * page tables will be allocated within the range. No
 352                  * need or a node (which we don't have yet).
 353                  */
 354
 355                 if (end >= RADIX_VMALLOC_START) {
 356                         pr_warn("Outside the supported range\n");
 357                         continue;
 358                 }
 359
 360                 WARN_ON(create_physical_mapping(start, end,
 361                                                 -1, PAGE_KERNEL));
 362         }
 363
 364         if (!cpu_has_feature(CPU_FTR_HVMODE) &&
 365                         cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
 366                 /*
 367                  * Older versions of KVM on these machines prefer if the
 368                  * guest only uses the low 19 PID bits.
 369                  */
 370                 mmu_pid_bits = 19;
 371         }
 372         mmu_base_pid = 1;
 373
 374         /*
 375          * Allocate Partition table and process table for the
 376          * host.
 377          */
 378         BUG_ON(PRTB_SIZE_SHIFT > 36);
 379         process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
 380         /*
 381          * Fill in the process table.
 382          */
 383         rts_field = radix__get_tree_size();
 384         process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
 385
 386         /*
 387          * The init_mm context is given the first available (non-zero) PID,
 388          * which is the "guard PID" and contains no page table. PIDR should
 389          * never be set to zero because that duplicates the kernel address
 390          * space at the 0x0... offset (quadrant 0)!
 391          *
 392          * An arbitrary PID that may later be allocated by the PID allocator
 393          * for userspace processes must not be used either, because that
 394          * would cause stale user mappings for that PID on CPUs outside of
 395          * the TLB invalidation scheme (because it won't be in mm_cpumask).
 396          *
 397          * So permanently carve out one PID for the purpose of a guard PID.
 398          */
 399         init_mm.context.id = mmu_base_pid;
 400         mmu_base_pid++;
 401 }
 402
 403 static void __init radix_init_partition_table(void)
 404 {
 405         unsigned long rts_field, dw0, dw1;
 406
 407         mmu_partition_table_init();
 408         rts_field = radix__get_tree_size();
 409         dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
 410         dw1 = __pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR;
 411         mmu_partition_table_set_entry(0, dw0, dw1, false);
 412
 413         pr_info("Initializing Radix MMU\n");
 414 }
 415
 416 static int __init get_idx_from_shift(unsigned int shift)
 417 {
 418         int idx = -1;
 419
 420         switch (shift) {
 421         case 0xc:
 422                 idx = MMU_PAGE_4K;
 423                 break;
 424         case 0x10:
 425                 idx = MMU_PAGE_64K;
 426                 break;
 427         case 0x15:
 428                 idx = MMU_PAGE_2M;
 429                 break;
 430         case 0x1e:
 431                 idx = MMU_PAGE_1G;
 432                 break;
 433         }
 434         return idx;
 435 }
 436
 437 static int __init radix_dt_scan_page_sizes(unsigned long node,
 438                                            const char *uname, int depth,
 439                                            void *data)
 440 {
 441         int size = 0;
 442         int shift, idx;
 443         unsigned int ap;
 444         const __be32 *prop;
 445         const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
 446
 447         /* We are scanning "cpu" nodes only */
 448         if (type == NULL || strcmp(type, "cpu") != 0)
 449                 return 0;
 450
 451         /* Grab page size encodings */
 452         prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
 453         if (!prop)
 454                 return 0;
 455
 456         pr_info("Page sizes from device-tree:\n");
 457         for (; size >= 4; size -= 4, ++prop) {
 458
 459                 struct mmu_psize_def *def;
 460
 461                 /* top 3 bit is AP encoding */
 462                 shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
 463                 ap = be32_to_cpu(prop[0]) >> 29;
 464                 pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
 465
 466                 idx = get_idx_from_shift(shift);
 467                 if (idx < 0)
 468                         continue;
 469
 470                 def = &mmu_psize_defs[idx];
 471                 def->shift = shift;
 472                 def->ap  = ap;
 473                 def->h_rpt_pgsize = psize_to_rpti_pgsize(idx);
 474         }
 475
 476         /* needed ? */
 477         cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
 478         return 1;
 479 }
 480
 481 #ifdef CONFIG_MEMORY_HOTPLUG
 482 static int __init probe_memory_block_size(unsigned long node, const char *uname, int
 483                                           depth, void *data)
 484 {
 485         unsigned long *mem_block_size = (unsigned long *)data;
 486         const __be32 *prop;
 487         int len;
 488
 489         if (depth != 1)
 490                 return 0;
 491
 492         if (strcmp(uname, "ibm,dynamic-reconfiguration-memory"))
 493                 return 0;
 494
 495         prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
 496
 497         if (!prop || len < dt_root_size_cells * sizeof(__be32))
 498                 /*
 499                  * Nothing in the device tree
 500                  */
 501                 *mem_block_size = MIN_MEMORY_BLOCK_SIZE;
 502         else
 503                 *mem_block_size = of_read_number(prop, dt_root_size_cells);
 504         return 1;
 505 }
 506
 507 static unsigned long __init radix_memory_block_size(void)
 508 {
 509         unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE;
 510
 511         /*
 512          * OPAL firmware feature is set by now. Hence we are ok
 513          * to test OPAL feature.
 514          */
 515         if (firmware_has_feature(FW_FEATURE_OPAL))
 516                 mem_block_size = 1UL * 1024 * 1024 * 1024;
 517         else
 518                 of_scan_flat_dt(probe_memory_block_size, &mem_block_size);
 519
 520         return mem_block_size;
 521 }
 522
 523 #else   /* CONFIG_MEMORY_HOTPLUG */
 524
 525 static unsigned long __init radix_memory_block_size(void)
 526 {
 527         return 1UL * 1024 * 1024 * 1024;
 528 }
 529
 530 #endif /* CONFIG_MEMORY_HOTPLUG */
 531
 532
 533 void __init radix__early_init_devtree(void)
 534 {
 535         int rc;
 536
 537         /*
 538          * Try to find the available page sizes in the device-tree
 539          */
 540         rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
 541         if (!rc) {
 542                 /*
 543                  * No page size details found in device tree.
 544                  * Let's assume we have page 4k and 64k support
 545                  */
 546                 mmu_psize_defs[MMU_PAGE_4K].shift = 12;
 547                 mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
 548                 mmu_psize_defs[MMU_PAGE_4K].h_rpt_pgsize =
 549                         psize_to_rpti_pgsize(MMU_PAGE_4K);
 550
 551                 mmu_psize_defs[MMU_PAGE_64K].shift = 16;
 552                 mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
 553                 mmu_psize_defs[MMU_PAGE_64K].h_rpt_pgsize =
 554                         psize_to_rpti_pgsize(MMU_PAGE_64K);
 555         }
 556
 557         /*
 558          * Max mapping size used when mapping pages. We don't use
 559          * ppc_md.memory_block_size() here because this get called
 560          * early and we don't have machine probe called yet. Also
 561          * the pseries implementation only check for ibm,lmb-size.
 562          * All hypervisor supporting radix do expose that device
 563          * tree node.
 564          */
 565         radix_mem_block_size = radix_memory_block_size();
 566         return;
 567 }
 568
 569 void __init radix__early_init_mmu(void)
 570 {
 571         unsigned long lpcr;
 572
 573 #ifdef CONFIG_PPC_64S_HASH_MMU
 574 #ifdef CONFIG_PPC_64K_PAGES
 575         /* PAGE_SIZE mappings */
 576         mmu_virtual_psize = MMU_PAGE_64K;
 577 #else
 578         mmu_virtual_psize = MMU_PAGE_4K;
 579 #endif
 580
 581 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 582         /* vmemmap mapping */
 583         if (mmu_psize_defs[MMU_PAGE_2M].shift) {
 584                 /*
 585                  * map vmemmap using 2M if available
 586                  */
 587                 mmu_vmemmap_psize = MMU_PAGE_2M;
 588         } else
 589                 mmu_vmemmap_psize = mmu_virtual_psize;
 590 #endif
 591 #endif
 592         /*
 593          * initialize page table size
 594          */
 595         __pte_index_size = RADIX_PTE_INDEX_SIZE;
 596         __pmd_index_size = RADIX_PMD_INDEX_SIZE;
 597         __pud_index_size = RADIX_PUD_INDEX_SIZE;
 598         __pgd_index_size = RADIX_PGD_INDEX_SIZE;
 599         __pud_cache_index = RADIX_PUD_INDEX_SIZE;
 600         __pte_table_size = RADIX_PTE_TABLE_SIZE;
 601         __pmd_table_size = RADIX_PMD_TABLE_SIZE;
 602         __pud_table_size = RADIX_PUD_TABLE_SIZE;
 603         __pgd_table_size = RADIX_PGD_TABLE_SIZE;
 604
 605         __pmd_val_bits = RADIX_PMD_VAL_BITS;
 606         __pud_val_bits = RADIX_PUD_VAL_BITS;
 607         __pgd_val_bits = RADIX_PGD_VAL_BITS;
 608
 609         __kernel_virt_start = RADIX_KERN_VIRT_START;
 610         __vmalloc_start = RADIX_VMALLOC_START;
 611         __vmalloc_end = RADIX_VMALLOC_END;
 612         __kernel_io_start = RADIX_KERN_IO_START;
 613         __kernel_io_end = RADIX_KERN_IO_END;
 614         vmemmap = (struct page *)RADIX_VMEMMAP_START;
 615         ioremap_bot = IOREMAP_BASE;
 616
 617 #ifdef CONFIG_PCI
 618         pci_io_base = ISA_IO_BASE;
 619 #endif
 620         __pte_frag_nr = RADIX_PTE_FRAG_NR;
 621         __pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
 622         __pmd_frag_nr = RADIX_PMD_FRAG_NR;
 623         __pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
 624
 625         radix_init_pgtable();
 626
 627         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 628                 lpcr = mfspr(SPRN_LPCR);
 629                 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 630                 radix_init_partition_table();
 631         } else {
 632                 radix_init_pseries();
 633         }
 634
 635         memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
 636
 637         /* Switch to the guard PID before turning on MMU */
 638         radix__switch_mmu_context(NULL, &init_mm);
 639         tlbiel_all();
 640 }
 641
 642 void radix__early_init_mmu_secondary(void)
 643 {
 644         unsigned long lpcr;
 645         /*
 646          * update partition table control register and UPRT
 647          */
 648         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 649                 lpcr = mfspr(SPRN_LPCR);
 650                 mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
 651
 652                 set_ptcr_when_no_uv(__pa(partition_tb) |
 653                                     (PATB_SIZE_SHIFT - 12));
 654         }
 655
 656         radix__switch_mmu_context(NULL, &init_mm);
 657         tlbiel_all();
 658
 659         /* Make sure userspace can't change the AMR */
 660         mtspr(SPRN_UAMOR, 0);
 661 }
 662
 663 /* Called during kexec sequence with MMU off */
 664 notrace void radix__mmu_cleanup_all(void)
 665 {
 666         unsigned long lpcr;
 667
 668         if (!firmware_has_feature(FW_FEATURE_LPAR)) {
 669                 lpcr = mfspr(SPRN_LPCR);
 670                 mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
 671                 set_ptcr_when_no_uv(0);
 672                 powernv_set_nmmu_ptcr(0);
 673                 radix__flush_tlb_all();
 674         }
 675 }
 676
 677 #ifdef CONFIG_MEMORY_HOTPLUG
 678 static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
 679 {
 680         pte_t *pte;
 681         int i;
 682
 683         for (i = 0; i < PTRS_PER_PTE; i++) {
 684                 pte = pte_start + i;
 685                 if (!pte_none(*pte))
 686                         return;
 687         }
 688
 689         pte_free_kernel(&init_mm, pte_start);
 690         pmd_clear(pmd);
 691 }
 692
 693 static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
 694 {
 695         pmd_t *pmd;
 696         int i;
 697
 698         for (i = 0; i < PTRS_PER_PMD; i++) {
 699                 pmd = pmd_start + i;
 700                 if (!pmd_none(*pmd))
 701                         return;
 702         }
 703
 704         pmd_free(&init_mm, pmd_start);
 705         pud_clear(pud);
 706 }
 707
 708 static void free_pud_table(pud_t *pud_start, p4d_t *p4d)
 709 {
 710         pud_t *pud;
 711         int i;
 712
 713         for (i = 0; i < PTRS_PER_PUD; i++) {
 714                 pud = pud_start + i;
 715                 if (!pud_none(*pud))
 716                         return;
 717         }
 718
 719         pud_free(&init_mm, pud_start);
 720         p4d_clear(p4d);
 721 }
 722
 723 static void remove_pte_table(pte_t *pte_start, unsigned long addr,
 724                              unsigned long end)
 725 {
 726         unsigned long next;
 727         pte_t *pte;
 728
 729         pte = pte_start + pte_index(addr);
 730         for (; addr < end; addr = next, pte++) {
 731                 next = (addr + PAGE_SIZE) & PAGE_MASK;
 732                 if (next > end)
 733                         next = end;
 734
 735                 if (!pte_present(*pte))
 736                         continue;
 737
 738                 if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
 739                         /*
 740                          * The vmemmap_free() and remove_section_mapping()
 741                          * codepaths call us with aligned addresses.
 742                          */
 743                         WARN_ONCE(1, "%s: unaligned range\n", __func__);
 744                         continue;
 745                 }
 746
 747                 pte_clear(&init_mm, addr, pte);
 748         }
 749 }
 750
 751 static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
 752                              unsigned long end)
 753 {
 754         unsigned long next;
 755         pte_t *pte_base;
 756         pmd_t *pmd;
 757
 758         pmd = pmd_start + pmd_index(addr);
 759         for (; addr < end; addr = next, pmd++) {
 760                 next = pmd_addr_end(addr, end);
 761
 762                 if (!pmd_present(*pmd))
 763                         continue;
 764
 765                 if (pmd_is_leaf(*pmd)) {
 766                         if (!IS_ALIGNED(addr, PMD_SIZE) ||
 767                             !IS_ALIGNED(next, PMD_SIZE)) {
 768                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 769                                 continue;
 770                         }
 771                         pte_clear(&init_mm, addr, (pte_t *)pmd);
 772                         continue;
 773                 }
 774
 775                 pte_base = (pte_t *)pmd_page_vaddr(*pmd);
 776                 remove_pte_table(pte_base, addr, next);
 777                 free_pte_table(pte_base, pmd);
 778         }
 779 }
 780
 781 static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
 782                              unsigned long end)
 783 {
 784         unsigned long next;
 785         pmd_t *pmd_base;
 786         pud_t *pud;
 787
 788         pud = pud_start + pud_index(addr);
 789         for (; addr < end; addr = next, pud++) {
 790                 next = pud_addr_end(addr, end);
 791
 792                 if (!pud_present(*pud))
 793                         continue;
 794
 795                 if (pud_is_leaf(*pud)) {
 796                         if (!IS_ALIGNED(addr, PUD_SIZE) ||
 797                             !IS_ALIGNED(next, PUD_SIZE)) {
 798                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 799                                 continue;
 800                         }
 801                         pte_clear(&init_mm, addr, (pte_t *)pud);
 802                         continue;
 803                 }
 804
 805                 pmd_base = pud_pgtable(*pud);
 806                 remove_pmd_table(pmd_base, addr, next);
 807                 free_pmd_table(pmd_base, pud);
 808         }
 809 }
 810
 811 static void __meminit remove_pagetable(unsigned long start, unsigned long end)
 812 {
 813         unsigned long addr, next;
 814         pud_t *pud_base;
 815         pgd_t *pgd;
 816         p4d_t *p4d;
 817
 818         spin_lock(&init_mm.page_table_lock);
 819
 820         for (addr = start; addr < end; addr = next) {
 821                 next = pgd_addr_end(addr, end);
 822
 823                 pgd = pgd_offset_k(addr);
 824                 p4d = p4d_offset(pgd, addr);
 825                 if (!p4d_present(*p4d))
 826                         continue;
 827
 828                 if (p4d_is_leaf(*p4d)) {
 829                         if (!IS_ALIGNED(addr, P4D_SIZE) ||
 830                             !IS_ALIGNED(next, P4D_SIZE)) {
 831                                 WARN_ONCE(1, "%s: unaligned range\n", __func__);
 832                                 continue;
 833                         }
 834
 835                         pte_clear(&init_mm, addr, (pte_t *)pgd);
 836                         continue;
 837                 }
 838
 839                 pud_base = p4d_pgtable(*p4d);
 840                 remove_pud_table(pud_base, addr, next);
 841                 free_pud_table(pud_base, p4d);
 842         }
 843
 844         spin_unlock(&init_mm.page_table_lock);
 845         radix__flush_tlb_kernel_range(start, end);
 846 }
 847
 848 int __meminit radix__create_section_mapping(unsigned long start,
 849                                             unsigned long end, int nid,
 850                                             pgprot_t prot)
 851 {
 852         if (end >= RADIX_VMALLOC_START) {
 853                 pr_warn("Outside the supported range\n");
 854                 return -1;
 855         }
 856
 857         return create_physical_mapping(__pa(start), __pa(end),
 858                                        nid, prot);
 859 }
 860
 861 int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
 862 {
 863         remove_pagetable(start, end);
 864         return 0;
 865 }
 866 #endif /* CONFIG_MEMORY_HOTPLUG */
 867
 868 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 869 static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
 870                                  pgprot_t flags, unsigned int map_page_size,
 871                                  int nid)
 872 {
 873         return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
 874 }
 875
 876 int __meminit radix__vmemmap_create_mapping(unsigned long start,
 877                                       unsigned long page_size,
 878                                       unsigned long phys)
 879 {
 880         /* Create a PTE encoding */
 881         unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
 882         int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
 883         int ret;
 884
 885         if ((start + page_size) >= RADIX_VMEMMAP_END) {
 886                 pr_warn("Outside the supported range\n");
 887                 return -1;
 888         }
 889
 890         ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
 891         BUG_ON(ret);
 892
 893         return 0;
 894 }
 895
 896 #ifdef CONFIG_MEMORY_HOTPLUG
 897 void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
 898 {
 899         remove_pagetable(start, start + page_size);
 900 }
 901 #endif
 902 #endif
 903
 904 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE)
 905 void radix__kernel_map_pages(struct page *page, int numpages, int enable)
 906 {
 907         unsigned long addr;
 908
 909         addr = (unsigned long)page_address(page);
 910
 911         if (enable)
 912                 set_memory_p(addr, numpages);
 913         else
 914                 set_memory_np(addr, numpages);
 915 }
 916 #endif
 917
 918 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 919
 920 unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
 921                                   pmd_t *pmdp, unsigned long clr,
 922                                   unsigned long set)
 923 {
 924         unsigned long old;
 925
 926 #ifdef CONFIG_DEBUG_VM
 927         WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
 928         assert_spin_locked(pmd_lockptr(mm, pmdp));
 929 #endif
 930
 931         old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
 932         trace_hugepage_update(addr, old, clr, set);
 933
 934         return old;
 935 }
 936
 937 pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
 938                         pmd_t *pmdp)
 939
 940 {
 941         pmd_t pmd;
 942
 943         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 944         VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
 945         VM_BUG_ON(pmd_devmap(*pmdp));
 946         /*
 947          * khugepaged calls this for normal pmd
 948          */
 949         pmd = *pmdp;
 950         pmd_clear(pmdp);
 951
 952         radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
 953
 954         return pmd;
 955 }
 956
 957 /*
 958  * For us pgtable_t is pte_t *. Inorder to save the deposisted
 959  * page table, we consider the allocated page table as a list
 960  * head. On withdraw we need to make sure we zero out the used
 961  * list_head memory area.
 962  */
 963 void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 964                                  pgtable_t pgtable)
 965 {
 966         struct list_head *lh = (struct list_head *) pgtable;
 967
 968         assert_spin_locked(pmd_lockptr(mm, pmdp));
 969
 970         /* FIFO */
 971         if (!pmd_huge_pte(mm, pmdp))
 972                 INIT_LIST_HEAD(lh);
 973         else
 974                 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
 975         pmd_huge_pte(mm, pmdp) = pgtable;
 976 }
 977
 978 pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
 979 {
 980         pte_t *ptep;
 981         pgtable_t pgtable;
 982         struct list_head *lh;
 983
 984         assert_spin_locked(pmd_lockptr(mm, pmdp));
 985
 986         /* FIFO */
 987         pgtable = pmd_huge_pte(mm, pmdp);
 988         lh = (struct list_head *) pgtable;
 989         if (list_empty(lh))
 990                 pmd_huge_pte(mm, pmdp) = NULL;
 991         else {
 992                 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
 993                 list_del(lh);
 994         }
 995         ptep = (pte_t *) pgtable;
 996         *ptep = __pte(0);
 997         ptep++;
 998         *ptep = __pte(0);
 999         return pgtable;
1000 }
1001
1002 pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
1003                                      unsigned long addr, pmd_t *pmdp)
1004 {
1005         pmd_t old_pmd;
1006         unsigned long old;
1007
1008         old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
1009         old_pmd = __pmd(old);
1010         return old_pmd;
1011 }
1012
1013 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1014
1015 void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
1016                                   pte_t entry, unsigned long address, int psize)
1017 {
1018         struct mm_struct *mm = vma->vm_mm;
1019         unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
1020                                               _PAGE_RW | _PAGE_EXEC);
1021
1022         unsigned long change = pte_val(entry) ^ pte_val(*ptep);
1023         /*
1024          * On POWER9, the NMMU is not able to relax PTE access permissions
1025          * for a translation with a TLB. The PTE must be invalidated, TLB
1026          * flushed before the new PTE is installed.
1027          *
1028          * This only needs to be done for radix, because hash translation does
1029          * flush when updating the linux pte (and we don't support NMMU
1030          * accelerators on HPT on POWER9 anyway XXX: do we?).
1031          *
1032          * POWER10 (and P9P) NMMU does behave as per ISA.
1033          */
1034         if (!cpu_has_feature(CPU_FTR_ARCH_31) && (change & _PAGE_RW) &&
1035             atomic_read(&mm->context.copros) > 0) {
1036                 unsigned long old_pte, new_pte;
1037
1038                 old_pte = __radix_pte_update(ptep, _PAGE_PRESENT, _PAGE_INVALID);
1039                 new_pte = old_pte | set;
1040                 radix__flush_tlb_page_psize(mm, address, psize);
1041                 __radix_pte_update(ptep, _PAGE_INVALID, new_pte);
1042         } else {
1043                 __radix_pte_update(ptep, 0, set);
1044                 /*
1045                  * Book3S does not require a TLB flush when relaxing access
1046                  * restrictions when the address space (modulo the POWER9 nest
1047                  * MMU issue above) because the MMU will reload the PTE after
1048                  * taking an access fault, as defined by the architecture. See
1049                  * "Setting a Reference or Change Bit or Upgrading Access
1050                  *  Authority (PTE Subject to Atomic Hardware Updates)" in
1051                  *  Power ISA Version 3.1B.
1052                  */
1053         }
1054         /* See ptesync comment in radix__set_pte_at */
1055 }
1056
1057 void radix__ptep_modify_prot_commit(struct vm_area_struct *vma,
1058                                     unsigned long addr, pte_t *ptep,
1059                                     pte_t old_pte, pte_t pte)
1060 {
1061         struct mm_struct *mm = vma->vm_mm;
1062
1063         /*
1064          * POWER9 NMMU must flush the TLB after clearing the PTE before
1065          * installing a PTE with more relaxed access permissions, see
1066          * radix__ptep_set_access_flags.
1067          */
1068         if (!cpu_has_feature(CPU_FTR_ARCH_31) &&
1069             is_pte_rw_upgrade(pte_val(old_pte), pte_val(pte)) &&
1070             (atomic_read(&mm->context.copros) > 0))
1071                 radix__flush_tlb_page(vma, addr);
1072
1073         set_pte_at(mm, addr, ptep, pte);
1074 }
1075
1076 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1077 {
1078         pte_t *ptep = (pte_t *)pud;
1079         pte_t new_pud = pfn_pte(__phys_to_pfn(addr), prot);
1080
1081         if (!radix_enabled())
1082                 return 0;
1083
1084         set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pud);
1085
1086         return 1;
1087 }
1088
1089 int pud_clear_huge(pud_t *pud)
1090 {
1091         if (pud_is_leaf(*pud)) {
1092                 pud_clear(pud);
1093                 return 1;
1094         }
1095
1096         return 0;
1097 }
1098
1099 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1100 {
1101         pmd_t *pmd;
1102         int i;
1103
1104         pmd = pud_pgtable(*pud);
1105         pud_clear(pud);
1106
1107         flush_tlb_kernel_range(addr, addr + PUD_SIZE);
1108
1109         for (i = 0; i < PTRS_PER_PMD; i++) {
1110                 if (!pmd_none(pmd[i])) {
1111                         pte_t *pte;
1112                         pte = (pte_t *)pmd_page_vaddr(pmd[i]);
1113
1114                         pte_free_kernel(&init_mm, pte);
1115                 }
1116         }
1117
1118         pmd_free(&init_mm, pmd);
1119
1120         return 1;
1121 }
1122
1123 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1124 {
1125         pte_t *ptep = (pte_t *)pmd;
1126         pte_t new_pmd = pfn_pte(__phys_to_pfn(addr), prot);
1127
1128         if (!radix_enabled())
1129                 return 0;
1130
1131         set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pmd);
1132
1133         return 1;
1134 }
1135
1136 int pmd_clear_huge(pmd_t *pmd)
1137 {
1138         if (pmd_is_leaf(*pmd)) {
1139                 pmd_clear(pmd);
1140                 return 1;
1141         }
1142
1143         return 0;
1144 }
1145
1146 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1147 {
1148         pte_t *pte;
1149
1150         pte = (pte_t *)pmd_page_vaddr(*pmd);
1151         pmd_clear(pmd);
1152
1153         flush_tlb_kernel_range(addr, addr + PMD_SIZE);
1154
1155         pte_free_kernel(&init_mm, pte);
1156
1157         return 1;
1158 }