virt/kvm/arm/mmu.c

   1 /*
   2  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
   3  * Author: Christoffer Dall <[email protected]>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License, version 2, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  17  */
  18
  19 #include <linux/mman.h>
  20 #include <linux/kvm_host.h>
  21 #include <linux/io.h>
  22 #include <linux/hugetlb.h>
  23 #include <linux/sched/signal.h>
  24 #include <trace/events/kvm.h>
  25 #include <asm/pgalloc.h>
  26 #include <asm/cacheflush.h>
  27 #include <asm/kvm_arm.h>
  28 #include <asm/kvm_mmu.h>
  29 #include <asm/kvm_mmio.h>
  30 #include <asm/kvm_asm.h>
  31 #include <asm/kvm_emulate.h>
  32 #include <asm/virt.h>
  33 #include <asm/system_misc.h>
  34
  35 #include "trace.h"
  36
  37 static pgd_t *boot_hyp_pgd;
  38 static pgd_t *hyp_pgd;
  39 static pgd_t *merged_hyp_pgd;
  40 static DEFINE_MUTEX(kvm_hyp_pgd_mutex);
  41
  42 static unsigned long hyp_idmap_start;
  43 static unsigned long hyp_idmap_end;
  44 static phys_addr_t hyp_idmap_vector;
  45
  46 static unsigned long io_map_base;
  47
  48 #define hyp_pgd_order get_order(PTRS_PER_PGD * sizeof(pgd_t))
  49
  50 #define KVM_S2PTE_FLAG_IS_IOMAP         (1UL << 0)
  51 #define KVM_S2_FLAG_LOGGING_ACTIVE      (1UL << 1)
  52
  53 static bool memslot_is_logging(struct kvm_memory_slot *memslot)
  54 {
  55         return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY);
  56 }
  57
  58 /**
  59  * kvm_flush_remote_tlbs() - flush all VM TLB entries for v7/8
  60  * @kvm:        pointer to kvm structure.
  61  *
  62  * Interface to HYP function to flush all VM TLB entries
  63  */
  64 void kvm_flush_remote_tlbs(struct kvm *kvm)
  65 {
  66         kvm_call_hyp(__kvm_tlb_flush_vmid, kvm);
  67 }
  68
  69 static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
  70 {
  71         kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa);
  72 }
  73
  74 /*
  75  * D-Cache management functions. They take the page table entries by
  76  * value, as they are flushing the cache using the kernel mapping (or
  77  * kmap on 32bit).
  78  */
  79 static void kvm_flush_dcache_pte(pte_t pte)
  80 {
  81         __kvm_flush_dcache_pte(pte);
  82 }
  83
  84 static void kvm_flush_dcache_pmd(pmd_t pmd)
  85 {
  86         __kvm_flush_dcache_pmd(pmd);
  87 }
  88
  89 static void kvm_flush_dcache_pud(pud_t pud)
  90 {
  91         __kvm_flush_dcache_pud(pud);
  92 }
  93
  94 static bool kvm_is_device_pfn(unsigned long pfn)
  95 {
  96         return !pfn_valid(pfn);
  97 }
  98
  99 /**
 100  * stage2_dissolve_pmd() - clear and flush huge PMD entry
 101  * @kvm:        pointer to kvm structure.
 102  * @addr:       IPA
 103  * @pmd:        pmd pointer for IPA
 104  *
 105  * Function clears a PMD entry, flushes addr 1st and 2nd stage TLBs. Marks all
 106  * pages in the range dirty.
 107  */
 108 static void stage2_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd)
 109 {
 110         if (!pmd_thp_or_huge(*pmd))
 111                 return;
 112
 113         pmd_clear(pmd);
 114         kvm_tlb_flush_vmid_ipa(kvm, addr);
 115         put_page(virt_to_page(pmd));
 116 }
 117
 118 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
 119                                   int min, int max)
 120 {
 121         void *page;
 122
 123         BUG_ON(max > KVM_NR_MEM_OBJS);
 124         if (cache->nobjs >= min)
 125                 return 0;
 126         while (cache->nobjs < max) {
 127                 page = (void *)__get_free_page(PGALLOC_GFP);
 128                 if (!page)
 129                         return -ENOMEM;
 130                 cache->objects[cache->nobjs++] = page;
 131         }
 132         return 0;
 133 }
 134
 135 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
 136 {
 137         while (mc->nobjs)
 138                 free_page((unsigned long)mc->objects[--mc->nobjs]);
 139 }
 140
 141 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
 142 {
 143         void *p;
 144
 145         BUG_ON(!mc || !mc->nobjs);
 146         p = mc->objects[--mc->nobjs];
 147         return p;
 148 }
 149
 150 static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)
 151 {
 152         pud_t *pud_table __maybe_unused = stage2_pud_offset(kvm, pgd, 0UL);
 153         stage2_pgd_clear(kvm, pgd);
 154         kvm_tlb_flush_vmid_ipa(kvm, addr);
 155         stage2_pud_free(kvm, pud_table);
 156         put_page(virt_to_page(pgd));
 157 }
 158
 159 static void clear_stage2_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)
 160 {
 161         pmd_t *pmd_table __maybe_unused = stage2_pmd_offset(kvm, pud, 0);
 162         VM_BUG_ON(stage2_pud_huge(kvm, *pud));
 163         stage2_pud_clear(kvm, pud);
 164         kvm_tlb_flush_vmid_ipa(kvm, addr);
 165         stage2_pmd_free(kvm, pmd_table);
 166         put_page(virt_to_page(pud));
 167 }
 168
 169 static void clear_stage2_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr)
 170 {
 171         pte_t *pte_table = pte_offset_kernel(pmd, 0);
 172         VM_BUG_ON(pmd_thp_or_huge(*pmd));
 173         pmd_clear(pmd);
 174         kvm_tlb_flush_vmid_ipa(kvm, addr);
 175         pte_free_kernel(NULL, pte_table);
 176         put_page(virt_to_page(pmd));
 177 }
 178
 179 static inline void kvm_set_pte(pte_t *ptep, pte_t new_pte)
 180 {
 181         WRITE_ONCE(*ptep, new_pte);
 182         dsb(ishst);
 183 }
 184
 185 static inline void kvm_set_pmd(pmd_t *pmdp, pmd_t new_pmd)
 186 {
 187         WRITE_ONCE(*pmdp, new_pmd);
 188         dsb(ishst);
 189 }
 190
 191 static inline void kvm_pmd_populate(pmd_t *pmdp, pte_t *ptep)
 192 {
 193         kvm_set_pmd(pmdp, kvm_mk_pmd(ptep));
 194 }
 195
 196 static inline void kvm_pud_populate(pud_t *pudp, pmd_t *pmdp)
 197 {
 198         WRITE_ONCE(*pudp, kvm_mk_pud(pmdp));
 199         dsb(ishst);
 200 }
 201
 202 static inline void kvm_pgd_populate(pgd_t *pgdp, pud_t *pudp)
 203 {
 204         WRITE_ONCE(*pgdp, kvm_mk_pgd(pudp));
 205         dsb(ishst);
 206 }
 207
 208 /*
 209  * Unmapping vs dcache management:
 210  *
 211  * If a guest maps certain memory pages as uncached, all writes will
 212  * bypass the data cache and go directly to RAM.  However, the CPUs
 213  * can still speculate reads (not writes) and fill cache lines with
 214  * data.
 215  *
 216  * Those cache lines will be *clean* cache lines though, so a
 217  * clean+invalidate operation is equivalent to an invalidate
 218  * operation, because no cache lines are marked dirty.
 219  *
 220  * Those clean cache lines could be filled prior to an uncached write
 221  * by the guest, and the cache coherent IO subsystem would therefore
 222  * end up writing old data to disk.
 223  *
 224  * This is why right after unmapping a page/section and invalidating
 225  * the corresponding TLBs, we call kvm_flush_dcache_p*() to make sure
 226  * the IO subsystem will never hit in the cache.
 227  *
 228  * This is all avoided on systems that have ARM64_HAS_STAGE2_FWB, as
 229  * we then fully enforce cacheability of RAM, no matter what the guest
 230  * does.
 231  */
 232 static void unmap_stage2_ptes(struct kvm *kvm, pmd_t *pmd,
 233                        phys_addr_t addr, phys_addr_t end)
 234 {
 235         phys_addr_t start_addr = addr;
 236         pte_t *pte, *start_pte;
 237
 238         start_pte = pte = pte_offset_kernel(pmd, addr);
 239         do {
 240                 if (!pte_none(*pte)) {
 241                         pte_t old_pte = *pte;
 242
 243                         kvm_set_pte(pte, __pte(0));
 244                         kvm_tlb_flush_vmid_ipa(kvm, addr);
 245
 246                         /* No need to invalidate the cache for device mappings */
 247                         if (!kvm_is_device_pfn(pte_pfn(old_pte)))
 248                                 kvm_flush_dcache_pte(old_pte);
 249
 250                         put_page(virt_to_page(pte));
 251                 }
 252         } while (pte++, addr += PAGE_SIZE, addr != end);
 253
 254         if (stage2_pte_table_empty(kvm, start_pte))
 255                 clear_stage2_pmd_entry(kvm, pmd, start_addr);
 256 }
 257
 258 static void unmap_stage2_pmds(struct kvm *kvm, pud_t *pud,
 259                        phys_addr_t addr, phys_addr_t end)
 260 {
 261         phys_addr_t next, start_addr = addr;
 262         pmd_t *pmd, *start_pmd;
 263
 264         start_pmd = pmd = stage2_pmd_offset(kvm, pud, addr);
 265         do {
 266                 next = stage2_pmd_addr_end(kvm, addr, end);
 267                 if (!pmd_none(*pmd)) {
 268                         if (pmd_thp_or_huge(*pmd)) {
 269                                 pmd_t old_pmd = *pmd;
 270
 271                                 pmd_clear(pmd);
 272                                 kvm_tlb_flush_vmid_ipa(kvm, addr);
 273
 274                                 kvm_flush_dcache_pmd(old_pmd);
 275
 276                                 put_page(virt_to_page(pmd));
 277                         } else {
 278                                 unmap_stage2_ptes(kvm, pmd, addr, next);
 279                         }
 280                 }
 281         } while (pmd++, addr = next, addr != end);
 282
 283         if (stage2_pmd_table_empty(kvm, start_pmd))
 284                 clear_stage2_pud_entry(kvm, pud, start_addr);
 285 }
 286
 287 static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,
 288                        phys_addr_t addr, phys_addr_t end)
 289 {
 290         phys_addr_t next, start_addr = addr;
 291         pud_t *pud, *start_pud;
 292
 293         start_pud = pud = stage2_pud_offset(kvm, pgd, addr);
 294         do {
 295                 next = stage2_pud_addr_end(kvm, addr, end);
 296                 if (!stage2_pud_none(kvm, *pud)) {
 297                         if (stage2_pud_huge(kvm, *pud)) {
 298                                 pud_t old_pud = *pud;
 299
 300                                 stage2_pud_clear(kvm, pud);
 301                                 kvm_tlb_flush_vmid_ipa(kvm, addr);
 302                                 kvm_flush_dcache_pud(old_pud);
 303                                 put_page(virt_to_page(pud));
 304                         } else {
 305                                 unmap_stage2_pmds(kvm, pud, addr, next);
 306                         }
 307                 }
 308         } while (pud++, addr = next, addr != end);
 309
 310         if (stage2_pud_table_empty(kvm, start_pud))
 311                 clear_stage2_pgd_entry(kvm, pgd, start_addr);
 312 }
 313
 314 /**
 315  * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
 316  * @kvm:   The VM pointer
 317  * @start: The intermediate physical base address of the range to unmap
 318  * @size:  The size of the area to unmap
 319  *
 320  * Clear a range of stage-2 mappings, lowering the various ref-counts.  Must
 321  * be called while holding mmu_lock (unless for freeing the stage2 pgd before
 322  * destroying the VM), otherwise another faulting VCPU may come in and mess
 323  * with things behind our backs.
 324  */
 325 static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
 326 {
 327         pgd_t *pgd;
 328         phys_addr_t addr = start, end = start + size;
 329         phys_addr_t next;
 330
 331         assert_spin_locked(&kvm->mmu_lock);
 332         WARN_ON(size & ~PAGE_MASK);
 333
 334         pgd = kvm->arch.pgd + stage2_pgd_index(kvm, addr);
 335         do {
 336                 /*
 337                  * Make sure the page table is still active, as another thread
 338                  * could have possibly freed the page table, while we released
 339                  * the lock.
 340                  */
 341                 if (!READ_ONCE(kvm->arch.pgd))
 342                         break;
 343                 next = stage2_pgd_addr_end(kvm, addr, end);
 344                 if (!stage2_pgd_none(kvm, *pgd))
 345                         unmap_stage2_puds(kvm, pgd, addr, next);
 346                 /*
 347                  * If the range is too large, release the kvm->mmu_lock
 348                  * to prevent starvation and lockup detector warnings.
 349                  */
 350                 if (next != end)
 351                         cond_resched_lock(&kvm->mmu_lock);
 352         } while (pgd++, addr = next, addr != end);
 353 }
 354
 355 static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,
 356                               phys_addr_t addr, phys_addr_t end)
 357 {
 358         pte_t *pte;
 359
 360         pte = pte_offset_kernel(pmd, addr);
 361         do {
 362                 if (!pte_none(*pte) && !kvm_is_device_pfn(pte_pfn(*pte)))
 363                         kvm_flush_dcache_pte(*pte);
 364         } while (pte++, addr += PAGE_SIZE, addr != end);
 365 }
 366
 367 static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,
 368                               phys_addr_t addr, phys_addr_t end)
 369 {
 370         pmd_t *pmd;
 371         phys_addr_t next;
 372
 373         pmd = stage2_pmd_offset(kvm, pud, addr);
 374         do {
 375                 next = stage2_pmd_addr_end(kvm, addr, end);
 376                 if (!pmd_none(*pmd)) {
 377                         if (pmd_thp_or_huge(*pmd))
 378                                 kvm_flush_dcache_pmd(*pmd);
 379                         else
 380                                 stage2_flush_ptes(kvm, pmd, addr, next);
 381                 }
 382         } while (pmd++, addr = next, addr != end);
 383 }
 384
 385 static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
 386                               phys_addr_t addr, phys_addr_t end)
 387 {
 388         pud_t *pud;
 389         phys_addr_t next;
 390
 391         pud = stage2_pud_offset(kvm, pgd, addr);
 392         do {
 393                 next = stage2_pud_addr_end(kvm, addr, end);
 394                 if (!stage2_pud_none(kvm, *pud)) {
 395                         if (stage2_pud_huge(kvm, *pud))
 396                                 kvm_flush_dcache_pud(*pud);
 397                         else
 398                                 stage2_flush_pmds(kvm, pud, addr, next);
 399                 }
 400         } while (pud++, addr = next, addr != end);
 401 }
 402
 403 static void stage2_flush_memslot(struct kvm *kvm,
 404                                  struct kvm_memory_slot *memslot)
 405 {
 406         phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
 407         phys_addr_t end = addr + PAGE_SIZE * memslot->npages;
 408         phys_addr_t next;
 409         pgd_t *pgd;
 410
 411         pgd = kvm->arch.pgd + stage2_pgd_index(kvm, addr);
 412         do {
 413                 next = stage2_pgd_addr_end(kvm, addr, end);
 414                 if (!stage2_pgd_none(kvm, *pgd))
 415                         stage2_flush_puds(kvm, pgd, addr, next);
 416         } while (pgd++, addr = next, addr != end);
 417 }
 418
 419 /**
 420  * stage2_flush_vm - Invalidate cache for pages mapped in stage 2
 421  * @kvm: The struct kvm pointer
 422  *
 423  * Go through the stage 2 page tables and invalidate any cache lines
 424  * backing memory already mapped to the VM.
 425  */
 426 static void stage2_flush_vm(struct kvm *kvm)
 427 {
 428         struct kvm_memslots *slots;
 429         struct kvm_memory_slot *memslot;
 430         int idx;
 431
 432         idx = srcu_read_lock(&kvm->srcu);
 433         spin_lock(&kvm->mmu_lock);
 434
 435         slots = kvm_memslots(kvm);
 436         kvm_for_each_memslot(memslot, slots)
 437                 stage2_flush_memslot(kvm, memslot);
 438
 439         spin_unlock(&kvm->mmu_lock);
 440         srcu_read_unlock(&kvm->srcu, idx);
 441 }
 442
 443 static void clear_hyp_pgd_entry(pgd_t *pgd)
 444 {
 445         pud_t *pud_table __maybe_unused = pud_offset(pgd, 0UL);
 446         pgd_clear(pgd);
 447         pud_free(NULL, pud_table);
 448         put_page(virt_to_page(pgd));
 449 }
 450
 451 static void clear_hyp_pud_entry(pud_t *pud)
 452 {
 453         pmd_t *pmd_table __maybe_unused = pmd_offset(pud, 0);
 454         VM_BUG_ON(pud_huge(*pud));
 455         pud_clear(pud);
 456         pmd_free(NULL, pmd_table);
 457         put_page(virt_to_page(pud));
 458 }
 459
 460 static void clear_hyp_pmd_entry(pmd_t *pmd)
 461 {
 462         pte_t *pte_table = pte_offset_kernel(pmd, 0);
 463         VM_BUG_ON(pmd_thp_or_huge(*pmd));
 464         pmd_clear(pmd);
 465         pte_free_kernel(NULL, pte_table);
 466         put_page(virt_to_page(pmd));
 467 }
 468
 469 static void unmap_hyp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end)
 470 {
 471         pte_t *pte, *start_pte;
 472
 473         start_pte = pte = pte_offset_kernel(pmd, addr);
 474         do {
 475                 if (!pte_none(*pte)) {
 476                         kvm_set_pte(pte, __pte(0));
 477                         put_page(virt_to_page(pte));
 478                 }
 479         } while (pte++, addr += PAGE_SIZE, addr != end);
 480
 481         if (hyp_pte_table_empty(start_pte))
 482                 clear_hyp_pmd_entry(pmd);
 483 }
 484
 485 static void unmap_hyp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end)
 486 {
 487         phys_addr_t next;
 488         pmd_t *pmd, *start_pmd;
 489
 490         start_pmd = pmd = pmd_offset(pud, addr);
 491         do {
 492                 next = pmd_addr_end(addr, end);
 493                 /* Hyp doesn't use huge pmds */
 494                 if (!pmd_none(*pmd))
 495                         unmap_hyp_ptes(pmd, addr, next);
 496         } while (pmd++, addr = next, addr != end);
 497
 498         if (hyp_pmd_table_empty(start_pmd))
 499                 clear_hyp_pud_entry(pud);
 500 }
 501
 502 static void unmap_hyp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
 503 {
 504         phys_addr_t next;
 505         pud_t *pud, *start_pud;
 506
 507         start_pud = pud = pud_offset(pgd, addr);
 508         do {
 509                 next = pud_addr_end(addr, end);
 510                 /* Hyp doesn't use huge puds */
 511                 if (!pud_none(*pud))
 512                         unmap_hyp_pmds(pud, addr, next);
 513         } while (pud++, addr = next, addr != end);
 514
 515         if (hyp_pud_table_empty(start_pud))
 516                 clear_hyp_pgd_entry(pgd);
 517 }
 518
 519 static unsigned int kvm_pgd_index(unsigned long addr, unsigned int ptrs_per_pgd)
 520 {
 521         return (addr >> PGDIR_SHIFT) & (ptrs_per_pgd - 1);
 522 }
 523
 524 static void __unmap_hyp_range(pgd_t *pgdp, unsigned long ptrs_per_pgd,
 525                               phys_addr_t start, u64 size)
 526 {
 527         pgd_t *pgd;
 528         phys_addr_t addr = start, end = start + size;
 529         phys_addr_t next;
 530
 531         /*
 532          * We don't unmap anything from HYP, except at the hyp tear down.
 533          * Hence, we don't have to invalidate the TLBs here.
 534          */
 535         pgd = pgdp + kvm_pgd_index(addr, ptrs_per_pgd);
 536         do {
 537                 next = pgd_addr_end(addr, end);
 538                 if (!pgd_none(*pgd))
 539                         unmap_hyp_puds(pgd, addr, next);
 540         } while (pgd++, addr = next, addr != end);
 541 }
 542
 543 static void unmap_hyp_range(pgd_t *pgdp, phys_addr_t start, u64 size)
 544 {
 545         __unmap_hyp_range(pgdp, PTRS_PER_PGD, start, size);
 546 }
 547
 548 static void unmap_hyp_idmap_range(pgd_t *pgdp, phys_addr_t start, u64 size)
 549 {
 550         __unmap_hyp_range(pgdp, __kvm_idmap_ptrs_per_pgd(), start, size);
 551 }
 552
 553 /**
 554  * free_hyp_pgds - free Hyp-mode page tables
 555  *
 556  * Assumes hyp_pgd is a page table used strictly in Hyp-mode and
 557  * therefore contains either mappings in the kernel memory area (above
 558  * PAGE_OFFSET), or device mappings in the idmap range.
 559  *
 560  * boot_hyp_pgd should only map the idmap range, and is only used in
 561  * the extended idmap case.
 562  */
 563 void free_hyp_pgds(void)
 564 {
 565         pgd_t *id_pgd;
 566
 567         mutex_lock(&kvm_hyp_pgd_mutex);
 568
 569         id_pgd = boot_hyp_pgd ? boot_hyp_pgd : hyp_pgd;
 570
 571         if (id_pgd) {
 572                 /* In case we never called hyp_mmu_init() */
 573                 if (!io_map_base)
 574                         io_map_base = hyp_idmap_start;
 575                 unmap_hyp_idmap_range(id_pgd, io_map_base,
 576                                       hyp_idmap_start + PAGE_SIZE - io_map_base);
 577         }
 578
 579         if (boot_hyp_pgd) {
 580                 free_pages((unsigned long)boot_hyp_pgd, hyp_pgd_order);
 581                 boot_hyp_pgd = NULL;
 582         }
 583
 584         if (hyp_pgd) {
 585                 unmap_hyp_range(hyp_pgd, kern_hyp_va(PAGE_OFFSET),
 586                                 (uintptr_t)high_memory - PAGE_OFFSET);
 587
 588                 free_pages((unsigned long)hyp_pgd, hyp_pgd_order);
 589                 hyp_pgd = NULL;
 590         }
 591         if (merged_hyp_pgd) {
 592                 clear_page(merged_hyp_pgd);
 593                 free_page((unsigned long)merged_hyp_pgd);
 594                 merged_hyp_pgd = NULL;
 595         }
 596
 597         mutex_unlock(&kvm_hyp_pgd_mutex);
 598 }
 599
 600 static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start,
 601                                     unsigned long end, unsigned long pfn,
 602                                     pgprot_t prot)
 603 {
 604         pte_t *pte;
 605         unsigned long addr;
 606
 607         addr = start;
 608         do {
 609                 pte = pte_offset_kernel(pmd, addr);
 610                 kvm_set_pte(pte, pfn_pte(pfn, prot));
 611                 get_page(virt_to_page(pte));
 612                 pfn++;
 613         } while (addr += PAGE_SIZE, addr != end);
 614 }
 615
 616 static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start,
 617                                    unsigned long end, unsigned long pfn,
 618                                    pgprot_t prot)
 619 {
 620         pmd_t *pmd;
 621         pte_t *pte;
 622         unsigned long addr, next;
 623
 624         addr = start;
 625         do {
 626                 pmd = pmd_offset(pud, addr);
 627
 628                 BUG_ON(pmd_sect(*pmd));
 629
 630                 if (pmd_none(*pmd)) {
 631                         pte = pte_alloc_one_kernel(NULL, addr);
 632                         if (!pte) {
 633                                 kvm_err("Cannot allocate Hyp pte\n");
 634                                 return -ENOMEM;
 635                         }
 636                         kvm_pmd_populate(pmd, pte);
 637                         get_page(virt_to_page(pmd));
 638                 }
 639
 640                 next = pmd_addr_end(addr, end);
 641
 642                 create_hyp_pte_mappings(pmd, addr, next, pfn, prot);
 643                 pfn += (next - addr) >> PAGE_SHIFT;
 644         } while (addr = next, addr != end);
 645
 646         return 0;
 647 }
 648
 649 static int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start,
 650                                    unsigned long end, unsigned long pfn,
 651                                    pgprot_t prot)
 652 {
 653         pud_t *pud;
 654         pmd_t *pmd;
 655         unsigned long addr, next;
 656         int ret;
 657
 658         addr = start;
 659         do {
 660                 pud = pud_offset(pgd, addr);
 661
 662                 if (pud_none_or_clear_bad(pud)) {
 663                         pmd = pmd_alloc_one(NULL, addr);
 664                         if (!pmd) {
 665                                 kvm_err("Cannot allocate Hyp pmd\n");
 666                                 return -ENOMEM;
 667                         }
 668                         kvm_pud_populate(pud, pmd);
 669                         get_page(virt_to_page(pud));
 670                 }
 671
 672                 next = pud_addr_end(addr, end);
 673                 ret = create_hyp_pmd_mappings(pud, addr, next, pfn, prot);
 674                 if (ret)
 675                         return ret;
 676                 pfn += (next - addr) >> PAGE_SHIFT;
 677         } while (addr = next, addr != end);
 678
 679         return 0;
 680 }
 681
 682 static int __create_hyp_mappings(pgd_t *pgdp, unsigned long ptrs_per_pgd,
 683                                  unsigned long start, unsigned long end,
 684                                  unsigned long pfn, pgprot_t prot)
 685 {
 686         pgd_t *pgd;
 687         pud_t *pud;
 688         unsigned long addr, next;
 689         int err = 0;
 690
 691         mutex_lock(&kvm_hyp_pgd_mutex);
 692         addr = start & PAGE_MASK;
 693         end = PAGE_ALIGN(end);
 694         do {
 695                 pgd = pgdp + kvm_pgd_index(addr, ptrs_per_pgd);
 696
 697                 if (pgd_none(*pgd)) {
 698                         pud = pud_alloc_one(NULL, addr);
 699                         if (!pud) {
 700                                 kvm_err("Cannot allocate Hyp pud\n");
 701                                 err = -ENOMEM;
 702                                 goto out;
 703                         }
 704                         kvm_pgd_populate(pgd, pud);
 705                         get_page(virt_to_page(pgd));
 706                 }
 707
 708                 next = pgd_addr_end(addr, end);
 709                 err = create_hyp_pud_mappings(pgd, addr, next, pfn, prot);
 710                 if (err)
 711                         goto out;
 712                 pfn += (next - addr) >> PAGE_SHIFT;
 713         } while (addr = next, addr != end);
 714 out:
 715         mutex_unlock(&kvm_hyp_pgd_mutex);
 716         return err;
 717 }
 718
 719 static phys_addr_t kvm_kaddr_to_phys(void *kaddr)
 720 {
 721         if (!is_vmalloc_addr(kaddr)) {
 722                 BUG_ON(!virt_addr_valid(kaddr));
 723                 return __pa(kaddr);
 724         } else {
 725                 return page_to_phys(vmalloc_to_page(kaddr)) +
 726                        offset_in_page(kaddr);
 727         }
 728 }
 729
 730 /**
 731  * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
 732  * @from:       The virtual kernel start address of the range
 733  * @to:         The virtual kernel end address of the range (exclusive)
 734  * @prot:       The protection to be applied to this range
 735  *
 736  * The same virtual address as the kernel virtual address is also used
 737  * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying
 738  * physical pages.
 739  */
 740 int create_hyp_mappings(void *from, void *to, pgprot_t prot)
 741 {
 742         phys_addr_t phys_addr;
 743         unsigned long virt_addr;
 744         unsigned long start = kern_hyp_va((unsigned long)from);
 745         unsigned long end = kern_hyp_va((unsigned long)to);
 746
 747         if (is_kernel_in_hyp_mode())
 748                 return 0;
 749
 750         start = start & PAGE_MASK;
 751         end = PAGE_ALIGN(end);
 752
 753         for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
 754                 int err;
 755
 756                 phys_addr = kvm_kaddr_to_phys(from + virt_addr - start);
 757                 err = __create_hyp_mappings(hyp_pgd, PTRS_PER_PGD,
 758                                             virt_addr, virt_addr + PAGE_SIZE,
 759                                             __phys_to_pfn(phys_addr),
 760                                             prot);
 761                 if (err)
 762                         return err;
 763         }
 764
 765         return 0;
 766 }
 767
 768 static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size,
 769                                         unsigned long *haddr, pgprot_t prot)
 770 {
 771         pgd_t *pgd = hyp_pgd;
 772         unsigned long base;
 773         int ret = 0;
 774
 775         mutex_lock(&kvm_hyp_pgd_mutex);
 776
 777         /*
 778          * This assumes that we we have enough space below the idmap
 779          * page to allocate our VAs. If not, the check below will
 780          * kick. A potential alternative would be to detect that
 781          * overflow and switch to an allocation above the idmap.
 782          *
 783          * The allocated size is always a multiple of PAGE_SIZE.
 784          */
 785         size = PAGE_ALIGN(size + offset_in_page(phys_addr));
 786         base = io_map_base - size;
 787
 788         /*
 789          * Verify that BIT(VA_BITS - 1) hasn't been flipped by
 790          * allocating the new area, as it would indicate we've
 791          * overflowed the idmap/IO address range.
 792          */
 793         if ((base ^ io_map_base) & BIT(VA_BITS - 1))
 794                 ret = -ENOMEM;
 795         else
 796                 io_map_base = base;
 797
 798         mutex_unlock(&kvm_hyp_pgd_mutex);
 799
 800         if (ret)
 801                 goto out;
 802
 803         if (__kvm_cpu_uses_extended_idmap())
 804                 pgd = boot_hyp_pgd;
 805
 806         ret = __create_hyp_mappings(pgd, __kvm_idmap_ptrs_per_pgd(),
 807                                     base, base + size,
 808                                     __phys_to_pfn(phys_addr), prot);
 809         if (ret)
 810                 goto out;
 811
 812         *haddr = base + offset_in_page(phys_addr);
 813
 814 out:
 815         return ret;
 816 }
 817
 818 /**
 819  * create_hyp_io_mappings - Map IO into both kernel and HYP
 820  * @phys_addr:  The physical start address which gets mapped
 821  * @size:       Size of the region being mapped
 822  * @kaddr:      Kernel VA for this mapping
 823  * @haddr:      HYP VA for this mapping
 824  */
 825 int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size,
 826                            void __iomem **kaddr,
 827                            void __iomem **haddr)
 828 {
 829         unsigned long addr;
 830         int ret;
 831
 832         *kaddr = ioremap(phys_addr, size);
 833         if (!*kaddr)
 834                 return -ENOMEM;
 835
 836         if (is_kernel_in_hyp_mode()) {
 837                 *haddr = *kaddr;
 838                 return 0;
 839         }
 840
 841         ret = __create_hyp_private_mapping(phys_addr, size,
 842                                            &addr, PAGE_HYP_DEVICE);
 843         if (ret) {
 844                 iounmap(*kaddr);
 845                 *kaddr = NULL;
 846                 *haddr = NULL;
 847                 return ret;
 848         }
 849
 850         *haddr = (void __iomem *)addr;
 851         return 0;
 852 }
 853
 854 /**
 855  * create_hyp_exec_mappings - Map an executable range into HYP
 856  * @phys_addr:  The physical start address which gets mapped
 857  * @size:       Size of the region being mapped
 858  * @haddr:      HYP VA for this mapping
 859  */
 860 int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size,
 861                              void **haddr)
 862 {
 863         unsigned long addr;
 864         int ret;
 865
 866         BUG_ON(is_kernel_in_hyp_mode());
 867
 868         ret = __create_hyp_private_mapping(phys_addr, size,
 869                                            &addr, PAGE_HYP_EXEC);
 870         if (ret) {
 871                 *haddr = NULL;
 872                 return ret;
 873         }
 874
 875         *haddr = (void *)addr;
 876         return 0;
 877 }
 878
 879 /**
 880  * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
 881  * @kvm:        The KVM struct pointer for the VM.
 882  *
 883  * Allocates only the stage-2 HW PGD level table(s) (can support either full
 884  * 40-bit input addresses or limited to 32-bit input addresses). Clears the
 885  * allocated pages.
 886  *
 887  * Note we don't need locking here as this is only called when the VM is
 888  * created, which can only be done once.
 889  */
 890 int kvm_alloc_stage2_pgd(struct kvm *kvm)
 891 {
 892         pgd_t *pgd;
 893
 894         if (kvm->arch.pgd != NULL) {
 895                 kvm_err("kvm_arch already initialized?\n");
 896                 return -EINVAL;
 897         }
 898
 899         /* Allocate the HW PGD, making sure that each page gets its own refcount */
 900         pgd = alloc_pages_exact(stage2_pgd_size(kvm), GFP_KERNEL | __GFP_ZERO);
 901         if (!pgd)
 902                 return -ENOMEM;
 903
 904         kvm->arch.pgd = pgd;
 905         return 0;
 906 }
 907
 908 static void stage2_unmap_memslot(struct kvm *kvm,
 909                                  struct kvm_memory_slot *memslot)
 910 {
 911         hva_t hva = memslot->userspace_addr;
 912         phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
 913         phys_addr_t size = PAGE_SIZE * memslot->npages;
 914         hva_t reg_end = hva + size;
 915
 916         /*
 917          * A memory region could potentially cover multiple VMAs, and any holes
 918          * between them, so iterate over all of them to find out if we should
 919          * unmap any of them.
 920          *
 921          *     +--------------------------------------------+
 922          * +---------------+----------------+   +----------------+
 923          * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
 924          * +---------------+----------------+   +----------------+
 925          *     |               memory region                |
 926          *     +--------------------------------------------+
 927          */
 928         do {
 929                 struct vm_area_struct *vma = find_vma(current->mm, hva);
 930                 hva_t vm_start, vm_end;
 931
 932                 if (!vma || vma->vm_start >= reg_end)
 933                         break;
 934
 935                 /*
 936                  * Take the intersection of this VMA with the memory region
 937                  */
 938                 vm_start = max(hva, vma->vm_start);
 939                 vm_end = min(reg_end, vma->vm_end);
 940
 941                 if (!(vma->vm_flags & VM_PFNMAP)) {
 942                         gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
 943                         unmap_stage2_range(kvm, gpa, vm_end - vm_start);
 944                 }
 945                 hva = vm_end;
 946         } while (hva < reg_end);
 947 }
 948
 949 /**
 950  * stage2_unmap_vm - Unmap Stage-2 RAM mappings
 951  * @kvm: The struct kvm pointer
 952  *
 953  * Go through the memregions and unmap any reguler RAM
 954  * backing memory already mapped to the VM.
 955  */
 956 void stage2_unmap_vm(struct kvm *kvm)
 957 {
 958         struct kvm_memslots *slots;
 959         struct kvm_memory_slot *memslot;
 960         int idx;
 961
 962         idx = srcu_read_lock(&kvm->srcu);
 963         down_read(&current->mm->mmap_sem);
 964         spin_lock(&kvm->mmu_lock);
 965
 966         slots = kvm_memslots(kvm);
 967         kvm_for_each_memslot(memslot, slots)
 968                 stage2_unmap_memslot(kvm, memslot);
 969
 970         spin_unlock(&kvm->mmu_lock);
 971         up_read(&current->mm->mmap_sem);
 972         srcu_read_unlock(&kvm->srcu, idx);
 973 }
 974
 975 /**
 976  * kvm_free_stage2_pgd - free all stage-2 tables
 977  * @kvm:        The KVM struct pointer for the VM.
 978  *
 979  * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all
 980  * underlying level-2 and level-3 tables before freeing the actual level-1 table
 981  * and setting the struct pointer to NULL.
 982  */
 983 void kvm_free_stage2_pgd(struct kvm *kvm)
 984 {
 985         void *pgd = NULL;
 986
 987         spin_lock(&kvm->mmu_lock);
 988         if (kvm->arch.pgd) {
 989                 unmap_stage2_range(kvm, 0, kvm_phys_size(kvm));
 990                 pgd = READ_ONCE(kvm->arch.pgd);
 991                 kvm->arch.pgd = NULL;
 992         }
 993         spin_unlock(&kvm->mmu_lock);
 994
 995         /* Free the HW pgd, one page at a time */
 996         if (pgd)
 997                 free_pages_exact(pgd, stage2_pgd_size(kvm));
 998 }
 999
1000 static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
1001                              phys_addr_t addr)
1002 {
1003         pgd_t *pgd;
1004         pud_t *pud;
1005
1006         pgd = kvm->arch.pgd + stage2_pgd_index(kvm, addr);
1007         if (stage2_pgd_none(kvm, *pgd)) {
1008                 if (!cache)
1009                         return NULL;
1010                 pud = mmu_memory_cache_alloc(cache);
1011                 stage2_pgd_populate(kvm, pgd, pud);
1012                 get_page(virt_to_page(pgd));
1013         }
1014
1015         return stage2_pud_offset(kvm, pgd, addr);
1016 }
1017
1018 static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
1019                              phys_addr_t addr)
1020 {
1021         pud_t *pud;
1022         pmd_t *pmd;
1023
1024         pud = stage2_get_pud(kvm, cache, addr);
1025         if (!pud)
1026                 return NULL;
1027
1028         if (stage2_pud_none(kvm, *pud)) {
1029                 if (!cache)
1030                         return NULL;
1031                 pmd = mmu_memory_cache_alloc(cache);
1032                 stage2_pud_populate(kvm, pud, pmd);
1033                 get_page(virt_to_page(pud));
1034         }
1035
1036         return stage2_pmd_offset(kvm, pud, addr);
1037 }
1038
1039 static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
1040                                *cache, phys_addr_t addr, const pmd_t *new_pmd)
1041 {
1042         pmd_t *pmd, old_pmd;
1043
1044         pmd = stage2_get_pmd(kvm, cache, addr);
1045         VM_BUG_ON(!pmd);
1046
1047         old_pmd = *pmd;
1048         if (pmd_present(old_pmd)) {
1049                 /*
1050                  * Multiple vcpus faulting on the same PMD entry, can
1051                  * lead to them sequentially updating the PMD with the
1052                  * same value. Following the break-before-make
1053                  * (pmd_clear() followed by tlb_flush()) process can
1054                  * hinder forward progress due to refaults generated
1055                  * on missing translations.
1056                  *
1057                  * Skip updating the page table if the entry is
1058                  * unchanged.
1059                  */
1060                 if (pmd_val(old_pmd) == pmd_val(*new_pmd))
1061                         return 0;
1062
1063                 /*
1064                  * Mapping in huge pages should only happen through a
1065                  * fault.  If a page is merged into a transparent huge
1066                  * page, the individual subpages of that huge page
1067                  * should be unmapped through MMU notifiers before we
1068                  * get here.
1069                  *
1070                  * Merging of CompoundPages is not supported; they
1071                  * should become splitting first, unmapped, merged,
1072                  * and mapped back in on-demand.
1073                  */
1074                 VM_BUG_ON(pmd_pfn(old_pmd) != pmd_pfn(*new_pmd));
1075
1076                 pmd_clear(pmd);
1077                 kvm_tlb_flush_vmid_ipa(kvm, addr);
1078         } else {
1079                 get_page(virt_to_page(pmd));
1080         }
1081
1082         kvm_set_pmd(pmd, *new_pmd);
1083         return 0;
1084 }
1085
1086 static bool stage2_is_exec(struct kvm *kvm, phys_addr_t addr)
1087 {
1088         pmd_t *pmdp;
1089         pte_t *ptep;
1090
1091         pmdp = stage2_get_pmd(kvm, NULL, addr);
1092         if (!pmdp || pmd_none(*pmdp) || !pmd_present(*pmdp))
1093                 return false;
1094
1095         if (pmd_thp_or_huge(*pmdp))
1096                 return kvm_s2pmd_exec(pmdp);
1097
1098         ptep = pte_offset_kernel(pmdp, addr);
1099         if (!ptep || pte_none(*ptep) || !pte_present(*ptep))
1100                 return false;
1101
1102         return kvm_s2pte_exec(ptep);
1103 }
1104
1105 static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
1106                           phys_addr_t addr, const pte_t *new_pte,
1107                           unsigned long flags)
1108 {
1109         pmd_t *pmd;
1110         pte_t *pte, old_pte;
1111         bool iomap = flags & KVM_S2PTE_FLAG_IS_IOMAP;
1112         bool logging_active = flags & KVM_S2_FLAG_LOGGING_ACTIVE;
1113
1114         VM_BUG_ON(logging_active && !cache);
1115
1116         /* Create stage-2 page table mapping - Levels 0 and 1 */
1117         pmd = stage2_get_pmd(kvm, cache, addr);
1118         if (!pmd) {
1119                 /*
1120                  * Ignore calls from kvm_set_spte_hva for unallocated
1121                  * address ranges.
1122                  */
1123                 return 0;
1124         }
1125
1126         /*
1127          * While dirty page logging - dissolve huge PMD, then continue on to
1128          * allocate page.
1129          */
1130         if (logging_active)
1131                 stage2_dissolve_pmd(kvm, addr, pmd);
1132
1133         /* Create stage-2 page mappings - Level 2 */
1134         if (pmd_none(*pmd)) {
1135                 if (!cache)
1136                         return 0; /* ignore calls from kvm_set_spte_hva */
1137                 pte = mmu_memory_cache_alloc(cache);
1138                 kvm_pmd_populate(pmd, pte);
1139                 get_page(virt_to_page(pmd));
1140         }
1141
1142         pte = pte_offset_kernel(pmd, addr);
1143
1144         if (iomap && pte_present(*pte))
1145                 return -EFAULT;
1146
1147         /* Create 2nd stage page table mapping - Level 3 */
1148         old_pte = *pte;
1149         if (pte_present(old_pte)) {
1150                 /* Skip page table update if there is no change */
1151                 if (pte_val(old_pte) == pte_val(*new_pte))
1152                         return 0;
1153
1154                 kvm_set_pte(pte, __pte(0));
1155                 kvm_tlb_flush_vmid_ipa(kvm, addr);
1156         } else {
1157                 get_page(virt_to_page(pte));
1158         }
1159
1160         kvm_set_pte(pte, *new_pte);
1161         return 0;
1162 }
1163
1164 #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1165 static int stage2_ptep_test_and_clear_young(pte_t *pte)
1166 {
1167         if (pte_young(*pte)) {
1168                 *pte = pte_mkold(*pte);
1169                 return 1;
1170         }
1171         return 0;
1172 }
1173 #else
1174 static int stage2_ptep_test_and_clear_young(pte_t *pte)
1175 {
1176         return __ptep_test_and_clear_young(pte);
1177 }
1178 #endif
1179
1180 static int stage2_pmdp_test_and_clear_young(pmd_t *pmd)
1181 {
1182         return stage2_ptep_test_and_clear_young((pte_t *)pmd);
1183 }
1184
1185 /**
1186  * kvm_phys_addr_ioremap - map a device range to guest IPA
1187  *
1188  * @kvm:        The KVM pointer
1189  * @guest_ipa:  The IPA at which to insert the mapping
1190  * @pa:         The physical address of the device
1191  * @size:       The size of the mapping
1192  */
1193 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
1194                           phys_addr_t pa, unsigned long size, bool writable)
1195 {
1196         phys_addr_t addr, end;
1197         int ret = 0;
1198         unsigned long pfn;
1199         struct kvm_mmu_memory_cache cache = { 0, };
1200
1201         end = (guest_ipa + size + PAGE_SIZE - 1) & PAGE_MASK;
1202         pfn = __phys_to_pfn(pa);
1203
1204         for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) {
1205                 pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE);
1206
1207                 if (writable)
1208                         pte = kvm_s2pte_mkwrite(pte);
1209
1210                 ret = mmu_topup_memory_cache(&cache,
1211                                              kvm_mmu_cache_min_pages(kvm),
1212                                              KVM_NR_MEM_OBJS);
1213                 if (ret)
1214                         goto out;
1215                 spin_lock(&kvm->mmu_lock);
1216                 ret = stage2_set_pte(kvm, &cache, addr, &pte,
1217                                                 KVM_S2PTE_FLAG_IS_IOMAP);
1218                 spin_unlock(&kvm->mmu_lock);
1219                 if (ret)
1220                         goto out;
1221
1222                 pfn++;
1223         }
1224
1225 out:
1226         mmu_free_memory_cache(&cache);
1227         return ret;
1228 }
1229
1230 static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, phys_addr_t *ipap)
1231 {
1232         kvm_pfn_t pfn = *pfnp;
1233         gfn_t gfn = *ipap >> PAGE_SHIFT;
1234         struct page *page = pfn_to_page(pfn);
1235
1236         /*
1237          * PageTransCompoungMap() returns true for THP and
1238          * hugetlbfs. Make sure the adjustment is done only for THP
1239          * pages.
1240          */
1241         if (!PageHuge(page) && PageTransCompoundMap(page)) {
1242                 unsigned long mask;
1243                 /*
1244                  * The address we faulted on is backed by a transparent huge
1245                  * page.  However, because we map the compound huge page and
1246                  * not the individual tail page, we need to transfer the
1247                  * refcount to the head page.  We have to be careful that the
1248                  * THP doesn't start to split while we are adjusting the
1249                  * refcounts.
1250                  *
1251                  * We are sure this doesn't happen, because mmu_notifier_retry
1252                  * was successful and we are holding the mmu_lock, so if this
1253                  * THP is trying to split, it will be blocked in the mmu
1254                  * notifier before touching any of the pages, specifically
1255                  * before being able to call __split_huge_page_refcount().
1256                  *
1257                  * We can therefore safely transfer the refcount from PG_tail
1258                  * to PG_head and switch the pfn from a tail page to the head
1259                  * page accordingly.
1260                  */
1261                 mask = PTRS_PER_PMD - 1;
1262                 VM_BUG_ON((gfn & mask) != (pfn & mask));
1263                 if (pfn & mask) {
1264                         *ipap &= PMD_MASK;
1265                         kvm_release_pfn_clean(pfn);
1266                         pfn &= ~mask;
1267                         kvm_get_pfn(pfn);
1268                         *pfnp = pfn;
1269                 }
1270
1271                 return true;
1272         }
1273
1274         return false;
1275 }
1276
1277 static bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
1278 {
1279         if (kvm_vcpu_trap_is_iabt(vcpu))
1280                 return false;
1281
1282         return kvm_vcpu_dabt_iswrite(vcpu);
1283 }
1284
1285 /**
1286  * stage2_wp_ptes - write protect PMD range
1287  * @pmd:        pointer to pmd entry
1288  * @addr:       range start address
1289  * @end:        range end address
1290  */
1291 static void stage2_wp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end)
1292 {
1293         pte_t *pte;
1294
1295         pte = pte_offset_kernel(pmd, addr);
1296         do {
1297                 if (!pte_none(*pte)) {
1298                         if (!kvm_s2pte_readonly(pte))
1299                                 kvm_set_s2pte_readonly(pte);
1300                 }
1301         } while (pte++, addr += PAGE_SIZE, addr != end);
1302 }
1303
1304 /**
1305  * stage2_wp_pmds - write protect PUD range
1306  * kvm:         kvm instance for the VM
1307  * @pud:        pointer to pud entry
1308  * @addr:       range start address
1309  * @end:        range end address
1310  */
1311 static void stage2_wp_pmds(struct kvm *kvm, pud_t *pud,
1312                            phys_addr_t addr, phys_addr_t end)
1313 {
1314         pmd_t *pmd;
1315         phys_addr_t next;
1316
1317         pmd = stage2_pmd_offset(kvm, pud, addr);
1318
1319         do {
1320                 next = stage2_pmd_addr_end(kvm, addr, end);
1321                 if (!pmd_none(*pmd)) {
1322                         if (pmd_thp_or_huge(*pmd)) {
1323                                 if (!kvm_s2pmd_readonly(pmd))
1324                                         kvm_set_s2pmd_readonly(pmd);
1325                         } else {
1326                                 stage2_wp_ptes(pmd, addr, next);
1327                         }
1328                 }
1329         } while (pmd++, addr = next, addr != end);
1330 }
1331
1332 /**
1333   * stage2_wp_puds - write protect PGD range
1334   * @pgd:       pointer to pgd entry
1335   * @addr:      range start address
1336   * @end:       range end address
1337   *
1338   * Process PUD entries, for a huge PUD we cause a panic.
1339   */
1340 static void  stage2_wp_puds(struct kvm *kvm, pgd_t *pgd,
1341                             phys_addr_t addr, phys_addr_t end)
1342 {
1343         pud_t *pud;
1344         phys_addr_t next;
1345
1346         pud = stage2_pud_offset(kvm, pgd, addr);
1347         do {
1348                 next = stage2_pud_addr_end(kvm, addr, end);
1349                 if (!stage2_pud_none(kvm, *pud)) {
1350                         /* TODO:PUD not supported, revisit later if supported */
1351                         BUG_ON(stage2_pud_huge(kvm, *pud));
1352                         stage2_wp_pmds(kvm, pud, addr, next);
1353                 }
1354         } while (pud++, addr = next, addr != end);
1355 }
1356
1357 /**
1358  * stage2_wp_range() - write protect stage2 memory region range
1359  * @kvm:        The KVM pointer
1360  * @addr:       Start address of range
1361  * @end:        End address of range
1362  */
1363 static void stage2_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
1364 {
1365         pgd_t *pgd;
1366         phys_addr_t next;
1367
1368         pgd = kvm->arch.pgd + stage2_pgd_index(kvm, addr);
1369         do {
1370                 /*
1371                  * Release kvm_mmu_lock periodically if the memory region is
1372                  * large. Otherwise, we may see kernel panics with
1373                  * CONFIG_DETECT_HUNG_TASK, CONFIG_LOCKUP_DETECTOR,
1374                  * CONFIG_LOCKDEP. Additionally, holding the lock too long
1375                  * will also starve other vCPUs. We have to also make sure
1376                  * that the page tables are not freed while we released
1377                  * the lock.
1378                  */
1379                 cond_resched_lock(&kvm->mmu_lock);
1380                 if (!READ_ONCE(kvm->arch.pgd))
1381                         break;
1382                 next = stage2_pgd_addr_end(kvm, addr, end);
1383                 if (stage2_pgd_present(kvm, *pgd))
1384                         stage2_wp_puds(kvm, pgd, addr, next);
1385         } while (pgd++, addr = next, addr != end);
1386 }
1387
1388 /**
1389  * kvm_mmu_wp_memory_region() - write protect stage 2 entries for memory slot
1390  * @kvm:        The KVM pointer
1391  * @slot:       The memory slot to write protect
1392  *
1393  * Called to start logging dirty pages after memory region
1394  * KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns
1395  * all present PMD and PTEs are write protected in the memory region.
1396  * Afterwards read of dirty page log can be called.
1397  *
1398  * Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired,
1399  * serializing operations for VM memory regions.
1400  */
1401 void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot)
1402 {
1403         struct kvm_memslots *slots = kvm_memslots(kvm);
1404         struct kvm_memory_slot *memslot = id_to_memslot(slots, slot);
1405         phys_addr_t start = memslot->base_gfn << PAGE_SHIFT;
1406         phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
1407
1408         spin_lock(&kvm->mmu_lock);
1409         stage2_wp_range(kvm, start, end);
1410         spin_unlock(&kvm->mmu_lock);
1411         kvm_flush_remote_tlbs(kvm);
1412 }
1413
1414 /**
1415  * kvm_mmu_write_protect_pt_masked() - write protect dirty pages
1416  * @kvm:        The KVM pointer
1417  * @slot:       The memory slot associated with mask
1418  * @gfn_offset: The gfn offset in memory slot
1419  * @mask:       The mask of dirty pages at offset 'gfn_offset' in this memory
1420  *              slot to be write protected
1421  *
1422  * Walks bits set in mask write protects the associated pte's. Caller must
1423  * acquire kvm_mmu_lock.
1424  */
1425 static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1426                 struct kvm_memory_slot *slot,
1427                 gfn_t gfn_offset, unsigned long mask)
1428 {
1429         phys_addr_t base_gfn = slot->base_gfn + gfn_offset;
1430         phys_addr_t start = (base_gfn +  __ffs(mask)) << PAGE_SHIFT;
1431         phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
1432
1433         stage2_wp_range(kvm, start, end);
1434 }
1435
1436 /*
1437  * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
1438  * dirty pages.
1439  *
1440  * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
1441  * enable dirty logging for them.
1442  */
1443 void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
1444                 struct kvm_memory_slot *slot,
1445                 gfn_t gfn_offset, unsigned long mask)
1446 {
1447         kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1448 }
1449
1450 static void clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size)
1451 {
1452         __clean_dcache_guest_page(pfn, size);
1453 }
1454
1455 static void invalidate_icache_guest_page(kvm_pfn_t pfn, unsigned long size)
1456 {
1457         __invalidate_icache_guest_page(pfn, size);
1458 }
1459
1460 static void kvm_send_hwpoison_signal(unsigned long address,
1461                                      struct vm_area_struct *vma)
1462 {
1463         short lsb;
1464
1465         if (is_vm_hugetlb_page(vma))
1466                 lsb = huge_page_shift(hstate_vma(vma));
1467         else
1468                 lsb = PAGE_SHIFT;
1469
1470         send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current);
1471 }
1472
1473 static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
1474                           struct kvm_memory_slot *memslot, unsigned long hva,
1475                           unsigned long fault_status)
1476 {
1477         int ret;
1478         bool write_fault, exec_fault, writable, hugetlb = false, force_pte = false;
1479         unsigned long mmu_seq;
1480         gfn_t gfn = fault_ipa >> PAGE_SHIFT;
1481         struct kvm *kvm = vcpu->kvm;
1482         struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
1483         struct vm_area_struct *vma;
1484         kvm_pfn_t pfn;
1485         pgprot_t mem_type = PAGE_S2;
1486         bool logging_active = memslot_is_logging(memslot);
1487         unsigned long flags = 0;
1488
1489         write_fault = kvm_is_write_fault(vcpu);
1490         exec_fault = kvm_vcpu_trap_is_iabt(vcpu);
1491         VM_BUG_ON(write_fault && exec_fault);
1492
1493         if (fault_status == FSC_PERM && !write_fault && !exec_fault) {
1494                 kvm_err("Unexpected L2 read permission error\n");
1495                 return -EFAULT;
1496         }
1497
1498         /* Let's check if we will get back a huge page backed by hugetlbfs */
1499         down_read(&current->mm->mmap_sem);
1500         vma = find_vma_intersection(current->mm, hva, hva + 1);
1501         if (unlikely(!vma)) {
1502                 kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
1503                 up_read(&current->mm->mmap_sem);
1504                 return -EFAULT;
1505         }
1506
1507         if (vma_kernel_pagesize(vma) == PMD_SIZE && !logging_active) {
1508                 hugetlb = true;
1509                 gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT;
1510         } else {
1511                 /*
1512                  * Pages belonging to memslots that don't have the same
1513                  * alignment for userspace and IPA cannot be mapped using
1514                  * block descriptors even if the pages belong to a THP for
1515                  * the process, because the stage-2 block descriptor will
1516                  * cover more than a single THP and we loose atomicity for
1517                  * unmapping, updates, and splits of the THP or other pages
1518                  * in the stage-2 block range.
1519                  */
1520                 if ((memslot->userspace_addr & ~PMD_MASK) !=
1521                     ((memslot->base_gfn << PAGE_SHIFT) & ~PMD_MASK))
1522                         force_pte = true;
1523         }
1524         up_read(&current->mm->mmap_sem);
1525
1526         /* We need minimum second+third level pages */
1527         ret = mmu_topup_memory_cache(memcache, kvm_mmu_cache_min_pages(kvm),
1528                                      KVM_NR_MEM_OBJS);
1529         if (ret)
1530                 return ret;
1531
1532         mmu_seq = vcpu->kvm->mmu_notifier_seq;
1533         /*
1534          * Ensure the read of mmu_notifier_seq happens before we call
1535          * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk
1536          * the page we just got a reference to gets unmapped before we have a
1537          * chance to grab the mmu_lock, which ensure that if the page gets
1538          * unmapped afterwards, the call to kvm_unmap_hva will take it away
1539          * from us again properly. This smp_rmb() interacts with the smp_wmb()
1540          * in kvm_mmu_notifier_invalidate_<page|range_end>.
1541          */
1542         smp_rmb();
1543
1544         pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable);
1545         if (pfn == KVM_PFN_ERR_HWPOISON) {
1546                 kvm_send_hwpoison_signal(hva, vma);
1547                 return 0;
1548         }
1549         if (is_error_noslot_pfn(pfn))
1550                 return -EFAULT;
1551
1552         if (kvm_is_device_pfn(pfn)) {
1553                 mem_type = PAGE_S2_DEVICE;
1554                 flags |= KVM_S2PTE_FLAG_IS_IOMAP;
1555         } else if (logging_active) {
1556                 /*
1557                  * Faults on pages in a memslot with logging enabled
1558                  * should not be mapped with huge pages (it introduces churn
1559                  * and performance degradation), so force a pte mapping.
1560                  */
1561                 force_pte = true;
1562                 flags |= KVM_S2_FLAG_LOGGING_ACTIVE;
1563
1564                 /*
1565                  * Only actually map the page as writable if this was a write
1566                  * fault.
1567                  */
1568                 if (!write_fault)
1569                         writable = false;
1570         }
1571
1572         spin_lock(&kvm->mmu_lock);
1573         if (mmu_notifier_retry(kvm, mmu_seq))
1574                 goto out_unlock;
1575
1576         if (!hugetlb && !force_pte)
1577                 hugetlb = transparent_hugepage_adjust(&pfn, &fault_ipa);
1578
1579         if (hugetlb) {
1580                 pmd_t new_pmd = pfn_pmd(pfn, mem_type);
1581                 new_pmd = pmd_mkhuge(new_pmd);
1582                 if (writable) {
1583                         new_pmd = kvm_s2pmd_mkwrite(new_pmd);
1584                         kvm_set_pfn_dirty(pfn);
1585                 }
1586
1587                 if (fault_status != FSC_PERM)
1588                         clean_dcache_guest_page(pfn, PMD_SIZE);
1589
1590                 if (exec_fault) {
1591                         new_pmd = kvm_s2pmd_mkexec(new_pmd);
1592                         invalidate_icache_guest_page(pfn, PMD_SIZE);
1593                 } else if (fault_status == FSC_PERM) {
1594                         /* Preserve execute if XN was already cleared */
1595                         if (stage2_is_exec(kvm, fault_ipa))
1596                                 new_pmd = kvm_s2pmd_mkexec(new_pmd);
1597                 }
1598
1599                 ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
1600         } else {
1601                 pte_t new_pte = pfn_pte(pfn, mem_type);
1602
1603                 if (writable) {
1604                         new_pte = kvm_s2pte_mkwrite(new_pte);
1605                         kvm_set_pfn_dirty(pfn);
1606                         mark_page_dirty(kvm, gfn);
1607                 }
1608
1609                 if (fault_status != FSC_PERM)
1610                         clean_dcache_guest_page(pfn, PAGE_SIZE);
1611
1612                 if (exec_fault) {
1613                         new_pte = kvm_s2pte_mkexec(new_pte);
1614                         invalidate_icache_guest_page(pfn, PAGE_SIZE);
1615                 } else if (fault_status == FSC_PERM) {
1616                         /* Preserve execute if XN was already cleared */
1617                         if (stage2_is_exec(kvm, fault_ipa))
1618                                 new_pte = kvm_s2pte_mkexec(new_pte);
1619                 }
1620
1621                 ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, flags);
1622         }
1623
1624 out_unlock:
1625         spin_unlock(&kvm->mmu_lock);
1626         kvm_set_pfn_accessed(pfn);
1627         kvm_release_pfn_clean(pfn);
1628         return ret;
1629 }
1630
1631 /*
1632  * Resolve the access fault by making the page young again.
1633  * Note that because the faulting entry is guaranteed not to be
1634  * cached in the TLB, we don't need to invalidate anything.
1635  * Only the HW Access Flag updates are supported for Stage 2 (no DBM),
1636  * so there is no need for atomic (pte|pmd)_mkyoung operations.
1637  */
1638 static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
1639 {
1640         pmd_t *pmd;
1641         pte_t *pte;
1642         kvm_pfn_t pfn;
1643         bool pfn_valid = false;
1644
1645         trace_kvm_access_fault(fault_ipa);
1646
1647         spin_lock(&vcpu->kvm->mmu_lock);
1648
1649         pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa);
1650         if (!pmd || pmd_none(*pmd))     /* Nothing there */
1651                 goto out;
1652
1653         if (pmd_thp_or_huge(*pmd)) {    /* THP, HugeTLB */
1654                 *pmd = pmd_mkyoung(*pmd);
1655                 pfn = pmd_pfn(*pmd);
1656                 pfn_valid = true;
1657                 goto out;
1658         }
1659
1660         pte = pte_offset_kernel(pmd, fault_ipa);
1661         if (pte_none(*pte))             /* Nothing there either */
1662                 goto out;
1663
1664         *pte = pte_mkyoung(*pte);       /* Just a page... */
1665         pfn = pte_pfn(*pte);
1666         pfn_valid = true;
1667 out:
1668         spin_unlock(&vcpu->kvm->mmu_lock);
1669         if (pfn_valid)
1670                 kvm_set_pfn_accessed(pfn);
1671 }
1672
1673 /**
1674  * kvm_handle_guest_abort - handles all 2nd stage aborts
1675  * @vcpu:       the VCPU pointer
1676  * @run:        the kvm_run structure
1677  *
1678  * Any abort that gets to the host is almost guaranteed to be caused by a
1679  * missing second stage translation table entry, which can mean that either the
1680  * guest simply needs more memory and we must allocate an appropriate page or it
1681  * can mean that the guest tried to access I/O memory, which is emulated by user
1682  * space. The distinction is based on the IPA causing the fault and whether this
1683  * memory region has been registered as standard RAM by user space.
1684  */
1685 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
1686 {
1687         unsigned long fault_status;
1688         phys_addr_t fault_ipa;
1689         struct kvm_memory_slot *memslot;
1690         unsigned long hva;
1691         bool is_iabt, write_fault, writable;
1692         gfn_t gfn;
1693         int ret, idx;
1694
1695         fault_status = kvm_vcpu_trap_get_fault_type(vcpu);
1696
1697         fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
1698         is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
1699
1700         /* Synchronous External Abort? */
1701         if (kvm_vcpu_dabt_isextabt(vcpu)) {
1702                 /*
1703                  * For RAS the host kernel may handle this abort.
1704                  * There is no need to pass the error into the guest.
1705                  */
1706                 if (!handle_guest_sea(fault_ipa, kvm_vcpu_get_hsr(vcpu)))
1707                         return 1;
1708
1709                 if (unlikely(!is_iabt)) {
1710                         kvm_inject_vabt(vcpu);
1711                         return 1;
1712                 }
1713         }
1714
1715         trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_hsr(vcpu),
1716                               kvm_vcpu_get_hfar(vcpu), fault_ipa);
1717
1718         /* Check the stage-2 fault is trans. fault or write fault */
1719         if (fault_status != FSC_FAULT && fault_status != FSC_PERM &&
1720             fault_status != FSC_ACCESS) {
1721                 kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
1722                         kvm_vcpu_trap_get_class(vcpu),
1723                         (unsigned long)kvm_vcpu_trap_get_fault(vcpu),
1724                         (unsigned long)kvm_vcpu_get_hsr(vcpu));
1725                 return -EFAULT;
1726         }
1727
1728         idx = srcu_read_lock(&vcpu->kvm->srcu);
1729
1730         gfn = fault_ipa >> PAGE_SHIFT;
1731         memslot = gfn_to_memslot(vcpu->kvm, gfn);
1732         hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
1733         write_fault = kvm_is_write_fault(vcpu);
1734         if (kvm_is_error_hva(hva) || (write_fault && !writable)) {
1735                 if (is_iabt) {
1736                         /* Prefetch Abort on I/O address */
1737                         kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
1738                         ret = 1;
1739                         goto out_unlock;
1740                 }
1741
1742                 /*
1743                  * Check for a cache maintenance operation. Since we
1744                  * ended-up here, we know it is outside of any memory
1745                  * slot. But we can't find out if that is for a device,
1746                  * or if the guest is just being stupid. The only thing
1747                  * we know for sure is that this range cannot be cached.
1748                  *
1749                  * So let's assume that the guest is just being
1750                  * cautious, and skip the instruction.
1751                  */
1752                 if (kvm_vcpu_dabt_is_cm(vcpu)) {
1753                         kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
1754                         ret = 1;
1755                         goto out_unlock;
1756                 }
1757
1758                 /*
1759                  * The IPA is reported as [MAX:12], so we need to
1760                  * complement it with the bottom 12 bits from the
1761                  * faulting VA. This is always 12 bits, irrespective
1762                  * of the page size.
1763                  */
1764                 fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1);
1765                 ret = io_mem_abort(vcpu, run, fault_ipa);
1766                 goto out_unlock;
1767         }
1768
1769         /* Userspace should not be able to register out-of-bounds IPAs */
1770         VM_BUG_ON(fault_ipa >= kvm_phys_size(vcpu->kvm));
1771
1772         if (fault_status == FSC_ACCESS) {
1773                 handle_access_fault(vcpu, fault_ipa);
1774                 ret = 1;
1775                 goto out_unlock;
1776         }
1777
1778         ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
1779         if (ret == 0)
1780                 ret = 1;
1781 out_unlock:
1782         srcu_read_unlock(&vcpu->kvm->srcu, idx);
1783         return ret;
1784 }
1785
1786 static int handle_hva_to_gpa(struct kvm *kvm,
1787                              unsigned long start,
1788                              unsigned long end,
1789                              int (*handler)(struct kvm *kvm,
1790                                             gpa_t gpa, u64 size,
1791                                             void *data),
1792                              void *data)
1793 {
1794         struct kvm_memslots *slots;
1795         struct kvm_memory_slot *memslot;
1796         int ret = 0;
1797
1798         slots = kvm_memslots(kvm);
1799
1800         /* we only care about the pages that the guest sees */
1801         kvm_for_each_memslot(memslot, slots) {
1802                 unsigned long hva_start, hva_end;
1803                 gfn_t gpa;
1804
1805                 hva_start = max(start, memslot->userspace_addr);
1806                 hva_end = min(end, memslot->userspace_addr +
1807                                         (memslot->npages << PAGE_SHIFT));
1808                 if (hva_start >= hva_end)
1809                         continue;
1810
1811                 gpa = hva_to_gfn_memslot(hva_start, memslot) << PAGE_SHIFT;
1812                 ret |= handler(kvm, gpa, (u64)(hva_end - hva_start), data);
1813         }
1814
1815         return ret;
1816 }
1817
1818 static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
1819 {
1820         unmap_stage2_range(kvm, gpa, size);
1821         return 0;
1822 }
1823
1824 int kvm_unmap_hva_range(struct kvm *kvm,
1825                         unsigned long start, unsigned long end)
1826 {
1827         if (!kvm->arch.pgd)
1828                 return 0;
1829
1830         trace_kvm_unmap_hva_range(start, end);
1831         handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL);
1832         return 0;
1833 }
1834
1835 static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
1836 {
1837         pte_t *pte = (pte_t *)data;
1838
1839         WARN_ON(size != PAGE_SIZE);
1840         /*
1841          * We can always call stage2_set_pte with KVM_S2PTE_FLAG_LOGGING_ACTIVE
1842          * flag clear because MMU notifiers will have unmapped a huge PMD before
1843          * calling ->change_pte() (which in turn calls kvm_set_spte_hva()) and
1844          * therefore stage2_set_pte() never needs to clear out a huge PMD
1845          * through this calling path.
1846          */
1847         stage2_set_pte(kvm, NULL, gpa, pte, 0);
1848         return 0;
1849 }
1850
1851
1852 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1853 {
1854         unsigned long end = hva + PAGE_SIZE;
1855         kvm_pfn_t pfn = pte_pfn(pte);
1856         pte_t stage2_pte;
1857
1858         if (!kvm->arch.pgd)
1859                 return;
1860
1861         trace_kvm_set_spte_hva(hva);
1862
1863         /*
1864          * We've moved a page around, probably through CoW, so let's treat it
1865          * just like a translation fault and clean the cache to the PoC.
1866          */
1867         clean_dcache_guest_page(pfn, PAGE_SIZE);
1868         stage2_pte = pfn_pte(pfn, PAGE_S2);
1869         handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
1870 }
1871
1872 static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
1873 {
1874         pmd_t *pmd;
1875         pte_t *pte;
1876
1877         WARN_ON(size != PAGE_SIZE && size != PMD_SIZE);
1878         pmd = stage2_get_pmd(kvm, NULL, gpa);
1879         if (!pmd || pmd_none(*pmd))     /* Nothing there */
1880                 return 0;
1881
1882         if (pmd_thp_or_huge(*pmd))      /* THP, HugeTLB */
1883                 return stage2_pmdp_test_and_clear_young(pmd);
1884
1885         pte = pte_offset_kernel(pmd, gpa);
1886         if (pte_none(*pte))
1887                 return 0;
1888
1889         return stage2_ptep_test_and_clear_young(pte);
1890 }
1891
1892 static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data)
1893 {
1894         pmd_t *pmd;
1895         pte_t *pte;
1896
1897         WARN_ON(size != PAGE_SIZE && size != PMD_SIZE);
1898         pmd = stage2_get_pmd(kvm, NULL, gpa);
1899         if (!pmd || pmd_none(*pmd))     /* Nothing there */
1900                 return 0;
1901
1902         if (pmd_thp_or_huge(*pmd))              /* THP, HugeTLB */
1903                 return pmd_young(*pmd);
1904
1905         pte = pte_offset_kernel(pmd, gpa);
1906         if (!pte_none(*pte))            /* Just a page... */
1907                 return pte_young(*pte);
1908
1909         return 0;
1910 }
1911
1912 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1913 {
1914         if (!kvm->arch.pgd)
1915                 return 0;
1916         trace_kvm_age_hva(start, end);
1917         return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
1918 }
1919
1920 int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1921 {
1922         if (!kvm->arch.pgd)
1923                 return 0;
1924         trace_kvm_test_age_hva(hva);
1925         return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL);
1926 }
1927
1928 void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
1929 {
1930         mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
1931 }
1932
1933 phys_addr_t kvm_mmu_get_httbr(void)
1934 {
1935         if (__kvm_cpu_uses_extended_idmap())
1936                 return virt_to_phys(merged_hyp_pgd);
1937         else
1938                 return virt_to_phys(hyp_pgd);
1939 }
1940
1941 phys_addr_t kvm_get_idmap_vector(void)
1942 {
1943         return hyp_idmap_vector;
1944 }
1945
1946 static int kvm_map_idmap_text(pgd_t *pgd)
1947 {
1948         int err;
1949
1950         /* Create the idmap in the boot page tables */
1951         err =   __create_hyp_mappings(pgd, __kvm_idmap_ptrs_per_pgd(),
1952                                       hyp_idmap_start, hyp_idmap_end,
1953                                       __phys_to_pfn(hyp_idmap_start),
1954                                       PAGE_HYP_EXEC);
1955         if (err)
1956                 kvm_err("Failed to idmap %lx-%lx\n",
1957                         hyp_idmap_start, hyp_idmap_end);
1958
1959         return err;
1960 }
1961
1962 int kvm_mmu_init(void)
1963 {
1964         int err;
1965
1966         hyp_idmap_start = kvm_virt_to_phys(__hyp_idmap_text_start);
1967         hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE);
1968         hyp_idmap_end = kvm_virt_to_phys(__hyp_idmap_text_end);
1969         hyp_idmap_end = ALIGN(hyp_idmap_end, PAGE_SIZE);
1970         hyp_idmap_vector = kvm_virt_to_phys(__kvm_hyp_init);
1971
1972         /*
1973          * We rely on the linker script to ensure at build time that the HYP
1974          * init code does not cross a page boundary.
1975          */
1976         BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);
1977
1978         kvm_debug("IDMAP page: %lx\n", hyp_idmap_start);
1979         kvm_debug("HYP VA range: %lx:%lx\n",
1980                   kern_hyp_va(PAGE_OFFSET),
1981                   kern_hyp_va((unsigned long)high_memory - 1));
1982
1983         if (hyp_idmap_start >= kern_hyp_va(PAGE_OFFSET) &&
1984             hyp_idmap_start <  kern_hyp_va((unsigned long)high_memory - 1) &&
1985             hyp_idmap_start != (unsigned long)__hyp_idmap_text_start) {
1986                 /*
1987                  * The idmap page is intersecting with the VA space,
1988                  * it is not safe to continue further.
1989                  */
1990                 kvm_err("IDMAP intersecting with HYP VA, unable to continue\n");
1991                 err = -EINVAL;
1992                 goto out;
1993         }
1994
1995         hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order);
1996         if (!hyp_pgd) {
1997                 kvm_err("Hyp mode PGD not allocated\n");
1998                 err = -ENOMEM;
1999                 goto out;
2000         }
2001
2002         if (__kvm_cpu_uses_extended_idmap()) {
2003                 boot_hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
2004                                                          hyp_pgd_order);
2005                 if (!boot_hyp_pgd) {
2006                         kvm_err("Hyp boot PGD not allocated\n");
2007                         err = -ENOMEM;
2008                         goto out;
2009                 }
2010
2011                 err = kvm_map_idmap_text(boot_hyp_pgd);
2012                 if (err)
2013                         goto out;
2014
2015                 merged_hyp_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
2016                 if (!merged_hyp_pgd) {
2017                         kvm_err("Failed to allocate extra HYP pgd\n");
2018                         goto out;
2019                 }
2020                 __kvm_extend_hypmap(boot_hyp_pgd, hyp_pgd, merged_hyp_pgd,
2021                                     hyp_idmap_start);
2022         } else {
2023                 err = kvm_map_idmap_text(hyp_pgd);
2024                 if (err)
2025                         goto out;
2026         }
2027
2028         io_map_base = hyp_idmap_start;
2029         return 0;
2030 out:
2031         free_hyp_pgds();
2032         return err;
2033 }
2034
2035 void kvm_arch_commit_memory_region(struct kvm *kvm,
2036                                    const struct kvm_userspace_memory_region *mem,
2037                                    const struct kvm_memory_slot *old,
2038                                    const struct kvm_memory_slot *new,
2039                                    enum kvm_mr_change change)
2040 {
2041         /*
2042          * At this point memslot has been committed and there is an
2043          * allocated dirty_bitmap[], dirty pages will be be tracked while the
2044          * memory slot is write protected.
2045          */
2046         if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES)
2047                 kvm_mmu_wp_memory_region(kvm, mem->slot);
2048 }
2049
2050 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2051                                    struct kvm_memory_slot *memslot,
2052                                    const struct kvm_userspace_memory_region *mem,
2053                                    enum kvm_mr_change change)
2054 {
2055         hva_t hva = mem->userspace_addr;
2056         hva_t reg_end = hva + mem->memory_size;
2057         bool writable = !(mem->flags & KVM_MEM_READONLY);
2058         int ret = 0;
2059
2060         if (change != KVM_MR_CREATE && change != KVM_MR_MOVE &&
2061                         change != KVM_MR_FLAGS_ONLY)
2062                 return 0;
2063
2064         /*
2065          * Prevent userspace from creating a memory region outside of the IPA
2066          * space addressable by the KVM guest IPA space.
2067          */
2068         if (memslot->base_gfn + memslot->npages >=
2069             (kvm_phys_size(kvm) >> PAGE_SHIFT))
2070                 return -EFAULT;
2071
2072         down_read(&current->mm->mmap_sem);
2073         /*
2074          * A memory region could potentially cover multiple VMAs, and any holes
2075          * between them, so iterate over all of them to find out if we can map
2076          * any of them right now.
2077          *
2078          *     +--------------------------------------------+
2079          * +---------------+----------------+   +----------------+
2080          * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
2081          * +---------------+----------------+   +----------------+
2082          *     |               memory region                |
2083          *     +--------------------------------------------+
2084          */
2085         do {
2086                 struct vm_area_struct *vma = find_vma(current->mm, hva);
2087                 hva_t vm_start, vm_end;
2088
2089                 if (!vma || vma->vm_start >= reg_end)
2090                         break;
2091
2092                 /*
2093                  * Mapping a read-only VMA is only allowed if the
2094                  * memory region is configured as read-only.
2095                  */
2096                 if (writable && !(vma->vm_flags & VM_WRITE)) {
2097                         ret = -EPERM;
2098                         break;
2099                 }
2100
2101                 /*
2102                  * Take the intersection of this VMA with the memory region
2103                  */
2104                 vm_start = max(hva, vma->vm_start);
2105                 vm_end = min(reg_end, vma->vm_end);
2106
2107                 if (vma->vm_flags & VM_PFNMAP) {
2108                         gpa_t gpa = mem->guest_phys_addr +
2109                                     (vm_start - mem->userspace_addr);
2110                         phys_addr_t pa;
2111
2112                         pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
2113                         pa += vm_start - vma->vm_start;
2114
2115                         /* IO region dirty page logging not allowed */
2116                         if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES) {
2117                                 ret = -EINVAL;
2118                                 goto out;
2119                         }
2120
2121                         ret = kvm_phys_addr_ioremap(kvm, gpa, pa,
2122                                                     vm_end - vm_start,
2123                                                     writable);
2124                         if (ret)
2125                                 break;
2126                 }
2127                 hva = vm_end;
2128         } while (hva < reg_end);
2129
2130         if (change == KVM_MR_FLAGS_ONLY)
2131                 goto out;
2132
2133         spin_lock(&kvm->mmu_lock);
2134         if (ret)
2135                 unmap_stage2_range(kvm, mem->guest_phys_addr, mem->memory_size);
2136         else
2137                 stage2_flush_memslot(kvm, memslot);
2138         spin_unlock(&kvm->mmu_lock);
2139 out:
2140         up_read(&current->mm->mmap_sem);
2141         return ret;
2142 }
2143
2144 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
2145                            struct kvm_memory_slot *dont)
2146 {
2147 }
2148
2149 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2150                             unsigned long npages)
2151 {
2152         return 0;
2153 }
2154
2155 void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots)
2156 {
2157 }
2158
2159 void kvm_arch_flush_shadow_all(struct kvm *kvm)
2160 {
2161         kvm_free_stage2_pgd(kvm);
2162 }
2163
2164 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
2165                                    struct kvm_memory_slot *slot)
2166 {
2167         gpa_t gpa = slot->base_gfn << PAGE_SHIFT;
2168         phys_addr_t size = slot->npages << PAGE_SHIFT;
2169
2170         spin_lock(&kvm->mmu_lock);
2171         unmap_stage2_range(kvm, gpa, size);
2172         spin_unlock(&kvm->mmu_lock);
2173 }
2174
2175 /*
2176  * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
2177  *
2178  * Main problems:
2179  * - S/W ops are local to a CPU (not broadcast)
2180  * - We have line migration behind our back (speculation)
2181  * - System caches don't support S/W at all (damn!)
2182  *
2183  * In the face of the above, the best we can do is to try and convert
2184  * S/W ops to VA ops. Because the guest is not allowed to infer the
2185  * S/W to PA mapping, it can only use S/W to nuke the whole cache,
2186  * which is a rather good thing for us.
2187  *
2188  * Also, it is only used when turning caches on/off ("The expected
2189  * usage of the cache maintenance instructions that operate by set/way
2190  * is associated with the cache maintenance instructions associated
2191  * with the powerdown and powerup of caches, if this is required by
2192  * the implementation.").
2193  *
2194  * We use the following policy:
2195  *
2196  * - If we trap a S/W operation, we enable VM trapping to detect
2197  *   caches being turned on/off, and do a full clean.
2198  *
2199  * - We flush the caches on both caches being turned on and off.
2200  *
2201  * - Once the caches are enabled, we stop trapping VM ops.
2202  */
2203 void kvm_set_way_flush(struct kvm_vcpu *vcpu)
2204 {
2205         unsigned long hcr = *vcpu_hcr(vcpu);
2206
2207         /*
2208          * If this is the first time we do a S/W operation
2209          * (i.e. HCR_TVM not set) flush the whole memory, and set the
2210          * VM trapping.
2211          *
2212          * Otherwise, rely on the VM trapping to wait for the MMU +
2213          * Caches to be turned off. At that point, we'll be able to
2214          * clean the caches again.
2215          */
2216         if (!(hcr & HCR_TVM)) {
2217                 trace_kvm_set_way_flush(*vcpu_pc(vcpu),
2218                                         vcpu_has_cache_enabled(vcpu));
2219                 stage2_flush_vm(vcpu->kvm);
2220                 *vcpu_hcr(vcpu) = hcr | HCR_TVM;
2221         }
2222 }
2223
2224 void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
2225 {
2226         bool now_enabled = vcpu_has_cache_enabled(vcpu);
2227
2228         /*
2229          * If switching the MMU+caches on, need to invalidate the caches.
2230          * If switching it off, need to clean the caches.
2231          * Clean + invalidate does the trick always.
2232          */
2233         if (now_enabled != was_enabled)
2234                 stage2_flush_vm(vcpu->kvm);
2235
2236         /* Caches are now on, stop trapping VM ops (until a S/W op) */
2237         if (now_enabled)
2238                 *vcpu_hcr(vcpu) &= ~HCR_TVM;
2239
2240         trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled);
2241 }