arch/mips/kvm/tlb.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS TLB handling, this file is part of the Linux host kernel so that
   7  * TLB handlers run from KSEG0
   8  *
   9  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
  10  * Authors: Sanjay Lal <[email protected]>
  11  */
  12
  13 #include <linux/sched.h>
  14 #include <linux/smp.h>
  15 #include <linux/mm.h>
  16 #include <linux/delay.h>
  17 #include <linux/export.h>
  18 #include <linux/kvm_host.h>
  19 #include <linux/srcu.h>
  20
  21 #include <asm/cpu.h>
  22 #include <asm/bootinfo.h>
  23 #include <asm/mipsregs.h>
  24 #include <asm/mmu_context.h>
  25 #include <asm/pgtable.h>
  26 #include <asm/cacheflush.h>
  27 #include <asm/tlb.h>
  28 #include <asm/tlbdebug.h>
  29
  30 #undef CONFIG_MIPS_MT
  31 #include <asm/r4kcache.h>
  32 #define CONFIG_MIPS_MT
  33
  34 #define KVM_GUEST_PC_TLB    0
  35 #define KVM_GUEST_SP_TLB    1
  36
  37 #ifdef CONFIG_KVM_MIPS_VZ
  38 unsigned long GUESTID_MASK;
  39 EXPORT_SYMBOL_GPL(GUESTID_MASK);
  40 unsigned long GUESTID_FIRST_VERSION;
  41 EXPORT_SYMBOL_GPL(GUESTID_FIRST_VERSION);
  42 unsigned long GUESTID_VERSION_MASK;
  43 EXPORT_SYMBOL_GPL(GUESTID_VERSION_MASK);
  44
  45 static u32 kvm_mips_get_root_asid(struct kvm_vcpu *vcpu)
  46 {
  47         struct mm_struct *gpa_mm = &vcpu->kvm->arch.gpa_mm;
  48
  49         if (cpu_has_guestid)
  50                 return 0;
  51         else
  52                 return cpu_asid(smp_processor_id(), gpa_mm);
  53 }
  54 #endif
  55
  56 static u32 kvm_mips_get_kernel_asid(struct kvm_vcpu *vcpu)
  57 {
  58         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
  59         int cpu = smp_processor_id();
  60
  61         return cpu_asid(cpu, kern_mm);
  62 }
  63
  64 static u32 kvm_mips_get_user_asid(struct kvm_vcpu *vcpu)
  65 {
  66         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
  67         int cpu = smp_processor_id();
  68
  69         return cpu_asid(cpu, user_mm);
  70 }
  71
  72 /* Structure defining an tlb entry data set. */
  73
  74 void kvm_mips_dump_host_tlbs(void)
  75 {
  76         unsigned long flags;
  77
  78         local_irq_save(flags);
  79
  80         kvm_info("HOST TLBs:\n");
  81         dump_tlb_regs();
  82         pr_info("\n");
  83         dump_tlb_all();
  84
  85         local_irq_restore(flags);
  86 }
  87 EXPORT_SYMBOL_GPL(kvm_mips_dump_host_tlbs);
  88
  89 void kvm_mips_dump_guest_tlbs(struct kvm_vcpu *vcpu)
  90 {
  91         struct mips_coproc *cop0 = vcpu->arch.cop0;
  92         struct kvm_mips_tlb tlb;
  93         int i;
  94
  95         kvm_info("Guest TLBs:\n");
  96         kvm_info("Guest EntryHi: %#lx\n", kvm_read_c0_guest_entryhi(cop0));
  97
  98         for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
  99                 tlb = vcpu->arch.guest_tlb[i];
 100                 kvm_info("TLB%c%3d Hi 0x%08lx ",
 101                          (tlb.tlb_lo[0] | tlb.tlb_lo[1]) & ENTRYLO_V
 102                                                         ? ' ' : '*',
 103                          i, tlb.tlb_hi);
 104                 kvm_info("Lo0=0x%09llx %c%c attr %lx ",
 105                          (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[0]),
 106                          (tlb.tlb_lo[0] & ENTRYLO_D) ? 'D' : ' ',
 107                          (tlb.tlb_lo[0] & ENTRYLO_G) ? 'G' : ' ',
 108                          (tlb.tlb_lo[0] & ENTRYLO_C) >> ENTRYLO_C_SHIFT);
 109                 kvm_info("Lo1=0x%09llx %c%c attr %lx sz=%lx\n",
 110                          (u64) mips3_tlbpfn_to_paddr(tlb.tlb_lo[1]),
 111                          (tlb.tlb_lo[1] & ENTRYLO_D) ? 'D' : ' ',
 112                          (tlb.tlb_lo[1] & ENTRYLO_G) ? 'G' : ' ',
 113                          (tlb.tlb_lo[1] & ENTRYLO_C) >> ENTRYLO_C_SHIFT,
 114                          tlb.tlb_mask);
 115         }
 116 }
 117 EXPORT_SYMBOL_GPL(kvm_mips_dump_guest_tlbs);
 118
 119 int kvm_mips_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long entryhi)
 120 {
 121         int i;
 122         int index = -1;
 123         struct kvm_mips_tlb *tlb = vcpu->arch.guest_tlb;
 124
 125         for (i = 0; i < KVM_MIPS_GUEST_TLB_SIZE; i++) {
 126                 if (TLB_HI_VPN2_HIT(tlb[i], entryhi) &&
 127                     TLB_HI_ASID_HIT(tlb[i], entryhi)) {
 128                         index = i;
 129                         break;
 130                 }
 131         }
 132
 133         kvm_debug("%s: entryhi: %#lx, index: %d lo0: %#lx, lo1: %#lx\n",
 134                   __func__, entryhi, index, tlb[i].tlb_lo[0], tlb[i].tlb_lo[1]);
 135
 136         return index;
 137 }
 138 EXPORT_SYMBOL_GPL(kvm_mips_guest_tlb_lookup);
 139
 140 static int _kvm_mips_host_tlb_inv(unsigned long entryhi)
 141 {
 142         int idx;
 143
 144         write_c0_entryhi(entryhi);
 145         mtc0_tlbw_hazard();
 146
 147         tlb_probe();
 148         tlb_probe_hazard();
 149         idx = read_c0_index();
 150
 151         if (idx >= current_cpu_data.tlbsize)
 152                 BUG();
 153
 154         if (idx >= 0) {
 155                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 156                 write_c0_entrylo0(0);
 157                 write_c0_entrylo1(0);
 158                 mtc0_tlbw_hazard();
 159
 160                 tlb_write_indexed();
 161                 tlbw_use_hazard();
 162         }
 163
 164         return idx;
 165 }
 166
 167 int kvm_mips_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va,
 168                           bool user, bool kernel)
 169 {
 170         /*
 171          * Initialize idx_user and idx_kernel to workaround bogus
 172          * maybe-initialized warning when using GCC 6.
 173          */
 174         int idx_user = 0, idx_kernel = 0;
 175         unsigned long flags, old_entryhi;
 176
 177         local_irq_save(flags);
 178
 179         old_entryhi = read_c0_entryhi();
 180
 181         if (user)
 182                 idx_user = _kvm_mips_host_tlb_inv((va & VPN2_MASK) |
 183                                                   kvm_mips_get_user_asid(vcpu));
 184         if (kernel)
 185                 idx_kernel = _kvm_mips_host_tlb_inv((va & VPN2_MASK) |
 186                                                 kvm_mips_get_kernel_asid(vcpu));
 187
 188         write_c0_entryhi(old_entryhi);
 189         mtc0_tlbw_hazard();
 190
 191         local_irq_restore(flags);
 192
 193         /*
 194          * We don't want to get reserved instruction exceptions for missing tlb
 195          * entries.
 196          */
 197         if (cpu_has_vtag_icache)
 198                 flush_icache_all();
 199
 200         if (user && idx_user >= 0)
 201                 kvm_debug("%s: Invalidated guest user entryhi %#lx @ idx %d\n",
 202                           __func__, (va & VPN2_MASK) |
 203                                     kvm_mips_get_user_asid(vcpu), idx_user);
 204         if (kernel && idx_kernel >= 0)
 205                 kvm_debug("%s: Invalidated guest kernel entryhi %#lx @ idx %d\n",
 206                           __func__, (va & VPN2_MASK) |
 207                                     kvm_mips_get_kernel_asid(vcpu), idx_kernel);
 208
 209         return 0;
 210 }
 211 EXPORT_SYMBOL_GPL(kvm_mips_host_tlb_inv);
 212
 213 #ifdef CONFIG_KVM_MIPS_VZ
 214
 215 /* GuestID management */
 216
 217 /**
 218  * clear_root_gid() - Set GuestCtl1.RID for normal root operation.
 219  */
 220 static inline void clear_root_gid(void)
 221 {
 222         if (cpu_has_guestid) {
 223                 clear_c0_guestctl1(MIPS_GCTL1_RID);
 224                 mtc0_tlbw_hazard();
 225         }
 226 }
 227
 228 /**
 229  * set_root_gid_to_guest_gid() - Set GuestCtl1.RID to match GuestCtl1.ID.
 230  *
 231  * Sets the root GuestID to match the current guest GuestID, for TLB operation
 232  * on the GPA->RPA mappings in the root TLB.
 233  *
 234  * The caller must be sure to disable HTW while the root GID is set, and
 235  * possibly longer if TLB registers are modified.
 236  */
 237 static inline void set_root_gid_to_guest_gid(void)
 238 {
 239         unsigned int guestctl1;
 240
 241         if (cpu_has_guestid) {
 242                 back_to_back_c0_hazard();
 243                 guestctl1 = read_c0_guestctl1();
 244                 guestctl1 = (guestctl1 & ~MIPS_GCTL1_RID) |
 245                         ((guestctl1 & MIPS_GCTL1_ID) >> MIPS_GCTL1_ID_SHIFT)
 246                                                      << MIPS_GCTL1_RID_SHIFT;
 247                 write_c0_guestctl1(guestctl1);
 248                 mtc0_tlbw_hazard();
 249         }
 250 }
 251
 252 int kvm_vz_host_tlb_inv(struct kvm_vcpu *vcpu, unsigned long va)
 253 {
 254         int idx;
 255         unsigned long flags, old_entryhi;
 256
 257         local_irq_save(flags);
 258         htw_stop();
 259
 260         /* Set root GuestID for root probe and write of guest TLB entry */
 261         set_root_gid_to_guest_gid();
 262
 263         old_entryhi = read_c0_entryhi();
 264
 265         idx = _kvm_mips_host_tlb_inv((va & VPN2_MASK) |
 266                                      kvm_mips_get_root_asid(vcpu));
 267
 268         write_c0_entryhi(old_entryhi);
 269         clear_root_gid();
 270         mtc0_tlbw_hazard();
 271
 272         htw_start();
 273         local_irq_restore(flags);
 274
 275         /*
 276          * We don't want to get reserved instruction exceptions for missing tlb
 277          * entries.
 278          */
 279         if (cpu_has_vtag_icache)
 280                 flush_icache_all();
 281
 282         if (idx > 0)
 283                 kvm_debug("%s: Invalidated root entryhi %#lx @ idx %d\n",
 284                           __func__, (va & VPN2_MASK) |
 285                                     kvm_mips_get_root_asid(vcpu), idx);
 286
 287         return 0;
 288 }
 289 EXPORT_SYMBOL_GPL(kvm_vz_host_tlb_inv);
 290
 291 /**
 292  * kvm_vz_guest_tlb_lookup() - Lookup a guest VZ TLB mapping.
 293  * @vcpu:       KVM VCPU pointer.
 294  * @gpa:        Guest virtual address in a TLB mapped guest segment.
 295  * @gpa:        Ponter to output guest physical address it maps to.
 296  *
 297  * Converts a guest virtual address in a guest TLB mapped segment to a guest
 298  * physical address, by probing the guest TLB.
 299  *
 300  * Returns:     0 if guest TLB mapping exists for @gva. *@gpa will have been
 301  *              written.
 302  *              -EFAULT if no guest TLB mapping exists for @gva. *@gpa may not
 303  *              have been written.
 304  */
 305 int kvm_vz_guest_tlb_lookup(struct kvm_vcpu *vcpu, unsigned long gva,
 306                             unsigned long *gpa)
 307 {
 308         unsigned long o_entryhi, o_entrylo[2], o_pagemask;
 309         unsigned int o_index;
 310         unsigned long entrylo[2], pagemask, pagemaskbit, pa;
 311         unsigned long flags;
 312         int index;
 313
 314         /* Probe the guest TLB for a mapping */
 315         local_irq_save(flags);
 316         /* Set root GuestID for root probe of guest TLB entry */
 317         htw_stop();
 318         set_root_gid_to_guest_gid();
 319
 320         o_entryhi = read_gc0_entryhi();
 321         o_index = read_gc0_index();
 322
 323         write_gc0_entryhi((o_entryhi & 0x3ff) | (gva & ~0xfffl));
 324         mtc0_tlbw_hazard();
 325         guest_tlb_probe();
 326         tlb_probe_hazard();
 327
 328         index = read_gc0_index();
 329         if (index < 0) {
 330                 /* No match, fail */
 331                 write_gc0_entryhi(o_entryhi);
 332                 write_gc0_index(o_index);
 333
 334                 clear_root_gid();
 335                 htw_start();
 336                 local_irq_restore(flags);
 337                 return -EFAULT;
 338         }
 339
 340         /* Match! read the TLB entry */
 341         o_entrylo[0] = read_gc0_entrylo0();
 342         o_entrylo[1] = read_gc0_entrylo1();
 343         o_pagemask = read_gc0_pagemask();
 344
 345         mtc0_tlbr_hazard();
 346         guest_tlb_read();
 347         tlb_read_hazard();
 348
 349         entrylo[0] = read_gc0_entrylo0();
 350         entrylo[1] = read_gc0_entrylo1();
 351         pagemask = ~read_gc0_pagemask() & ~0x1fffl;
 352
 353         write_gc0_entryhi(o_entryhi);
 354         write_gc0_index(o_index);
 355         write_gc0_entrylo0(o_entrylo[0]);
 356         write_gc0_entrylo1(o_entrylo[1]);
 357         write_gc0_pagemask(o_pagemask);
 358
 359         clear_root_gid();
 360         htw_start();
 361         local_irq_restore(flags);
 362
 363         /* Select one of the EntryLo values and interpret the GPA */
 364         pagemaskbit = (pagemask ^ (pagemask & (pagemask - 1))) >> 1;
 365         pa = entrylo[!!(gva & pagemaskbit)];
 366
 367         /*
 368          * TLB entry may have become invalid since TLB probe if physical FTLB
 369          * entries are shared between threads (e.g. I6400).
 370          */
 371         if (!(pa & ENTRYLO_V))
 372                 return -EFAULT;
 373
 374         /*
 375          * Note, this doesn't take guest MIPS32 XPA into account, where PFN is
 376          * split with XI/RI in the middle.
 377          */
 378         pa = (pa << 6) & ~0xfffl;
 379         pa |= gva & ~(pagemask | pagemaskbit);
 380
 381         *gpa = pa;
 382         return 0;
 383 }
 384 EXPORT_SYMBOL_GPL(kvm_vz_guest_tlb_lookup);
 385
 386 /**
 387  * kvm_vz_local_flush_roottlb_all_guests() - Flush all root TLB entries for
 388  * guests.
 389  *
 390  * Invalidate all entries in root tlb which are GPA mappings.
 391  */
 392 void kvm_vz_local_flush_roottlb_all_guests(void)
 393 {
 394         unsigned long flags;
 395         unsigned long old_entryhi, old_pagemask, old_guestctl1;
 396         int entry;
 397
 398         if (WARN_ON(!cpu_has_guestid))
 399                 return;
 400
 401         local_irq_save(flags);
 402         htw_stop();
 403
 404         /* TLBR may clobber EntryHi.ASID, PageMask, and GuestCtl1.RID */
 405         old_entryhi = read_c0_entryhi();
 406         old_pagemask = read_c0_pagemask();
 407         old_guestctl1 = read_c0_guestctl1();
 408
 409         /*
 410          * Invalidate guest entries in root TLB while leaving root entries
 411          * intact when possible.
 412          */
 413         for (entry = 0; entry < current_cpu_data.tlbsize; entry++) {
 414                 write_c0_index(entry);
 415                 mtc0_tlbw_hazard();
 416                 tlb_read();
 417                 tlb_read_hazard();
 418
 419                 /* Don't invalidate non-guest (RVA) mappings in the root TLB */
 420                 if (!(read_c0_guestctl1() & MIPS_GCTL1_RID))
 421                         continue;
 422
 423                 /* Make sure all entries differ. */
 424                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
 425                 write_c0_entrylo0(0);
 426                 write_c0_entrylo1(0);
 427                 write_c0_guestctl1(0);
 428                 mtc0_tlbw_hazard();
 429                 tlb_write_indexed();
 430         }
 431
 432         write_c0_entryhi(old_entryhi);
 433         write_c0_pagemask(old_pagemask);
 434         write_c0_guestctl1(old_guestctl1);
 435         tlbw_use_hazard();
 436
 437         htw_start();
 438         local_irq_restore(flags);
 439 }
 440 EXPORT_SYMBOL_GPL(kvm_vz_local_flush_roottlb_all_guests);
 441
 442 /**
 443  * kvm_vz_local_flush_guesttlb_all() - Flush all guest TLB entries.
 444  *
 445  * Invalidate all entries in guest tlb irrespective of guestid.
 446  */
 447 void kvm_vz_local_flush_guesttlb_all(void)
 448 {
 449         unsigned long flags;
 450         unsigned long old_index;
 451         unsigned long old_entryhi;
 452         unsigned long old_entrylo[2];
 453         unsigned long old_pagemask;
 454         int entry;
 455         u64 cvmmemctl2 = 0;
 456
 457         local_irq_save(flags);
 458
 459         /* Preserve all clobbered guest registers */
 460         old_index = read_gc0_index();
 461         old_entryhi = read_gc0_entryhi();
 462         old_entrylo[0] = read_gc0_entrylo0();
 463         old_entrylo[1] = read_gc0_entrylo1();
 464         old_pagemask = read_gc0_pagemask();
 465
 466         switch (current_cpu_type()) {
 467         case CPU_CAVIUM_OCTEON3:
 468                 /* Inhibit machine check due to multiple matching TLB entries */
 469                 cvmmemctl2 = read_c0_cvmmemctl2();
 470                 cvmmemctl2 |= CVMMEMCTL2_INHIBITTS;
 471                 write_c0_cvmmemctl2(cvmmemctl2);
 472                 break;
 473         }
 474
 475         /* Invalidate guest entries in guest TLB */
 476         write_gc0_entrylo0(0);
 477         write_gc0_entrylo1(0);
 478         write_gc0_pagemask(0);
 479         for (entry = 0; entry < current_cpu_data.guest.tlbsize; entry++) {
 480                 /* Make sure all entries differ. */
 481                 write_gc0_index(entry);
 482                 write_gc0_entryhi(UNIQUE_GUEST_ENTRYHI(entry));
 483                 mtc0_tlbw_hazard();
 484                 guest_tlb_write_indexed();
 485         }
 486
 487         if (cvmmemctl2) {
 488                 cvmmemctl2 &= ~CVMMEMCTL2_INHIBITTS;
 489                 write_c0_cvmmemctl2(cvmmemctl2);
 490         }
 491
 492         write_gc0_index(old_index);
 493         write_gc0_entryhi(old_entryhi);
 494         write_gc0_entrylo0(old_entrylo[0]);
 495         write_gc0_entrylo1(old_entrylo[1]);
 496         write_gc0_pagemask(old_pagemask);
 497         tlbw_use_hazard();
 498
 499         local_irq_restore(flags);
 500 }
 501 EXPORT_SYMBOL_GPL(kvm_vz_local_flush_guesttlb_all);
 502
 503 /**
 504  * kvm_vz_save_guesttlb() - Save a range of guest TLB entries.
 505  * @buf:        Buffer to write TLB entries into.
 506  * @index:      Start index.
 507  * @count:      Number of entries to save.
 508  *
 509  * Save a range of guest TLB entries. The caller must ensure interrupts are
 510  * disabled.
 511  */
 512 void kvm_vz_save_guesttlb(struct kvm_mips_tlb *buf, unsigned int index,
 513                           unsigned int count)
 514 {
 515         unsigned int end = index + count;
 516         unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask;
 517         unsigned int guestctl1 = 0;
 518         int old_index, i;
 519
 520         /* Save registers we're about to clobber */
 521         old_index = read_gc0_index();
 522         old_entryhi = read_gc0_entryhi();
 523         old_entrylo0 = read_gc0_entrylo0();
 524         old_entrylo1 = read_gc0_entrylo1();
 525         old_pagemask = read_gc0_pagemask();
 526
 527         /* Set root GuestID for root probe */
 528         htw_stop();
 529         set_root_gid_to_guest_gid();
 530         if (cpu_has_guestid)
 531                 guestctl1 = read_c0_guestctl1();
 532
 533         /* Read each entry from guest TLB */
 534         for (i = index; i < end; ++i, ++buf) {
 535                 write_gc0_index(i);
 536
 537                 mtc0_tlbr_hazard();
 538                 guest_tlb_read();
 539                 tlb_read_hazard();
 540
 541                 if (cpu_has_guestid &&
 542                     (read_c0_guestctl1() ^ guestctl1) & MIPS_GCTL1_RID) {
 543                         /* Entry invalid or belongs to another guest */
 544                         buf->tlb_hi = UNIQUE_GUEST_ENTRYHI(i);
 545                         buf->tlb_lo[0] = 0;
 546                         buf->tlb_lo[1] = 0;
 547                         buf->tlb_mask = 0;
 548                 } else {
 549                         /* Entry belongs to the right guest */
 550                         buf->tlb_hi = read_gc0_entryhi();
 551                         buf->tlb_lo[0] = read_gc0_entrylo0();
 552                         buf->tlb_lo[1] = read_gc0_entrylo1();
 553                         buf->tlb_mask = read_gc0_pagemask();
 554                 }
 555         }
 556
 557         /* Clear root GuestID again */
 558         clear_root_gid();
 559         htw_start();
 560
 561         /* Restore clobbered registers */
 562         write_gc0_index(old_index);
 563         write_gc0_entryhi(old_entryhi);
 564         write_gc0_entrylo0(old_entrylo0);
 565         write_gc0_entrylo1(old_entrylo1);
 566         write_gc0_pagemask(old_pagemask);
 567
 568         tlbw_use_hazard();
 569 }
 570 EXPORT_SYMBOL_GPL(kvm_vz_save_guesttlb);
 571
 572 /**
 573  * kvm_vz_load_guesttlb() - Save a range of guest TLB entries.
 574  * @buf:        Buffer to read TLB entries from.
 575  * @index:      Start index.
 576  * @count:      Number of entries to load.
 577  *
 578  * Load a range of guest TLB entries. The caller must ensure interrupts are
 579  * disabled.
 580  */
 581 void kvm_vz_load_guesttlb(const struct kvm_mips_tlb *buf, unsigned int index,
 582                           unsigned int count)
 583 {
 584         unsigned int end = index + count;
 585         unsigned long old_entryhi, old_entrylo0, old_entrylo1, old_pagemask;
 586         int old_index, i;
 587
 588         /* Save registers we're about to clobber */
 589         old_index = read_gc0_index();
 590         old_entryhi = read_gc0_entryhi();
 591         old_entrylo0 = read_gc0_entrylo0();
 592         old_entrylo1 = read_gc0_entrylo1();
 593         old_pagemask = read_gc0_pagemask();
 594
 595         /* Set root GuestID for root probe */
 596         htw_stop();
 597         set_root_gid_to_guest_gid();
 598
 599         /* Write each entry to guest TLB */
 600         for (i = index; i < end; ++i, ++buf) {
 601                 write_gc0_index(i);
 602                 write_gc0_entryhi(buf->tlb_hi);
 603                 write_gc0_entrylo0(buf->tlb_lo[0]);
 604                 write_gc0_entrylo1(buf->tlb_lo[1]);
 605                 write_gc0_pagemask(buf->tlb_mask);
 606
 607                 mtc0_tlbw_hazard();
 608                 guest_tlb_write_indexed();
 609         }
 610
 611         /* Clear root GuestID again */
 612         clear_root_gid();
 613         htw_start();
 614
 615         /* Restore clobbered registers */
 616         write_gc0_index(old_index);
 617         write_gc0_entryhi(old_entryhi);
 618         write_gc0_entrylo0(old_entrylo0);
 619         write_gc0_entrylo1(old_entrylo1);
 620         write_gc0_pagemask(old_pagemask);
 621
 622         tlbw_use_hazard();
 623 }
 624 EXPORT_SYMBOL_GPL(kvm_vz_load_guesttlb);
 625
 626 #ifdef CONFIG_CPU_LOONGSON64
 627 void kvm_loongson_clear_guest_vtlb(void)
 628 {
 629         int idx = read_gc0_index();
 630
 631         /* Set root GuestID for root probe and write of guest TLB entry */
 632         set_root_gid_to_guest_gid();
 633
 634         write_gc0_index(0);
 635         guest_tlbinvf();
 636         write_gc0_index(idx);
 637
 638         clear_root_gid();
 639         set_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
 640 }
 641 EXPORT_SYMBOL_GPL(kvm_loongson_clear_guest_vtlb);
 642
 643 void kvm_loongson_clear_guest_ftlb(void)
 644 {
 645         int i;
 646         int idx = read_gc0_index();
 647
 648         /* Set root GuestID for root probe and write of guest TLB entry */
 649         set_root_gid_to_guest_gid();
 650
 651         for (i = current_cpu_data.tlbsizevtlb;
 652              i < (current_cpu_data.tlbsizevtlb +
 653                      current_cpu_data.tlbsizeftlbsets);
 654              i++) {
 655                 write_gc0_index(i);
 656                 guest_tlbinvf();
 657         }
 658         write_gc0_index(idx);
 659
 660         clear_root_gid();
 661         set_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
 662 }
 663 EXPORT_SYMBOL_GPL(kvm_loongson_clear_guest_ftlb);
 664 #endif
 665
 666 #endif
 667
 668 /**
 669  * kvm_mips_suspend_mm() - Suspend the active mm.
 670  * @cpu         The CPU we're running on.
 671  *
 672  * Suspend the active_mm, ready for a switch to a KVM guest virtual address
 673  * space. This is left active for the duration of guest context, including time
 674  * with interrupts enabled, so we need to be careful not to confuse e.g. cache
 675  * management IPIs.
 676  *
 677  * kvm_mips_resume_mm() should be called before context switching to a different
 678  * process so we don't need to worry about reference counting.
 679  *
 680  * This needs to be in static kernel code to avoid exporting init_mm.
 681  */
 682 void kvm_mips_suspend_mm(int cpu)
 683 {
 684         cpumask_clear_cpu(cpu, mm_cpumask(current->active_mm));
 685         current->active_mm = &init_mm;
 686 }
 687 EXPORT_SYMBOL_GPL(kvm_mips_suspend_mm);
 688
 689 /**
 690  * kvm_mips_resume_mm() - Resume the current process mm.
 691  * @cpu         The CPU we're running on.
 692  *
 693  * Resume the mm of the current process, after a switch back from a KVM guest
 694  * virtual address space (see kvm_mips_suspend_mm()).
 695  */
 696 void kvm_mips_resume_mm(int cpu)
 697 {
 698         cpumask_set_cpu(cpu, mm_cpumask(current->mm));
 699         current->active_mm = current->mm;
 700 }
 701 EXPORT_SYMBOL_GPL(kvm_mips_resume_mm);